From 4b7558bc0daf6e42eb22ac05c8272e44f8544754 Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Mon, 25 Nov 2024 13:42:47 +0100
Subject: [PATCH 001/147] move test_store here as load_mrt bin

---
 Cargo.toml          |  13 ++-
 src/bin/load_mrt.rs | 269 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 280 insertions(+), 2 deletions(-)
 create mode 100644 src/bin/load_mrt.rs

diff --git a/Cargo.toml b/Cargo.toml
index f91ae5fe..8c4e2f39 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,17 +25,21 @@ rust-version = "1.80"
 [dependencies]
 crossbeam-epoch = "^0.9"
 crossbeam-utils = "^0.8"
+parking_lot_core = "0.9.10"
 
 inetnum   = "0.1"
 log       = "^0.4"
 roaring 	= "0.10.3"
 rotonda-macros = { path = "proc_macros", version = "0.4.0" }
-routecore       = { version = "0.5", features = ["bgp", "bmp", "fsm", "serde"] }
+routecore       = { version = "0.5", features = ["bgp", "bmp", "fsm", "serde", "mrt"] }
 
 ansi_term   = { version = "0.12", optional = true }
 csv         = { version = "1", optional = true }
 rustyline   = { version = "13", optional = true }
-parking_lot_core = "0.9.10"
+clap        = { version = "4.4", optional = true, features = ["derive"] }
+rayon = { version = "1.10", optional = true }
+memmap2 = { version = "0.9", optional = true }
+rand = { version = "^0.8", optional = true }
 
 [dev-dependencies]
 csv         = { version = "1" }
@@ -44,8 +48,13 @@ rand        = "^0.8"
 
 [features]
 cli = ["ansi_term", "rustyline", "csv"]
+mrt = ["clap", "rayon"]
 default = []
 
 [[bin]]
 name = "cli"
 required-features = ["cli"]
+
+[[bin]]
+name = "load_mrt"
+required-features = ["mrt"]
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
new file mode 100644
index 00000000..9c47aa45
--- /dev/null
+++ b/src/bin/load_mrt.rs
@@ -0,0 +1,269 @@
+use std::fmt;
+use std::fs::File;
+use std::path::PathBuf;
+use std::time::Instant;
+
+use clap::Parser;
+use inetnum::addr::Prefix;
+use memmap2::Mmap;
+use rayon::iter::ParallelBridge;
+use rayon::iter::ParallelIterator;
+use rayon::prelude::*;
+use rotonda_store::custom_alloc::UpsertReport;
+use rotonda_store::prelude::multi::PrefixStoreError;
+use rotonda_store::prelude::multi::{MultiThreadedStore, RouteStatus};
+use rotonda_store::PublicRecord;
+use routecore::mrt::MrtFile;
+
+use rand::seq::SliceRandom;
+
+#[derive(Clone, Debug)]
+//struct MyPaMap(PaMap);
+struct MyPaMap(Vec<u8>);
+impl std::fmt::Display for MyPaMap {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?}", self.0)
+    }
+}
+
+impl rotonda_store::Meta for MyPaMap {
+    type Orderable<'a> = u32;
+
+    type TBI = u32;
+
+    fn as_orderable(&self, _tbi: Self::TBI) -> Self::Orderable<'_> {
+        todo!()
+    }
+}
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    /// Enable concurrent route inserts
+    #[arg(short, long, default_value_t = false)]
+    mt: bool,
+
+    /// Prime store by sequentially inserting prefixes first
+    #[arg(short, long, default_value_t = false)]
+    prime: bool,
+
+    /// Enable concurrent priming inserts
+    #[arg(long, default_value_t = false)]
+    mt_prime: bool,
+
+    /// Shuffle prefixes before priming the store. Enables priming.
+    #[arg(short, long, default_value_t = false)]
+    shuffle: bool,
+
+    /// Use the same store for all MRT_FILES
+    #[arg(long, default_value_t = false)]
+    single_store: bool,
+
+    /// MRT files to process.
+    #[arg(required = true)]
+    mrt_files: Vec<PathBuf>,
+
+    /// Don't insert in store, only parse MRT_FILES
+    #[arg(long, default_value_t = false)]
+    parse_only: bool,
+}
+
+fn insert<T: rotonda_store::Meta>(
+    store: &MultiThreadedStore<T>,
+    prefix: &Prefix,
+    mui: u32,
+    ltime: u64,
+    route_status: RouteStatus,
+    value: T,
+) -> Result<UpsertReport, PrefixStoreError> {
+    let record = PublicRecord::new(mui, ltime, route_status, value);
+    store
+        .insert(prefix, record, None)
+        .inspect_err(|e| eprintln!("Error in test_store: {e}"))
+}
+
+fn main() {
+    let args = Cli::parse();
+
+    let mut store = MultiThreadedStore::<MyPaMap>::new().unwrap();
+    let t_total = Instant::now();
+    let mut routes_total: usize = 0;
+    let mut mib_total: usize = 0;
+    for mrtfile in &args.mrt_files {
+        if !args.single_store {
+            store = MultiThreadedStore::<MyPaMap>::new().unwrap();
+        }
+
+        let file = File::open(mrtfile).unwrap();
+        let mmap = unsafe { Mmap::map(&file).unwrap() };
+        println!("{}: {}MiB", mrtfile.to_string_lossy(), mmap.len() >> 20);
+        mib_total += mmap.len() >> 20;
+
+        let t_0 = Instant::now();
+        let t_prev = t_0;
+
+        let mrt_file = MrtFile::new(&mmap[..]);
+        let tables = mrt_file.tables().unwrap();
+
+        if !args.parse_only && (args.shuffle || args.prime || args.mt_prime) {
+            let mut prefixes = mrt_file
+                .tables()
+                .unwrap()
+                .par_bridge()
+                .map(|(_fam, reh)| {
+                    let iter = routecore::mrt::SingleEntryIterator::new(reh);
+                    iter.map(|(prefix, _, _)| prefix)
+                })
+                .flatten_iter()
+                .collect::<Vec<_>>();
+
+            eprintln!(
+                "collected {} prefixes to prime store, took {}ms",
+                prefixes.len(),
+                t_prev.elapsed().as_millis(),
+            );
+
+            if args.shuffle {
+                let t_s = Instant::now();
+                eprint!("shuffling before priming... ");
+                prefixes.shuffle(&mut rand::thread_rng());
+                eprintln!("done! took {}ms", t_s.elapsed().as_millis());
+            }
+
+            let t_prev = Instant::now();
+
+            if args.mt_prime {
+                if prefixes
+                    .par_iter()
+                    .try_for_each(|p| {
+                        insert(
+                            &store,
+                            p,
+                            0,
+                            0,
+                            RouteStatus::InActive,
+                            MyPaMap(vec![]),
+                        )
+                        .map(|_| ())
+                    })
+                    .is_err()
+                {
+                    return;
+                }
+            } else {
+                for p in &prefixes {
+                    if insert(
+                        &store,
+                        p,
+                        0,
+                        0,
+                        RouteStatus::InActive,
+                        MyPaMap(vec![]),
+                    )
+                    .is_err()
+                    {
+                        return;
+                    }
+                }
+            }
+            eprintln!(
+                "primed store with {} prefixes, took {}ms",
+                prefixes.len(),
+                t_prev.elapsed().as_millis(),
+            );
+        }
+
+        let t_prev = Instant::now();
+
+        let mut num_routes = 0;
+        if args.mt {
+            // TODO turn into a try_fold or something, allowing to return
+            // on the first error
+            num_routes = tables
+                .par_bridge()
+                .map(|(_fam, reh)| {
+                    let iter = routecore::mrt::SingleEntryIterator::new(reh);
+
+                    let mut cnt = 0;
+                    for e in iter {
+                        cnt += 1;
+                        let (prefix, peer_idx, pamap) = e;
+                        let mui = peer_idx.into();
+                        let ltime = 0;
+                        let val = MyPaMap(pamap);
+
+                        if !args.parse_only {
+                            let _ = insert(
+                                &store,
+                                &prefix,
+                                mui,
+                                ltime,
+                                RouteStatus::Active,
+                                val,
+                            );
+                        }
+                    }
+                    cnt
+                })
+                .fold(|| 0, |sum, e| sum + e)
+                .sum::<usize>();
+        } else {
+            // single threaded
+            let rib_entries = mrt_file.rib_entries().unwrap();
+
+            for e in rib_entries {
+                num_routes += 1;
+                let (_, peer_idx, _, prefix, pamap) = e;
+                let mui = peer_idx.into();
+                let ltime = 0;
+                let val = MyPaMap(pamap);
+
+                if !args.parse_only {
+                    let _ = insert(
+                        &store,
+                        &prefix,
+                        mui,
+                        ltime,
+                        RouteStatus::Active,
+                        val,
+                    );
+                }
+            }
+        }
+        if !args.parse_only {
+            let threading = if args.mt {
+                " (multi-threaded)"
+            } else {
+                " (single-threaded)"
+            };
+            eprintln!(
+                "inserted {} routes{}, took {}ms, total for this file {}ms",
+                num_routes,
+                threading,
+                t_prev.elapsed().as_millis(),
+                t_0.elapsed().as_millis(),
+            );
+        } else {
+            eprintln!(
+                "parsed {}, no insertions,  total for this file {}ms",
+                num_routes,
+                t_0.elapsed().as_millis(),
+            );
+        }
+        eprintln!("--------");
+
+        routes_total += num_routes;
+    }
+    eprintln!(
+        "Processed {} routes in {} files in {:.2}s",
+        routes_total,
+        args.mrt_files.len(),
+        t_total.elapsed().as_millis() as f64 / 1000.0
+    );
+    eprintln!(
+        "{:.0} routes per second\n\
+            {:.0} MiB per second",
+        routes_total as f64 / t_total.elapsed().as_secs() as f64,
+        mib_total as f64 / t_total.elapsed().as_secs() as f64
+    );
+}

From cf0cb3b17aea808a8df85ee469fe90271c4610d5 Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Mon, 25 Nov 2024 13:48:34 +0100
Subject: [PATCH 002/147] reflow w/ rustfmt

---
 src/local_array/store/custom_alloc.rs | 97 ++++++++++++++-------------
 1 file changed, 51 insertions(+), 46 deletions(-)

diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 5139e1d5..c20f5376 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -644,60 +644,65 @@ impl<
     ) -> Result<UpsertReport, PrefixStoreError> {
         let mut prefix_new = true;
 
-        let (mui_new, insert_retry_count) =
-            match self.non_recursive_retrieve_prefix_mut(prefix) {
-                // There's no StoredPrefix at this location yet. Create a new
-                // PrefixRecord and try to store it in the empty slot.
-                (locked_prefix, false) => {
-                    if log_enabled!(log::Level::Debug) {
-                        debug!(
-                            "{} store: Create new prefix record",
-                            std::thread::current().name().unwrap_or("unnamed-thread")
-                        );
-                    }
-
-                    // We're creating a StoredPrefix without our record first,
-                    // to avoid having to clone it on retry.
-                    let res = locked_prefix
-                        // .get_or_init(|| {
-                        //     StoredPrefix::new::<PB>(
-                        //         PrefixId::new(prefix.get_net(), prefix.get_len()),
-                        //         level,
-                        //     )
-                        // })
-                        // .0
-                        .record_map
-                        .upsert_record(record);
-
-                    self.counters.inc_prefixes_count(prefix.get_len());
-                    res
+        let (mui_new, insert_retry_count) = match self
+            .non_recursive_retrieve_prefix_mut(prefix)
+        {
+            // There's no StoredPrefix at this location yet. Create a new
+            // PrefixRecord and try to store it in the empty slot.
+            (locked_prefix, false) => {
+                if log_enabled!(log::Level::Debug) {
+                    debug!(
+                        "{} store: Create new prefix record",
+                        std::thread::current()
+                            .name()
+                            .unwrap_or("unnamed-thread")
+                    );
                 }
-                // There already is a StoredPrefix with a record at this
-                // location.
-                (stored_prefix, true) => {
-                    if log_enabled!(log::Level::Debug) {
-                        debug!(
+
+                // We're creating a StoredPrefix without our record first,
+                // to avoid having to clone it on retry.
+                let res = locked_prefix
+                    // .get_or_init(|| {
+                    //     StoredPrefix::new::<PB>(
+                    //         PrefixId::new(prefix.get_net(), prefix.get_len()),
+                    //         level,
+                    //     )
+                    // })
+                    // .0
+                    .record_map
+                    .upsert_record(record);
+
+                self.counters.inc_prefixes_count(prefix.get_len());
+                res
+            }
+            // There already is a StoredPrefix with a record at this
+            // location.
+            (stored_prefix, true) => {
+                if log_enabled!(log::Level::Debug) {
+                    debug!(
                         "{} store: Found existing prefix record for {}/{}",
-                        std::thread::current().name().unwrap_or("unnamed-thread"),
+                        std::thread::current()
+                            .name()
+                            .unwrap_or("unnamed-thread"),
                         prefix.get_net(),
                         prefix.get_len()
                     );
-                    }
-                    prefix_new = false;
-
-                    // Update the already existing record_map with our caller's
-                    // record.
-                    stored_prefix.set_ps_outdated(guard)?;
-                    let res = stored_prefix.record_map.upsert_record(record);
+                }
+                prefix_new = false;
 
-                    if let Some(tbi) = update_path_selections {
-                        stored_prefix
-                            .calculate_and_store_best_backup(&tbi, guard)?;
-                    }
+                // Update the already existing record_map with our
+                // caller's record.
+                stored_prefix.set_ps_outdated(guard)?;
+                let res = stored_prefix.record_map.upsert_record(record);
 
-                    res
+                if let Some(tbi) = update_path_selections {
+                    stored_prefix
+                        .calculate_and_store_best_backup(&tbi, guard)?;
                 }
-            };
+
+                res
+            }
+        };
 
         Ok(UpsertReport {
             prefix_new,

From 3d3657665004b422fbfab47c4a70513cc3c35e9e Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Mon, 25 Nov 2024 14:18:14 +0100
Subject: [PATCH 003/147] reflow/cleanup

---
 src/local_array/store/atomic_types.rs |  2 -
 src/local_array/store/custom_alloc.rs | 86 ++++++++++++---------------
 2 files changed, 38 insertions(+), 50 deletions(-)

diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index c9e0c307..d4a3242d 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -460,7 +460,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let mut record_map = c_map.lock().unwrap();
         if let Some(rec) = record_map.get_mut(&mui) {
             rec.status = RouteStatus::Withdrawn;
-            // record_map.insert(mui, rec);
         }
     }
 
@@ -470,7 +469,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let mut r_map = record_map.lock().unwrap();
         if let Some(rec) = r_map.get_mut(&mui) {
             rec.status = RouteStatus::Active;
-            // r_map.insert(mui, rec);
         }
     }
 
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index c20f5376..55785a6c 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -644,42 +644,32 @@ impl<
     ) -> Result<UpsertReport, PrefixStoreError> {
         let mut prefix_new = true;
 
-        let (mui_new, insert_retry_count) = match self
-            .non_recursive_retrieve_prefix_mut(prefix)
-        {
-            // There's no StoredPrefix at this location yet. Create a new
-            // PrefixRecord and try to store it in the empty slot.
-            (locked_prefix, false) => {
-                if log_enabled!(log::Level::Debug) {
-                    debug!(
-                        "{} store: Create new prefix record",
-                        std::thread::current()
-                            .name()
-                            .unwrap_or("unnamed-thread")
-                    );
+        let (mui_new, insert_retry_count) =
+            match self.non_recursive_retrieve_prefix_mut(prefix) {
+                // There's no StoredPrefix at this location yet. Create a new
+                // PrefixRecord and try to store it in the empty slot.
+                (locked_prefix, false) => {
+                    if log_enabled!(log::Level::Debug) {
+                        debug!(
+                            "{} store: Create new prefix record",
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread")
+                        );
+                    }
+
+                    // We're creating a StoredPrefix without our record first,
+                    // to avoid having to clone it on retry.
+                    let res = locked_prefix.record_map.upsert_record(record);
+
+                    self.counters.inc_prefixes_count(prefix.get_len());
+                    res
                 }
-
-                // We're creating a StoredPrefix without our record first,
-                // to avoid having to clone it on retry.
-                let res = locked_prefix
-                    // .get_or_init(|| {
-                    //     StoredPrefix::new::<PB>(
-                    //         PrefixId::new(prefix.get_net(), prefix.get_len()),
-                    //         level,
-                    //     )
-                    // })
-                    // .0
-                    .record_map
-                    .upsert_record(record);
-
-                self.counters.inc_prefixes_count(prefix.get_len());
-                res
-            }
-            // There already is a StoredPrefix with a record at this
-            // location.
-            (stored_prefix, true) => {
-                if log_enabled!(log::Level::Debug) {
-                    debug!(
+                // There already is a StoredPrefix with a record at this
+                // location.
+                (stored_prefix, true) => {
+                    if log_enabled!(log::Level::Debug) {
+                        debug!(
                         "{} store: Found existing prefix record for {}/{}",
                         std::thread::current()
                             .name()
@@ -687,22 +677,22 @@ impl<
                         prefix.get_net(),
                         prefix.get_len()
                     );
-                }
-                prefix_new = false;
+                    }
+                    prefix_new = false;
 
-                // Update the already existing record_map with our
-                // caller's record.
-                stored_prefix.set_ps_outdated(guard)?;
-                let res = stored_prefix.record_map.upsert_record(record);
+                    // Update the already existing record_map with our caller's
+                    // record.
+                    stored_prefix.set_ps_outdated(guard)?;
+                    let res = stored_prefix.record_map.upsert_record(record);
 
-                if let Some(tbi) = update_path_selections {
-                    stored_prefix
-                        .calculate_and_store_best_backup(&tbi, guard)?;
-                }
+                    if let Some(tbi) = update_path_selections {
+                        stored_prefix
+                            .calculate_and_store_best_backup(&tbi, guard)?;
+                    }
 
-                res
-            }
-        };
+                    res
+                }
+            };
 
         Ok(UpsertReport {
             prefix_new,

From cd6851349fcd998b9ab2f49e4ffa9aff89b791f2 Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Wed, 27 Nov 2024 19:53:44 +0100
Subject: [PATCH 004/147] lsm_tree part 1 * lsm_tree dep * persistence_key
 calculation * weave PREFIX_SIZE and KEY_SIZE into CustomAllocStorage *
 PersisTree

---
 Cargo.toml                                   |   4 +-
 examples/full_table_multiple_trees_json.rs   |  19 ++-
 examples/more_specifics.rs                   |  17 +-
 examples/multi_no_thread.rs                  |  25 ++-
 examples/multi_single_thread.rs              |  15 +-
 examples/multi_thread_2.rs                   |  16 +-
 examples/multi_thread_3.rs                   |  87 +++++-----
 examples/multi_thread_4.rs                   |   7 +
 examples/multi_thread_multi_prefix.rs        |   2 +-
 examples/multi_thread_single_prefix.rs       |  12 +-
 examples/numbers_treebitmap.rs               |  21 +--
 examples/real_single_thread_24.rs            |  19 ++-
 examples/single_thread_24.rs                 |  20 +--
 proc_macros/src/lib.rs                       | 153 ++++++++++++++----
 src/af.rs                                    |  58 ++++++-
 src/bin/cli.rs                               |   7 +-
 src/bin/load_mrt.rs                          |  10 ++
 src/local_array/query.rs                     |   3 +-
 src/local_array/store/atomic_types.rs        |  34 ++--
 src/local_array/store/custom_alloc.rs        | 123 ++++++++++++--
 src/local_array/store/default_store.rs       |  24 +--
 src/local_array/store/iterators.rs           |  13 +-
 src/local_array/tree.rs                      | 120 ++++++++++----
 src/local_vec/tests/full_table_single.rs     |  44 ++---
 src/local_vec/tests/more_specifics_single.rs |  22 ++-
 src/meta_examples.rs                         |  33 +++-
 src/prefix_record.rs                         | 129 +++++++++------
 tests/best-path.rs                           | 161 ++++++++++++++-----
 tests/concurrency.rs                         |  73 ++++++---
 tests/full-table.rs                          |  64 +++++---
 tests/more-more-specifics.rs                 |  29 ++--
 tests/more-specifics.rs                      |  22 +--
 tests/treebitmap.rs                          |   7 +-
 tests/treebitmap_v6.rs                       |  14 +-
 34 files changed, 1002 insertions(+), 405 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 8c4e2f39..41e8e1a8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,6 +40,7 @@ clap        = { version = "4.4", optional = true, features = ["derive"] }
 rayon = { version = "1.10", optional = true }
 memmap2 = { version = "0.9", optional = true }
 rand = { version = "^0.8", optional = true }
+lsm-tree = { version = "2.4.0", optional = true }
 
 [dev-dependencies]
 csv         = { version = "1" }
@@ -49,6 +50,7 @@ rand        = "^0.8"
 [features]
 cli = ["ansi_term", "rustyline", "csv"]
 mrt = ["clap", "rayon"]
+persist = ["lsm-tree"]
 default = []
 
 [[bin]]
@@ -57,4 +59,4 @@ required-features = ["cli"]
 
 [[bin]]
 name = "load_mrt"
-required-features = ["mrt"]
+required-features = ["mrt","persist"]
diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index 4b57927b..bbcd3dbb 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -1,15 +1,15 @@
 // extern crate self as roto;
-use rotonda_store::prelude::*;
-use rotonda_store::prelude::multi::*;
 use rotonda_store::meta_examples::PrefixAs;
+use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
 use std::error::Error;
 use std::fs::File;
 use std::process;
 
 #[create_store((
-    [4, 4, 4, 4, 4, 4, 4, 4],
-    [3,4,5,4]
+    ([4, 4, 4, 4, 4, 4, 4, 4], 5, 17),
+    ([3, 4, 5, 4], 17, 29)
 ))]
 struct MyStore;
 
@@ -32,7 +32,12 @@ fn main() -> Result<(), Box<dyn Error>> {
             let asn: u32 = record[2].parse().unwrap();
             let pfx = PrefixRecord::<PrefixAs>::new(
                 Prefix::new(net.into(), len)?,
-                vec![Record::new(0, 0, RouteStatus::Active, PrefixAs(asn))],
+                vec![Record::new(
+                    0,
+                    0,
+                    RouteStatus::Active,
+                    PrefixAs::new(asn.into()),
+                )],
             );
             pfxs.push(pfx);
         }
@@ -85,9 +90,9 @@ fn main() -> Result<(), Box<dyn Error>> {
                                     include_withdrawn: false,
                                     include_less_specifics: false,
                                     include_more_specifics: false,
-                                    mui: None
+                                    mui: None,
                                 },
-                                guard
+                                guard,
                             );
                         }
                     }
diff --git a/examples/more_specifics.rs b/examples/more_specifics.rs
index 78941768..8190ca4f 100644
--- a/examples/more_specifics.rs
+++ b/examples/more_specifics.rs
@@ -1,14 +1,13 @@
 use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::prelude::*;
 use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
-use rotonda_store::AddressFamily;
 use inetnum::addr::Prefix;
+use rotonda_store::AddressFamily;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // type StoreType = InMemStorage<u32, PrefixAs>;
-    let tree_bitmap =
-        MultiThreadedStore::<PrefixAs>::new()?;
+    let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
     let pfxs = vec![
         Prefix::new_relaxed(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
@@ -215,7 +214,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     for pfx in pfxs.into_iter() {
         // println!("insert {:?}", pfx);
         let p: Prefix = pfx.unwrap();
-        tree_bitmap.insert(&p, Record::new(0,0, RouteStatus::Active, PrefixAs(666)), None)?;
+        tree_bitmap.insert(
+            &p,
+            Record::new(0, 0, RouteStatus::Active, PrefixAs::new(666.into())),
+            None,
+        )?;
     }
     println!("------ end of inserts\n");
     // println!(
@@ -284,9 +287,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_withdrawn: false,
                 include_less_specifics: true,
                 include_more_specifics: true,
-                mui: None
+                mui: None,
             },
-            guard
+            guard,
         );
         println!("em/m-s: {:#?}", s_spfx);
         println!("-----------");
diff --git a/examples/multi_no_thread.rs b/examples/multi_no_thread.rs
index d3fe3aff..0263de77 100644
--- a/examples/multi_no_thread.rs
+++ b/examples/multi_no_thread.rs
@@ -1,8 +1,8 @@
 use log::trace;
 
-use rotonda_store::prelude::*;
-use rotonda_store::prelude::multi::*;
 use rotonda_store::meta_examples::PrefixAs;
+use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
@@ -22,14 +22,27 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     loop {
         x += 1;
         // print!("{}-", i);
-        match tree_bitmap.insert(&pfx.unwrap(), Record::new(0, 0, RouteStatus::Active, PrefixAs(x % 1000)), None) {
+        match tree_bitmap.insert(
+            &pfx.unwrap(),
+            Record::new(
+                0,
+                0,
+                RouteStatus::Active,
+                PrefixAs::new((x % 1000).into()),
+            ),
+            None,
+        ) {
             Ok(_) => {}
             Err(e) => {
                 println!("{}", e);
             }
         };
-        if (x % 1_000_000) == 0 { println!("inserts: {}", x); }
-        if x == 100_000_000 { break; }
+        if (x % 1_000_000) == 0 {
+            println!("inserts: {}", x);
+        }
+        if x == 100_000_000 {
+            break;
+        }
     }
     println!("--thread {} done.", 1);
 
@@ -44,7 +57,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_withdrawn: true,
             include_less_specifics: true,
             include_more_specifics: true,
-            mui: None
+            mui: None,
         },
         guard,
     );
diff --git a/examples/multi_single_thread.rs b/examples/multi_single_thread.rs
index 4e9f4765..f49d4b02 100644
--- a/examples/multi_single_thread.rs
+++ b/examples/multi_single_thread.rs
@@ -1,11 +1,11 @@
 use log::trace;
-use std::time::Duration;
 use std::thread;
+use std::time::Duration;
 
 use rand::Rng;
 
-use rotonda_store::prelude::*;
 use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
 use rotonda_store::meta_examples::PrefixAs;
 
@@ -31,7 +31,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .name(1_u8.to_string())
         .spawn(move || -> Result<(), Box<dyn std::error::Error + Send>> {
             // while !start_flag.load(std::sync::atomic::Ordering::Acquire) {
-            let mut rng= rand::thread_rng();
+            let mut rng = rand::thread_rng();
 
             println!("park thread {}", 1);
             thread::park();
@@ -45,8 +45,13 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let asn: u32 = rng.gen();
                 match tree_bitmap.insert(
                     &pfx.unwrap(),
-                    Record::new(0, 0, RouteStatus::Active, PrefixAs(asn)),
-                    None
+                    Record::new(
+                        0,
+                        0,
+                        RouteStatus::Active,
+                        PrefixAs::new(asn.into()),
+                    ),
+                    None,
                 ) {
                     Ok(_) => {}
                     Err(e) => {
diff --git a/examples/multi_thread_2.rs b/examples/multi_thread_2.rs
index 4fe85aba..666740ec 100644
--- a/examples/multi_thread_2.rs
+++ b/examples/multi_thread_2.rs
@@ -2,8 +2,8 @@ use log::trace;
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
-use rotonda_store::prelude::*;
 use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
 use rotonda_store::meta_examples::PrefixAs;
 
@@ -32,13 +32,21 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                     thread::park();
                 }
 
-                match tree_bitmap.insert(&pfx.unwrap(), Record::new(0, 0, RouteStatus::Active, PrefixAs(i as u32)), None) {
+                match tree_bitmap.insert(
+                    &pfx.unwrap(),
+                    Record::new(
+                        0,
+                        0,
+                        RouteStatus::Active,
+                        PrefixAs::new((i as u32).into()),
+                    ),
+                    None,
+                ) {
                     Ok(_) => {}
                     Err(e) => {
                         println!("{}", e);
                     }
                 };
-
             })
             .unwrap()
     });
@@ -61,7 +69,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_withdrawn: true,
             include_less_specifics: true,
             include_more_specifics: true,
-            mui: None
+            mui: None,
         },
         guard,
     );
diff --git a/examples/multi_thread_3.rs b/examples/multi_thread_3.rs
index 2dcc85cc..0c2165c4 100644
--- a/examples/multi_thread_3.rs
+++ b/examples/multi_thread_3.rs
@@ -2,8 +2,8 @@ use log::trace;
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
-use rotonda_store::prelude::*;
 use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
 use rotonda_store::meta_examples::PrefixAs;
 
@@ -20,45 +20,60 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         32,
     );
 
-    let threads =
-        (0..16).enumerate().map(|(i, _)| {
-            let tree_bitmap = tree_bitmap.clone();
-            // let start_flag = Arc::clone(&f);
+    let threads = (0..16).enumerate().map(|(i, _)| {
+        let tree_bitmap = tree_bitmap.clone();
+        // let start_flag = Arc::clone(&f);
 
-            std::thread::Builder::new().name(i.to_string()).spawn(
-            move || -> Result<(), Box<dyn std::error::Error + Send>> {
-                // while !start_flag.load(std::sync::atomic::Ordering::Acquire) {
+        std::thread::Builder::new()
+            .name(i.to_string())
+            .spawn(
+                move || -> Result<(), Box<dyn std::error::Error + Send>> {
+                    // while !start_flag.load(std::sync::atomic::Ordering::Acquire) {
                     println!("park thread {}", i);
                     thread::park();
-                // }
-
-                print!("\nstart {} ---", i);
-                let mut x = 0;
-                loop {
-                    x += 1;
-                    // print!("{}-", i);
-                    match tree_bitmap
-                        .insert(
+                    // }
+
+                    print!("\nstart {} ---", i);
+                    let mut x = 0;
+                    loop {
+                        x += 1;
+                        // print!("{}-", i);
+                        match tree_bitmap.insert(
                             &pfx.unwrap(),
-                            Record::new(0,0, RouteStatus::Active, PrefixAs(i as u32)),
-                            None
-                        )
-                    {
-                        Ok(metrics) => {
-                            if metrics.cas_count > 0  {
-                                println!("{} {:?} {:?} retry count {},", std::thread::current().name().unwrap(), metrics, pfx, metrics.cas_count);
+                            Record::new(
+                                0,
+                                0,
+                                RouteStatus::Active,
+                                PrefixAs::new((i as u32).into()),
+                            ),
+                            None,
+                        ) {
+                            Ok(metrics) => {
+                                if metrics.cas_count > 0 {
+                                    println!(
+                                        "{} {:?} {:?} retry count {},",
+                                        std::thread::current()
+                                            .name()
+                                            .unwrap(),
+                                        metrics,
+                                        pfx,
+                                        metrics.cas_count
+                                    );
+                                }
                             }
+                            Err(e) => {
+                                println!("{}", e);
+                            }
+                        };
+                        if x % 1_000_000 == 0 {
+                            println!("{}", x);
                         }
-                        Err(e) => {
-                            println!("{}", e);
-                        }
-                    };
-                    if x % 1_000_000 == 0 { println!("{}", x); }
-                }
-                // println!("--thread {} done.", i);
-            },
-        ).unwrap()
-        });
+                    }
+                    // println!("--thread {} done.", i);
+                },
+            )
+            .unwrap()
+    });
 
     // thread::sleep(Duration::from_secs(60));
 
@@ -80,14 +95,14 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_withdrawn: true,
             include_less_specifics: true,
             include_more_specifics: true,
-            mui: None
+            mui: None,
         },
         guard,
     );
     println!("query result");
     println!("{}", s_spfx);
     println!("{}", s_spfx.more_specifics.unwrap());
-   
+
     println!("-----------");
 
     Ok(())
diff --git a/examples/multi_thread_4.rs b/examples/multi_thread_4.rs
index 8283695c..342019d5 100644
--- a/examples/multi_thread_4.rs
+++ b/examples/multi_thread_4.rs
@@ -50,6 +50,13 @@ impl std::fmt::Display for ComplexPrefixAs {
         write!(f, "AS{:?}", self.0)
     }
 }
+
+impl AsRef<[u8]> for ComplexPrefixAs {
+    fn as_ref(&self) -> &[u8] {
+        todo!()
+    }
+}
+
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
     env_logger::init();
diff --git a/examples/multi_thread_multi_prefix.rs b/examples/multi_thread_multi_prefix.rs
index d9385fe3..251bfdac 100644
--- a/examples/multi_thread_multi_prefix.rs
+++ b/examples/multi_thread_multi_prefix.rs
@@ -45,7 +45,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                         let pfx = Prefix::new_relaxed(pfx_int.clone().load(std::sync::atomic::Ordering::Relaxed).into_ipaddr(), 32).unwrap();
                         let guard = &crossbeam_epoch::pin();
                         while x < 100 {
-                            let asn = PrefixAs(rng.gen());
+                            let asn = PrefixAs::new_from_u32(rng.gen());
                             match tree_bitmap.insert(
                                 &pfx,
                                 Record::new(0, 0, RouteStatus::Active, asn),
diff --git a/examples/multi_thread_single_prefix.rs b/examples/multi_thread_single_prefix.rs
index c727a67b..07cc79ec 100644
--- a/examples/multi_thread_single_prefix.rs
+++ b/examples/multi_thread_single_prefix.rs
@@ -7,9 +7,9 @@ use std::time::Duration;
 
 use rand::Rng;
 
+use rotonda_store::meta_examples::PrefixAs;
 use rotonda_store::prelude::*;
 use rotonda_store::MultiThreadedStore;
-use rotonda_store::meta_examples::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
@@ -41,11 +41,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                     loop {
                         let guard = &crossbeam_epoch::pin();
                         while x < 10_000 {
-                            let asn = PrefixAs(rng.gen());
+                            let asn = PrefixAs::new_from_u32(rng.gen());
                             match tree_bitmap.insert(
                                 &pfx.unwrap(),
                                 Record::new(0, 0, RouteStatus::Active, asn),
-                                None
+                                None,
                             ) {
                                 Ok(metrics) => {
                                     if metrics.prefix_new {
@@ -78,9 +78,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                         guard.flush();
                         thread::sleep(Duration::from_secs(3));
                         println!("wake thread {}", i);
-                        println!("prefix count {:?}", tree_bitmap.prefixes_count());
+                        println!(
+                            "prefix count {:?}",
+                            tree_bitmap.prefixes_count()
+                        );
                         x = 0;
-
                     }
                 },
             )
diff --git a/examples/numbers_treebitmap.rs b/examples/numbers_treebitmap.rs
index 5bee5fb8..0d6a5859 100644
--- a/examples/numbers_treebitmap.rs
+++ b/examples/numbers_treebitmap.rs
@@ -1,18 +1,18 @@
 use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::prelude::*;
 use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
-use std::sync::atomic::Ordering;
 use std::env;
 use std::error::Error;
 use std::ffi::OsString;
 use std::fs::File;
 use std::net::{IpAddr, Ipv4Addr};
 use std::process;
+use std::sync::atomic::Ordering;
 
 #[create_store((
-    [4, 4, 4, 4, 4, 4, 4, 4],
-    [3, 4, 5, 4]
+    ([4, 4, 4, 4, 4, 4, 4, 4], 5, 17),
+    ([3, 4, 5, 4], 16, 29)
 ))]
 struct MyStore<PrefixAs>;
 
@@ -44,7 +44,12 @@ fn load_prefixes(
         let asn: u32 = record[2].parse().unwrap();
         let pfx = PrefixRecord::<PrefixAs>::new(
             Prefix::new(net, len)?,
-            vec![Record::new(0, 0, RouteStatus::Active, PrefixAs(asn))],
+            vec![Record::new(
+                0,
+                0,
+                RouteStatus::Active,
+                PrefixAs::new_from_u32(asn),
+            )],
         );
         pfxs.push(pfx);
         // trie.insert(&pfx);
@@ -66,11 +71,9 @@ fn main() -> Result<(), Box<dyn Error>> {
         }
 
         for pfx in pfxs.into_iter() {
-            tree_bitmap.insert(
-                &pfx.prefix, pfx.meta[0].clone(), None
-            )?;
+            tree_bitmap.insert(&pfx.prefix, pfx.meta[0].clone(), None)?;
         }
-        
+
         #[cfg(feature = "cli")]
         println!("{:?}", tree_bitmap.print_funky_stats());
     }
diff --git a/examples/real_single_thread_24.rs b/examples/real_single_thread_24.rs
index de196753..b0a2b1a3 100644
--- a/examples/real_single_thread_24.rs
+++ b/examples/real_single_thread_24.rs
@@ -1,6 +1,6 @@
 use log::trace;
-use std::time::Duration;
 use std::thread;
+use std::time::Duration;
 
 use rand::Rng;
 
@@ -15,14 +15,15 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     trace!("Starting one-threaded yolo testing....");
     let v4 = vec![8];
     let v6 = vec![8];
-    let mut tree_bitmap = rotonda_store::SingleThreadedStore::<PrefixAs>::new(v4, v6);
+    let mut tree_bitmap =
+        rotonda_store::SingleThreadedStore::<PrefixAs>::new(v4, v6);
 
     let mut pfx_int = 0_u32;
 
     let thread = std::thread::Builder::new()
         .name(1_u8.to_string())
         .spawn(move || -> Result<(), Box<dyn std::error::Error + Send>> {
-            let mut rng= rand::thread_rng();
+            let mut rng = rand::thread_rng();
 
             println!("park thread {}", 1);
             thread::park();
@@ -31,14 +32,13 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
             while pfx_int <= 24 {
                 pfx_int += 1;
-                let pfx = Prefix::new_relaxed(
-                    pfx_int.into_ipaddr(),
-                    32,
-                );
+                let pfx = Prefix::new_relaxed(pfx_int.into_ipaddr(), 32);
 
                 print!("{}-", pfx_int);
                 let asn: u32 = rng.gen();
-                match tree_bitmap.insert(&pfx.unwrap(), PrefixAs(asn)) {
+                match tree_bitmap
+                    .insert(&pfx.unwrap(), PrefixAs::new_from_u32(asn))
+                {
                     Ok(_) => {}
                     Err(e) => {
                         println!("{}", e);
@@ -49,10 +49,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             println!("--thread {} done.", 1);
 
             Ok(())
-
         })
         .unwrap();
- 
+
     thread.thread().unpark();
     thread::sleep(Duration::from_secs(10));
 
diff --git a/examples/single_thread_24.rs b/examples/single_thread_24.rs
index d0308456..71c03466 100644
--- a/examples/single_thread_24.rs
+++ b/examples/single_thread_24.rs
@@ -1,11 +1,11 @@
 use log::trace;
-use std::time::Duration;
 use std::thread;
+use std::time::Duration;
 
 use rand::Rng;
 
-use rotonda_store::prelude::*;
 use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
 use rotonda_store::meta_examples::PrefixAs;
 
@@ -28,7 +28,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .name(1_u8.to_string())
         .spawn(move || -> Result<(), Box<dyn std::error::Error + Send>> {
             // while !start_flag.load(std::sync::atomic::Ordering::Acquire) {
-            let mut rng= rand::thread_rng();
+            let mut rng = rand::thread_rng();
 
             println!("park thread {}", 1);
             thread::park();
@@ -38,17 +38,19 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             // let mut x = 0;
             while pfx_int <= 24 {
                 pfx_int += 1;
-                let pfx = Prefix::new_relaxed(
-                    pfx_int.into_ipaddr(),
-                    32,
-                );
+                let pfx = Prefix::new_relaxed(pfx_int.into_ipaddr(), 32);
                 // x += 1;
                 // print!("{}-", i);
                 let asn: u32 = rng.gen();
                 match tree_bitmap.insert(
                     &pfx.unwrap(),
-                    Record::new(0, 0, RouteStatus::Active, PrefixAs(asn)),
-                    None
+                    Record::new(
+                        0,
+                        0,
+                        RouteStatus::Active,
+                        PrefixAs::new_from_u32(asn),
+                    ),
+                    None,
                 ) {
                     Ok(_) => {}
                     Err(e) => {
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index f3f07edf..49acac5a 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -8,18 +8,42 @@ use std::iter::Iterator;
 use syn::parse_macro_input;
 
 #[proc_macro_attribute]
-pub fn stride_sizes(attr: TokenStream, input: TokenStream) -> TokenStream {
+pub fn stride_sizes(
+    attr: TokenStream,
+    struct_def: TokenStream,
+) -> TokenStream {
     // The arguments for the macro invocation
     let attrs = parse_macro_input!(attr as syn::ExprTuple);
 
     let attrs = attrs.elems.iter().collect::<Vec<_>>();
 
-    let input = parse_macro_input!(input as syn::ItemStruct);
-    let type_name = &input.ident;
-    let ip_af = match attrs[0] {
+    let struct_def = parse_macro_input!(struct_def as syn::ItemStruct);
+    let type_name = &struct_def.ident;
+
+    let ip_af = match attrs
+        .first()
+        .unwrap_or_else(|| panic!("Missing Address Family"))
+    {
         syn::Expr::Path(t) => t,
-        _ => panic!("Expected Family Type"),
+        _ => panic!("Expected Adress Family Type"),
+    };
+
+    let prefix_size = match attrs
+        .get(2)
+        .unwrap_or_else(|| panic!("Missing Prefix Size for Address Family"))
+    {
+        syn::Expr::Lit(l) => l,
+        l => panic!("Expected Prefix Size for Address Family, got {:?}", l),
+    };
+
+    let key_size = match attrs
+        .get(3)
+        .unwrap_or_else(|| panic!("Missing Key Size for Address Family"))
+    {
+        syn::Expr::Lit(l) => l,
+        l => panic!("Expected Key Size for Address Family, got {:?}", l),
     };
+
     let prefixes_all_len;
     let all_len;
     let prefixes_buckets_name: syn::Ident;
@@ -304,7 +328,7 @@ pub fn stride_sizes(attr: TokenStream, input: TokenStream) -> TokenStream {
     };
 
     let type_alias = quote! {
-        type #type_name<M> = TreeBitMap<#ip_af, M, #buckets_name<#ip_af>, #prefixes_buckets_name<#ip_af, M>>;
+        type #type_name<M> = TreeBitMap<#ip_af, M, #buckets_name<#ip_af>, #prefixes_buckets_name<#ip_af, M>, #prefix_size, #key_size>;
     };
 
     let result = quote! {
@@ -323,7 +347,9 @@ pub fn stride_sizes(attr: TokenStream, input: TokenStream) -> TokenStream {
 // PrefixStore. Therefore all methods defined in here should be public.
 
 /// Creates a new, user-named struct with user-defined specified stride sizes
-/// that can used as a store type.
+/// that can used as a store type. The size of the prefix and the total key in
+/// the persisted storage should als be included, although these should
+/// probably not be changed from their defaults.
 ///
 /// # Usage
 /// ```ignore
@@ -331,21 +357,29 @@ pub fn stride_sizes(attr: TokenStream, input: TokenStream) -> TokenStream {
 /// use rotonda_store::prelude::multi::*;
 /// use rotonda_store::meta_examples::PrefixAs;
 ///
-/// const IP4_STRIDE_ARRAY = [4; 8];
-/// const IP6_STRIDE_ARRAY = [4; 32];
+/// const IP4_STRIDE_ARRAY: [u32; 8] = [4; 8];
+/// const IP6_STRIDE_ARRAY: [u32; 32] = [4; 32];
 ///
-/// #[create_store((IPV4_STRIDE_ARRAY, IPV6_STRIDE_ARRAY))]
+/// #[create_store(((IPV4_STRIDE_ARRAY, 5, 17), (IPV6_STRIDE_ARRAY, 17, 29)))]
 /// struct NuStorage;
 /// ```
 ///
 /// This will create a `NuStorage` struct, that can be used as a regular
 /// store.
 ///
-/// The stride-sizes can be any of \[3,4,5\], and they should add up
-/// to the total number of bits in the address family (32 for IPv4 and
-/// 128 for IPv6). Stride sizes in the array will be repeated if the sum
-/// of them falls short of the total number of bits for the address
-/// family.
+/// The stride-sizes can be any of \[3,4,5\], and they should add up to the
+/// total number of bits in the address family (32 for IPv4 and 128 for IPv6).
+/// Stride sizes in the array will be repeated if the sum of them falls short
+/// of the total number of bits for the address family.i
+///
+/// The numbers 5, 17 after the first array for IPv4, and the numbers 17,
+/// 29 after the second array, represent the number of bytes a prefix in the
+/// key of a record in the persisted storage (disk), and the total number of
+/// bytes of the key of a record in the persisted storage. An IPv4 prefix is
+/// therefore 5 bytes (address part + prefix length), and 17 bytes for IPv6.
+/// The total number of bytes of the key is calculated thus: prefix (5 or 17)
+/// + multi_unique_id (4 bytes) + logical time of reception of the PDU into
+/// Rotonda (8 bytes).
 ///
 /// # Example
 /// ```ignore
@@ -355,23 +389,68 @@ pub fn stride_sizes(attr: TokenStream, input: TokenStream) -> TokenStream {
 ///
 /// // The default stride sizes for IPv4, IPv6, resp.
 /// #[create_store((
-///     [5, 5, 4, 3, 3, 3, 3, 3, 3, 3],
-///     [4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-///     4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]
+///     ([5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 5, 17),
+///     ([4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
+///     4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], 17, 29)
 /// ))]
 /// struct NuStore;
 ///
 /// let store = Arc::new(NuStore::<PrefixAs>::new().unwrap());
 /// ```
 #[proc_macro_attribute]
-pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
-    let input = parse_macro_input!(item as syn::ItemStruct);
-    let store_name = &input.ident;
+pub fn create_store(
+    attr: TokenStream,
+    struct_def: TokenStream,
+) -> TokenStream {
+    let struct_def = parse_macro_input!(struct_def as syn::ItemStruct);
+    let store_name = &struct_def.ident;
 
     let attr = parse_macro_input!(attr as syn::ExprTuple);
     let attrs = attr.elems.iter().collect::<Vec<_>>();
-    let strides4 = attrs[0].clone();
-    let strides6 = attrs[1].clone();
+
+    let tuple_4 = attrs
+        .first()
+        .unwrap_or_else(|| panic!("No tuple ([u8], usize) for IPv4 defined"));
+    let tuple_4 = match tuple_4 {
+        syn::Expr::Tuple(t) => t,
+        t => panic!("Expected tuple ([u8], usize), got {:?}", t),
+    };
+
+    let tuple_6 = attrs
+        .get(1)
+        .unwrap_or_else(|| panic!("No tuple ([u8], usize) for IPv6 defined"));
+    let tuple_6 = match tuple_6 {
+        syn::Expr::Tuple(t) => t,
+        t => panic!("Expected tuple ([u8], usize), got {:?}", t),
+    };
+
+    let strides4 = tuple_4.elems.first().unwrap_or_else(|| {
+        panic!(
+            "Expected stride sizes array for IPv4, got {:?}",
+            tuple_4.attrs
+        )
+    });
+    let strides6 = tuple_6.elems.first().unwrap_or_else(|| {
+        panic!(
+            "Expected stride sizes array for IPv6, got {:?}",
+            tuple_6.attrs
+        )
+    });
+
+    let key_size4 = tuple_4.elems.get(1).unwrap_or_else(|| {
+        panic!("Expected Key Size for IPv4, got {:?}", tuple_4.elems)
+    });
+    let key_size6 = tuple_6.elems.get(1).unwrap_or_else(|| {
+        panic!("Expected Key Size for IPv6, got {:?}", tuple_6.attrs)
+    });
+
+    let prefix_size4 = tuple_4.elems.get(2).unwrap_or_else(|| {
+        panic!("Expected Prefix Size for IPv4, got {:?}", tuple_4.elems)
+    });
+    let prefix_size6 = tuple_6.elems.get(2).unwrap_or_else(|| {
+        panic!("Expected Prefix Size for IPv6, got {:?}", tuple_6.attrs)
+    });
+
     let strides4_name = format_ident!("{}IPv4", store_name);
     let strides6_name = format_ident!("{}IPv6", store_name);
 
@@ -379,10 +458,10 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
         use ::std::marker::PhantomData;
         use ::inetnum::addr::Prefix;
 
-        #[stride_sizes((IPv4, #strides4))]
+        #[stride_sizes((IPv4, #strides4, #key_size4, #prefix_size4))]
         struct #strides4_name;
 
-        #[stride_sizes((IPv6, #strides6))]
+        #[stride_sizes((IPv6, #strides6, #key_size6, #prefix_size6))]
         struct #strides6_name;
     };
 
@@ -560,7 +639,7 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
             ///
             /// store.insert(
             ///     &Prefix::new(pfx_addr, 22).unwrap(),
-            ///     Record::new(0, 0, RouteStatus::Active, PrefixAs(211321)),
+            ///     Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321)),
             ///     None
             /// );
             ///
@@ -576,7 +655,7 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
             ///     guard
             /// );
             ///
-            /// assert_eq!(res.prefix_meta[0].meta.0, 211321);
+            /// assert_eq!(res.prefix_meta[0].meta.asn(), 211321.into());
             ///
             /// let res = store.match_prefix(
             ///     &Prefix::new(pfx_addr, 24).unwrap(),
@@ -874,7 +953,7 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
             ///
             /// store.insert(
             ///     &Prefix::new(pfx_addr, 22).unwrap(),
-            ///     Record::new(0, 0, RouteStatus::Active, PrefixAs(211321)),
+            ///     Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321)),
             ///     None
             /// );
             ///
@@ -884,7 +963,7 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
             ///     false,
             ///     &guard
             /// ) {
-            ///    assert_eq!(prefix_record.meta[0].meta.0, 211321);
+            ///    assert_eq!(prefix_record.meta[0].meta.asn(), 211321.into());
             /// }
             /// ```
             pub fn less_specifics_iter_from(&'a self,
@@ -962,7 +1041,7 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
             ///
             /// store.insert(
             ///     &Prefix::new(pfx_addr, 24).unwrap(),
-            ///     Record::new(0, 0, RouteStatus::Active, PrefixAs(211321)),
+            ///     Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321)),
             ///     None
             /// );
             ///
@@ -972,7 +1051,7 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
             ///     false,
             ///     &guard
             /// ) {
-            ///    assert_eq!(prefix_record.meta[0].meta.0, 211321);
+            ///    assert_eq!(prefix_record.meta[0].meta.asn(), 211321.into());
             /// }
             /// ```
             pub fn more_specifics_iter_from(&'a self,
@@ -1157,7 +1236,7 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
             /// let pfx_addr = "185.49.140.0".parse::<Ipv4Addr>()
             ///         .unwrap()
             ///         .into();
-            /// let our_asn = Record::new(0, 0, RouteStatus::Active, PrefixAs(211321));
+            /// let our_asn = Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321));
             ///
             /// store.insert(&Prefix::new(pfx_addr, 22).unwrap(), our_asn.clone(), None);
             /// store.insert(&Prefix::new(pfx_addr, 23).unwrap(), our_asn.clone(), None);
@@ -1208,7 +1287,7 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
             /// let pfx_addr = "185.49.140.0".parse::<Ipv4Addr>()
             ///         .unwrap()
             ///         .into();
-            /// let our_asn = Record::new(0, 0, RouteStatus::Active, PrefixAs(211321));
+            /// let our_asn = Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321));
             ///
             /// store.insert(&Prefix::new(pfx_addr, 22).unwrap(), our_asn.clone(), None);
             /// store.insert(&Prefix::new(pfx_addr, 23).unwrap(), our_asn.clone(), None);
@@ -1255,7 +1334,7 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
             /// let pfx_addr = "2a04:b900::".parse::<Ipv6Addr>()
             ///         .unwrap()
             ///         .into();
-            /// let our_asn = Record::new(0, 0, RouteStatus::Active, PrefixAs(211321));
+            /// let our_asn = Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321));
             ///
             /// store.insert(&Prefix::new(pfx_addr, 29).unwrap(), our_asn.clone(), None);
             /// store.insert(&Prefix::new(pfx_addr, 48).unwrap(), our_asn.clone(), None);
@@ -1546,6 +1625,12 @@ pub fn create_store(attr: TokenStream, item: TokenStream) -> TokenStream {
                     v6: self.v6.store.counters.get_prefix_stats(),
                 }
             }
+
+            /// Persist to disk
+            pub fn flush_to_disk(&self) {
+                self.v4.store.persistence.flush_to_disk();
+                self.v6.store.persistence.flush_to_disk();
+            }
         }
     };
 
diff --git a/src/af.rs b/src/af.rs
index 5510825c..9366b8b0 100644
--- a/src/af.rs
+++ b/src/af.rs
@@ -27,6 +27,7 @@ pub trait AddressFamily:
     const BITMASK: Self;
     /// The number of bits in the byte representation of the family.
     const BITS: u8;
+
     fn fmt_net(net: Self) -> String;
     // returns the specified nibble from `start_bit` to (and including)
     // `start_bit + len` and shifted to the right.
@@ -52,6 +53,15 @@ pub trait AddressFamily:
     // in debug mode. A failed check will simply retutrn zero. Used in
     // finding node_ids (always zero for 0/0).
     fn checked_shr_or_zero(self, rhs: u32) -> Self;
+
+    fn as_prefix_bytes<const PREFIX_SIZE: usize>(
+        &self,
+        pfx_len: u8,
+    ) -> [u8; PREFIX_SIZE];
+
+    fn from_prefix_bytes<const PREFIX_SIZE: usize>(
+        value: [u8; PREFIX_SIZE],
+    ) -> (Self, u8);
 }
 
 //-------------- Ipv4 Type --------------------------------------------------
@@ -116,8 +126,7 @@ impl AddressFamily for IPv4 {
     /// let (new_prefix, new_len) = prefix.add_nibble(30, nibble, 7);
     /// ```
     fn add_nibble(self, len: u8, nibble: u32, nibble_len: u8) -> (u32, u8) {
-        let res =
-            self | (nibble << (32 - len - nibble_len) as usize);
+        let res = self | (nibble << (32 - len - nibble_len) as usize);
         (res, len + nibble_len)
     }
 
@@ -150,6 +159,25 @@ impl AddressFamily for IPv4 {
     fn checked_shr_or_zero(self, rhs: u32) -> Self {
         self.checked_shr(rhs).unwrap_or(0)
     }
+
+    fn as_prefix_bytes<const PREFIX_SIZE: usize>(
+        &self,
+        pfx_len: u8,
+    ) -> [u8; PREFIX_SIZE] {
+        let bytes = &mut [0_u8; PREFIX_SIZE];
+        *bytes.first_chunk_mut::<4>().unwrap() = self.to_le_bytes();
+        bytes[PREFIX_SIZE - 1] = pfx_len;
+        *bytes
+    }
+
+    fn from_prefix_bytes<const PREFIX_SIZE: usize>(
+        value: [u8; PREFIX_SIZE],
+    ) -> (Self, u8) {
+        (
+            u32::from_le_bytes(*value.first_chunk::<4>().unwrap()),
+            value[PREFIX_SIZE - 1],
+        )
+    }
 }
 
 //-------------- Ipv6 Type --------------------------------------------------
@@ -160,6 +188,7 @@ pub type IPv6 = u128;
 impl AddressFamily for IPv6 {
     const BITMASK: u128 = 0x1u128.rotate_right(1);
     const BITS: u8 = 128;
+
     fn fmt_net(net: Self) -> String {
         std::net::Ipv6Addr::from(net).to_string()
     }
@@ -190,7 +219,7 @@ impl AddressFamily for IPv6 {
     /// # Panics only in debug mode!
     ///
     /// In release mode this will be UB (Undefined Behaviour)!
-    /// 
+    ///
     /// Will panic if there is insufficient space to add the given nibble,
     /// i.e. if `len + nibble_len >= 128`.
     ///
@@ -202,8 +231,8 @@ impl AddressFamily for IPv6 {
     /// let (new_prefix, new_len) = prefix.add_nibble(112, nibble, 32);
     /// ```
     fn add_nibble(self, len: u8, nibble: u32, nibble_len: u8) -> (Self, u8) {
-        let res = self
-            | ((nibble as u128) << (128 - len - nibble_len) as usize);
+        let res =
+            self | ((nibble as u128) << (128 - len - nibble_len) as usize);
         (res, len + nibble_len)
     }
 
@@ -266,6 +295,25 @@ impl AddressFamily for IPv6 {
     fn checked_shr_or_zero(self, rhs: u32) -> Self {
         self.checked_shr(rhs).unwrap_or(0)
     }
+
+    fn as_prefix_bytes<const PREFIX_SIZE: usize>(
+        &self,
+        pfx_len: u8,
+    ) -> [u8; PREFIX_SIZE] {
+        let res = &mut [0_u8; PREFIX_SIZE];
+        *res.first_chunk_mut::<16>().unwrap() = self.to_le_bytes();
+        res[PREFIX_SIZE - 1] = pfx_len;
+        *res
+    }
+
+    fn from_prefix_bytes<const PREFIX_SIZE: usize>(
+        value: [u8; PREFIX_SIZE],
+    ) -> (Self, u8) {
+        (
+            u128::from_le_bytes(*value.first_chunk::<16>().unwrap()),
+            value[PREFIX_SIZE - 1],
+        )
+    }
 }
 
 // ----------- Zero Trait ---------------------------------------------------
diff --git a/src/bin/cli.rs b/src/bin/cli.rs
index df2abb1d..edb71951 100644
--- a/src/bin/cli.rs
+++ b/src/bin/cli.rs
@@ -40,7 +40,12 @@ fn load_prefixes(
         let asn: u32 = record[2].parse().unwrap();
         let pfx = PrefixRecord::new(
             Prefix::new(ip, len)?,
-            vec![Record::new(0, 0, RouteStatus::Active, PrefixAs(asn))],
+            vec![Record::new(
+                0,
+                0,
+                RouteStatus::Active,
+                PrefixAs::new_from_u32(asn),
+            )],
         );
 
         // let ip: Vec<_> = record[0]
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index 9c47aa45..1e20f19a 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -20,12 +20,19 @@ use rand::seq::SliceRandom;
 #[derive(Clone, Debug)]
 //struct MyPaMap(PaMap);
 struct MyPaMap(Vec<u8>);
+
 impl std::fmt::Display for MyPaMap {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{:?}", self.0)
     }
 }
 
+impl AsRef<[u8]> for MyPaMap {
+    fn as_ref(&self) -> &[u8] {
+        self.0.as_ref()
+    }
+}
+
 impl rotonda_store::Meta for MyPaMap {
     type Orderable<'a> = u32;
 
@@ -254,6 +261,9 @@ fn main() {
 
         routes_total += num_routes;
     }
+
+    store.flush_to_disk();
+
     eprintln!(
         "Processed {} routes in {} files in {:.2}s",
         routes_total,
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 8411a04f..dd16a130 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -19,7 +19,8 @@ use super::store::atomic_types::{RouteStatus, StoredPrefix};
 
 //------------ Prefix Matching ----------------------------------------------
 
-impl<'a, AF, M, NB, PB> TreeBitMap<AF, M, NB, PB>
+impl<'a, AF, M, NB, PB, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
+    TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 where
     AF: AddressFamily,
     M: Meta,
diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index d4a3242d..c26b0a13 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -8,7 +8,7 @@ use std::{
 use crossbeam_epoch::{self as epoch, Atomic};
 
 use crossbeam_utils::Backoff;
-use log::{debug, log_enabled, trace};
+use log::{debug, info, log_enabled, trace};
 
 use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
@@ -280,7 +280,7 @@ pub(crate) struct MultiMapValue<M> {
     pub status: RouteStatus,
 }
 
-impl<M: Clone> MultiMapValue<M> {
+impl<M: Clone + AsRef<[u8]>> MultiMapValue<M> {
     pub(crate) fn _new(meta: M, ltime: u64, status: RouteStatus) -> Self {
         Self {
             meta,
@@ -290,13 +290,15 @@ impl<M: Clone> MultiMapValue<M> {
     }
 }
 
-impl<M: crate::prefix_record::Meta> std::fmt::Display for MultiMapValue<M> {
+impl<M: crate::prefix_record::Meta + AsRef<[u8]>> std::fmt::Display
+    for MultiMapValue<M>
+{
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{} {} {}", self.meta, self.ltime, self.status)
     }
 }
 
-impl<M: Meta> From<PublicRecord<M>> for MultiMapValue<M> {
+impl<M: Meta + AsRef<[u8]>> From<PublicRecord<M>> for MultiMapValue<M> {
     fn from(value: PublicRecord<M>) -> Self {
         Self {
             meta: value.meta,
@@ -306,6 +308,12 @@ impl<M: Meta> From<PublicRecord<M>> for MultiMapValue<M> {
     }
 }
 
+impl<M: Clone + AsRef<[u8]>> AsRef<[u8]> for MultiMapValue<M> {
+    fn as_ref(&self) -> &[u8] {
+        self.meta.as_ref()
+    }
+}
+
 // ----------- MultiMap ------------------------------------------------------
 // This is the record that holds the aggregates at the top-level for a given
 // prefix.
@@ -476,20 +484,20 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     // record.multi_uniq_id. Returns the number of entries in the HashMap
     // after updating it, if it's more than 1. Returns None if this is the
     // first entry.
-    pub fn upsert_record(
+    pub(crate) fn upsert_record(
         &self,
         record: PublicRecord<M>,
-    ) -> (Option<usize>, usize) {
-        let c_map = self.clone();
-        let (mut record_map, retry_count) = c_map.guard_with_retry(0);
+    ) -> (Option<(MultiMapValue<M>, usize)>, usize) {
+        // let c_map = self.clone();
+        let (mut record_map, retry_count) = self.guard_with_retry(0);
 
-        if record_map
+        match record_map
             .insert(record.multi_uniq_id, MultiMapValue::from(record))
-            .is_some()
         {
-            (Some(record_map.len()), retry_count)
-        } else {
-            (None, retry_count)
+            Some(exist_rec) => {
+                (Some((exist_rec, record_map.len())), retry_count)
+            }
+            _ => (None, retry_count),
         }
     }
 }
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 55785a6c..0bfbdc9b 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -183,16 +183,14 @@
 // an actual memory leak in the mt-prefix-store (and even more if they can
 // produce a fix for it).
 
-use std::{
-    fmt::Debug,
-    sync::atomic::{AtomicUsize, Ordering},
-};
+use std::sync::atomic::{AtomicUsize, Ordering};
 
 use log::{debug, info, log_enabled, trace};
 
 use crossbeam_epoch::{self as epoch, Atomic};
 use crossbeam_utils::Backoff;
 use epoch::{Guard, Owned};
+use lsm_tree::AbstractTree;
 use roaring::RoaringBitmap;
 
 use std::marker::PhantomData;
@@ -298,6 +296,57 @@ pub struct UpsertReport {
     pub mui_count: usize,
 }
 
+pub struct PersistTree<
+    AF: AddressFamily,
+    // The size in bytes of the prefix in the peristed storage (disk), this
+    // amounnts to the bytes for the addres (4 for IPv4, 16 for IPv6) and 1
+    // bytefor the prefix length.
+    const PREFIX_SIZE: usize,
+    // The size in bytes of the complete key in the persisted storage, this
+    // is PREFIX_SIZE bytes (4; 16) + mui size (4) + ltime (8)
+    const KEY_SIZE: usize, (lsm_tree::Tree, PhantomData<AF>);
+
+impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
+    PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
+{
+    pub fn insert(&self, key: [u8; KEY_SIZE], value: &[u8]) -> (u32, u32) {
+        self.0.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
+    }
+
+    pub fn flush_to_disk(&self) {
+        self.0.flush_active_memtable(0).unwrap();
+    }
+
+    pub fn persistence_key(
+        // PREFIX_SIZE bytes
+        prefix_id: PrefixId<AF>,
+        // 4 bytes
+        mui: u32,
+        // 8 bytes
+        ltime: u64,
+    ) -> [u8; KEY_SIZE] {
+        eprintln!("prefix_size {} key_size {}", PREFIX_SIZE, KEY_SIZE);
+        assert!(KEY_SIZE > PREFIX_SIZE);
+        let key = &mut [0_u8; KEY_SIZE];
+        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
+        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
+            .first_chunk_mut::<4>()
+            .unwrap() = mui.to_le_bytes();
+        *key[PREFIX_SIZE + 4..].first_chunk_mut::<8>().unwrap() =
+            ltime.to_le_bytes();
+
+        *key
+    }
+}
+
+impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
+    std::fmt::Debug for PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
+{
+    fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        todo!()
+    }
+}
+
 // ----------- CustomAllocStorage -------------------------------------------
 //
 // CustomAllocStorage is a storage backend that uses a custom allocator, that
@@ -308,9 +357,13 @@ pub struct CustomAllocStorage<
     M: crate::prefix_record::Meta,
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
+    const PREFIX_SIZE: usize,
+    const KEY_SIZE: usize,
 > {
     pub(crate) buckets: NB,
     pub prefixes: PB,
+    #[cfg(feature = "persist")]
+    pub persistence: PersistTree<AF, PREFIX_SIZE, KEY_SIZE>,
     pub default_route_prefix_serial: AtomicUsize,
     // Global Roaring Bitmap INdex that stores MUIs.
     pub withdrawn_muis_bmin: Atomic<RoaringBitmap>,
@@ -325,7 +378,9 @@ impl<
         M: crate::prefix_record::Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-    > CustomAllocStorage<AF, M, NB, PB>
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > CustomAllocStorage<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
     pub(crate) fn init(
         root_node: SizedStrideNode<AF>,
@@ -343,6 +398,10 @@ impl<
         let store = CustomAllocStorage {
             buckets: NodeBuckets::<AF>::init(),
             prefixes: PrefixBuckets::<AF, M>::init(),
+            persistence: PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>(
+                lsm_tree::Config::new("/tmp/rotonda/").open()?,
+                PhantomData,
+            ),
             default_route_prefix_serial: AtomicUsize::new(0),
             withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
@@ -644,7 +703,7 @@ impl<
     ) -> Result<UpsertReport, PrefixStoreError> {
         let mut prefix_new = true;
 
-        let (mui_new, insert_retry_count) =
+        let (mui_is_new, insert_retry_count) =
             match self.non_recursive_retrieve_prefix_mut(prefix) {
                 // There's no StoredPrefix at this location yet. Create a new
                 // PrefixRecord and try to store it in the empty slot.
@@ -658,12 +717,19 @@ impl<
                         );
                     }
 
-                    // We're creating a StoredPrefix without our record first,
-                    // to avoid having to clone it on retry.
+                    let mui = record.multi_uniq_id;
                     let res = locked_prefix.record_map.upsert_record(record);
+                    let mut mui_is_new = None;
+                    let retry_count = res.1;
+
+                    #[cfg(feature = "persist")]
+                    if let Some((p_rec, min)) = res.0 {
+                        self.persist_record(prefix, mui, p_rec);
+                        mui_is_new = Some(min);
+                    }
 
                     self.counters.inc_prefixes_count(prefix.get_len());
-                    res
+                    (mui_is_new, retry_count)
                 }
                 // There already is a StoredPrefix with a record at this
                 // location.
@@ -680,25 +746,35 @@ impl<
                     }
                     prefix_new = false;
 
-                    // Update the already existing record_map with our caller's
-                    // record.
+                    // Update the already existing record_map with our
+                    // caller's record.
                     stored_prefix.set_ps_outdated(guard)?;
+
+                    let mui = record.multi_uniq_id;
                     let res = stored_prefix.record_map.upsert_record(record);
+                    let retry_count = res.1;
+                    let mut mui_is_new = None;
+
+                    #[cfg(feature = "persist")]
+                    if let Some((p_rec, min)) = res.0 {
+                        self.persist_record(prefix, mui, p_rec);
+                        mui_is_new = Some(min);
+                    }
 
                     if let Some(tbi) = update_path_selections {
                         stored_prefix
                             .calculate_and_store_best_backup(&tbi, guard)?;
                     }
 
-                    res
+                    (mui_is_new, retry_count)
                 }
             };
 
         Ok(UpsertReport {
             prefix_new,
             cas_count: insert_retry_count,
-            mui_new: mui_new.is_none(),
-            mui_count: mui_new.unwrap_or(1),
+            mui_new: mui_is_new.is_none(),
+            mui_count: mui_is_new.unwrap_or(1),
         })
     }
 
@@ -1163,6 +1239,25 @@ impl<
             >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
             .dangerously_truncate_to_u32() as usize
     }
+
+    // persistance features
+
+    #[cfg(feature = "persist")]
+    fn persist_record(
+        &self,
+        prefix: PrefixId<AF>,
+        mui: u32,
+        record: MultiMapValue<M>,
+    ) {
+        self.persistence.insert(
+            PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
+                prefix,
+                mui,
+                record.ltime,
+            ),
+            record.as_ref(),
+        );
+    }
 }
 
 //------------ Upsert -------------------------------------------------------
diff --git a/src/local_array/store/default_store.rs b/src/local_array/store/default_store.rs
index e2ddcf4c..a9893765 100644
--- a/src/local_array/store/default_store.rs
+++ b/src/local_array/store/default_store.rs
@@ -1,20 +1,23 @@
-use std::fmt;
-use crate::prelude::*;
 use crate::prelude::multi::*;
+use crate::prelude::*;
+use std::fmt;
 
 // The default stride sizes for IPv4, IPv6, resp.
 #[create_store((
-    [5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 
-    [4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-    4, 4, 4, 4, 4, 4, 4, 4, 4]
+    ([5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 5, 17),
+    ([4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
+    4, 4, 4, 4, 4, 4, 4, 4, 4], 17, 29)
 ))]
 struct DefaultStore;
 
 impl<
         M: Meta,
         NB: NodeBuckets<IPv4>,
-        PB: PrefixBuckets<IPv4, M>
-    > fmt::Display for CustomAllocStorage<IPv4, M, NB, PB>
+        PB: PrefixBuckets<IPv4, M>,
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > fmt::Display
+    for CustomAllocStorage<IPv4, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(
@@ -28,8 +31,11 @@ impl<
 impl<
         M: Meta,
         NB: NodeBuckets<IPv6>,
-        PB: PrefixBuckets<IPv6, M>
-    > fmt::Display for CustomAllocStorage<IPv6, M, NB, PB>
+        PB: PrefixBuckets<IPv6, M>,
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > fmt::Display
+    for CustomAllocStorage<IPv6, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(
diff --git a/src/local_array/store/iterators.rs b/src/local_array/store/iterators.rs
index fff93458..1fb5f6ec 100644
--- a/src/local_array/store/iterators.rs
+++ b/src/local_array/store/iterators.rs
@@ -290,8 +290,10 @@ pub(crate) struct MoreSpecificPrefixIter<
     M: Meta,
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
+    const PREFIX_SIZE: usize,
+    const KEY_SIZE: usize,
 > {
-    store: &'a CustomAllocStorage<AF, M, NB, PB>,
+    store: &'a CustomAllocStorage<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
     cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
     cur_pfx_iter: SizedPrefixIter<AF>,
     start_bit_span: BitSpan,
@@ -312,7 +314,10 @@ impl<
         M: Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-    > Iterator for MoreSpecificPrefixIter<'a, AF, M, NB, PB>
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > Iterator
+    for MoreSpecificPrefixIter<'a, AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
     type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
 
@@ -711,7 +716,9 @@ impl<
         M: crate::prefix_record::Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-    > CustomAllocStorage<AF, M, NB, PB>
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > CustomAllocStorage<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
     // Iterator over all more-specific prefixes, starting from the given
     // prefix at the given level and cursor.
diff --git a/src/local_array/tree.rs b/src/local_array/tree.rs
index a6d68c5e..f457e5a8 100644
--- a/src/local_array/tree.rs
+++ b/src/local_array/tree.rs
@@ -113,6 +113,13 @@ impl<AF: AddressFamily> PrefixId<AF> {
     pub fn inc_len(self) -> Self {
         Self(self.0.map(|(net, len)| (net, len + 1)))
     }
+
+    pub fn as_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
+        match self.0 {
+            Some(r) => r.0.as_prefix_bytes(r.1),
+            _ => [255; PREFIX_SIZE],
+        }
+    }
 }
 
 impl<AF: AddressFamily> std::default::Default for PrefixId<AF> {
@@ -127,6 +134,22 @@ impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
     }
 }
 
+impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<PrefixId<AF>>
+    for [u8; PREFIX_SIZE]
+{
+    fn from(value: PrefixId<AF>) -> Self {
+        value.as_bytes::<PREFIX_SIZE>()
+    }
+}
+
+impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<[u8; PREFIX_SIZE]>
+    for PrefixId<AF>
+{
+    fn from(value: [u8; PREFIX_SIZE]) -> Self {
+        PrefixId(Some(AF::from_prefix_bytes(value)))
+    }
+}
+
 //--------------------- Per-Stride-Node-Id Type -----------------------------
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
@@ -356,8 +379,10 @@ pub struct TreeBitMap<
     M: Meta,
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
+    const PREFIX_SIZE: usize,
+    const KEY_SIZE: usize,
 > {
-    pub store: CustomAllocStorage<AF, M, NB, PB>,
+    pub store: CustomAllocStorage<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
 }
 
 impl<
@@ -366,39 +391,58 @@ impl<
         M: Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-    > TreeBitMap<AF, M, NB, PB>
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
-    pub fn new(
-    ) -> Result<TreeBitMap<AF, M, NB, PB>, Box<dyn std::error::Error>> {
-        let root_node =
-            match CustomAllocStorage::<AF, M, NB, PB>::get_first_stride_size()
-            {
-                3 => SizedStrideNode::Stride3(TreeBitMapNode {
-                    ptrbitarr: AtomicStride2(AtomicU8::new(0)),
-                    pfxbitarr: AtomicStride3(AtomicU16::new(0)),
-                    _af: PhantomData,
-                }),
-                4 => SizedStrideNode::Stride4(TreeBitMapNode {
-                    ptrbitarr: AtomicStride3(AtomicU16::new(0)),
-                    pfxbitarr: AtomicStride4(AtomicU32::new(0)),
-                    _af: PhantomData,
-                }),
-                5 => SizedStrideNode::Stride5(TreeBitMapNode {
-                    ptrbitarr: AtomicStride4(AtomicU32::new(0)),
-                    pfxbitarr: AtomicStride5(AtomicU64::new(0)),
-                    _af: PhantomData,
-                }),
-                unknown_stride_size => {
-                    panic!(
-                        "unknown stride size {} encountered in STRIDES array",
-                        unknown_stride_size
-                    );
-                }
-            };
+    pub fn new() -> Result<
+        TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
+        Box<dyn std::error::Error>,
+    > {
+        let root_node = match CustomAllocStorage::<
+            AF,
+            M,
+            NB,
+            PB,
+            PREFIX_SIZE,
+            KEY_SIZE,
+        >::get_first_stride_size()
+        {
+            3 => SizedStrideNode::Stride3(TreeBitMapNode {
+                ptrbitarr: AtomicStride2(AtomicU8::new(0)),
+                pfxbitarr: AtomicStride3(AtomicU16::new(0)),
+                _af: PhantomData,
+            }),
+            4 => SizedStrideNode::Stride4(TreeBitMapNode {
+                ptrbitarr: AtomicStride3(AtomicU16::new(0)),
+                pfxbitarr: AtomicStride4(AtomicU32::new(0)),
+                _af: PhantomData,
+            }),
+            5 => SizedStrideNode::Stride5(TreeBitMapNode {
+                ptrbitarr: AtomicStride4(AtomicU32::new(0)),
+                pfxbitarr: AtomicStride5(AtomicU64::new(0)),
+                _af: PhantomData,
+            }),
+            unknown_stride_size => {
+                panic!(
+                    "unknown stride size {} encountered in STRIDES array",
+                    unknown_stride_size
+                );
+            }
+        };
 
-        Ok(TreeBitMap {
-            store: CustomAllocStorage::<AF, M, NB, PB>::init(root_node)?,
-        })
+        Ok(
+            TreeBitMap {
+                store: CustomAllocStorage::<
+                    AF,
+                    M,
+                    NB,
+                    PB,
+                    PREFIX_SIZE,
+                    KEY_SIZE,
+                >::init(root_node)?,
+            },
+        )
     }
 
     // Partition for stride 4
@@ -507,7 +551,9 @@ impl<
                         );
                         error!(
                             "{} {}",
-                            std::thread::current().name().unwrap_or("unnamed-thread"),
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread"),
                             err
                         );
                     }
@@ -668,7 +714,9 @@ impl<
         M: Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-    > Default for TreeBitMap<AF, M, NB, PB>
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > Default for TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
     fn default() -> Self {
         Self::new().unwrap()
@@ -682,7 +730,9 @@ impl<
         M: Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-    > std::fmt::Display for TreeBitMap<AF, M, NB, PB>
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > std::fmt::Display for TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
     fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(_f, "{} prefixes created", self.store.get_prefixes_count())?;
diff --git a/src/local_vec/tests/full_table_single.rs b/src/local_vec/tests/full_table_single.rs
index 40826e3d..c62003f3 100644
--- a/src/local_vec/tests/full_table_single.rs
+++ b/src/local_vec/tests/full_table_single.rs
@@ -1,13 +1,10 @@
-
 #![cfg(test)]
-    
-mod full_table {    
+
+mod full_table {
     use inetnum::addr::Prefix;
     use inetnum::asn::Asn;
 
-    use crate::{
-        prelude::*, PublicPrefixSingleRecord, SingleThreadedStore
-    };
+    use crate::{prelude::*, PublicPrefixSingleRecord, SingleThreadedStore};
 
     use std::error::Error;
     use std::fs::File;
@@ -43,11 +40,17 @@ mod full_table {
     //     }
     // }
 
+    impl AsRef<[u8]> for ComplexPrefixAs {
+        fn as_ref(&self) -> &[u8] {
+            todo!()
+        }
+    }
+
     impl Meta for ComplexPrefixAs {
         type Orderable<'a> = Asn;
         type TBI = ();
 
-        fn as_orderable(&self, _tbi: Self::TBI) -> Asn { 
+        fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
             self.0[0].into()
         }
     }
@@ -55,7 +58,7 @@ mod full_table {
     // impl Orderable<u32, Rfc4271> for ComplexPrefixAs {
     //     fn get_id(&self) -> &Self {
     //         &self.0
-    //     }   
+    //     }
     // }
 
     impl std::fmt::Display for ComplexPrefixAs {
@@ -105,10 +108,13 @@ mod full_table {
             // vec![3, 4, 4, 6, 7, 8],
         ];
         for _strides in strides_vec.iter().enumerate() {
-            let mut pfxs: Vec<PublicPrefixSingleRecord<ComplexPrefixAs>> = vec![];
+            let mut pfxs: Vec<PublicPrefixSingleRecord<ComplexPrefixAs>> =
+                vec![];
             let v4_strides = vec![8];
             let v6_strides = vec![8];
-            let mut tree_bitmap = SingleThreadedStore::<ComplexPrefixAs>::new(v4_strides, v6_strides);
+            let mut tree_bitmap = SingleThreadedStore::<ComplexPrefixAs>::new(
+                v4_strides, v6_strides,
+            );
 
             if let Err(err) = load_prefixes(&mut pfxs) {
                 println!("error running example: {}", err);
@@ -125,25 +131,26 @@ mod full_table {
                     }
                 };
 
-                let query = tree_bitmap.match_prefix(&pfx.prefix,
-                        &MatchOptions {
+                let query = tree_bitmap.match_prefix(
+                    &pfx.prefix,
+                    &MatchOptions {
                         match_type: MatchType::LongestMatch,
                         include_withdrawn: false,
                         include_less_specifics: false,
                         include_more_specifics: false,
-                        mui: None
+                        mui: None,
                     },
                 );
 
-                if query.prefix.is_none() { panic!("STOPSTOPSTOPST"); }
-                else { 
+                if query.prefix.is_none() {
+                    panic!("STOPSTOPSTOPST");
+                } else {
                     assert_eq!(query.prefix.unwrap(), pfx.prefix);
                 }
             }
 
             println!("done inserting {} prefixes", inserts_num);
 
-
             let inet_max = 255;
             let len_max = 32;
 
@@ -154,7 +161,6 @@ mod full_table {
             (0..inet_max).for_each(|i_net| {
                 len_count = 0;
                 (0..len_max).for_each(|s_len| {
-
                     (0..inet_max).for_each(|ii_net| {
                         let pfx = Prefix::new_relaxed(
                             std::net::Ipv4Addr::new(i_net, ii_net, 0, 0)
@@ -169,7 +175,7 @@ mod full_table {
                                 include_withdrawn: false,
                                 include_less_specifics: false,
                                 include_more_specifics: false,
-                                mui: None
+                                mui: None,
                             },
                         );
                         if let Some(_pfx) = res.prefix {
@@ -211,4 +217,4 @@ mod full_table {
         }
         Ok(())
     }
-}
\ No newline at end of file
+}
diff --git a/src/local_vec/tests/more_specifics_single.rs b/src/local_vec/tests/more_specifics_single.rs
index 502fd806..4f61faef 100644
--- a/src/local_vec/tests/more_specifics_single.rs
+++ b/src/local_vec/tests/more_specifics_single.rs
@@ -2,11 +2,7 @@
 mod tests {
     use inetnum::addr::Prefix;
 
-    use crate::{ 
-        prelude::*,
-        meta_examples::PrefixAs,
-        SingleThreadedStore
-    };
+    use crate::{meta_examples::PrefixAs, prelude::*, SingleThreadedStore};
 
     use std::error::Error;
 
@@ -15,7 +11,8 @@ mod tests {
     ) -> Result<(), Box<dyn Error>> {
         let v4_strides = vec![8];
         let v6_strides = vec![8];
-        let mut tree_bitmap = SingleThreadedStore::<PrefixAs>::new(v4_strides, v6_strides);
+        let mut tree_bitmap =
+            SingleThreadedStore::<PrefixAs>::new(v4_strides, v6_strides);
         let pfxs = vec![
             Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18)?, // 0
             Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24)?, // 1
@@ -34,7 +31,7 @@ mod tests {
         ];
 
         for pfx in pfxs.iter() {
-            tree_bitmap.insert(pfx, PrefixAs(666))?;
+            tree_bitmap.insert(pfx, PrefixAs::new_from_u32(666))?;
         }
         println!("------ end of inserts\n");
 
@@ -59,7 +56,7 @@ mod tests {
                     include_withdrawn: false,
                     include_less_specifics: false,
                     include_more_specifics: true,
-                    mui: None
+                    mui: None,
                 },
             );
             println!("em/m-s: {:#?}", found_result);
@@ -86,7 +83,8 @@ mod tests {
     {
         let v4_strides = vec![8];
         let v6_strides = vec![8];
-        let mut tree_bitmap = SingleThreadedStore::<PrefixAs>::new(v4_strides, v6_strides);
+        let mut tree_bitmap =
+            SingleThreadedStore::<PrefixAs>::new(v4_strides, v6_strides);
         let pfxs = vec![
             Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18), // 0
             Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24), // 1
@@ -105,10 +103,10 @@ mod tests {
         ];
 
         for pfx in pfxs.iter() {
-            tree_bitmap.insert(&pfx.unwrap(), PrefixAs(666))?;
+            tree_bitmap.insert(&pfx.unwrap(), PrefixAs::new_from_u32(666))?;
         }
         println!("------ end of inserts\n");
-        
+
         for spfx in &[
             (
                 &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9),
@@ -135,7 +133,7 @@ mod tests {
                     include_withdrawn: false,
                     include_less_specifics: false,
                     include_more_specifics: true,
-                    mui: None
+                    mui: None,
                 },
             );
             println!("em/m-s: {}", found_result);
diff --git a/src/meta_examples.rs b/src/meta_examples.rs
index d36e8eac..321b4bcb 100644
--- a/src/meta_examples.rs
+++ b/src/meta_examples.rs
@@ -5,7 +5,21 @@ use inetnum::asn::Asn;
 use crate::Meta;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
-pub struct PrefixAs(pub u32);
+pub struct PrefixAs([u8; 4]);
+
+impl PrefixAs {
+    pub fn new(asn: Asn) -> Self {
+        PrefixAs(u32::from_be_bytes(asn.to_raw()).to_le_bytes())
+    }
+
+    pub fn new_from_u32(value: u32) -> Self {
+        PrefixAs(value.to_le_bytes())
+    }
+
+    pub fn asn(&self) -> Asn {
+        Asn::from_u32(u32::from_le_bytes(self.0))
+    }
+}
 
 // impl MergeUpdate for PrefixAs {
 //     type UserDataIn = ();
@@ -36,13 +50,19 @@ impl Meta for PrefixAs {
     type Orderable<'a> = Asn;
     type TBI = ();
     fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
-        self.0.into()
+        u32::from_le_bytes(self.0).into()
+    }
+}
+
+impl AsRef<[u8]> for PrefixAs {
+    fn as_ref(&self) -> &[u8] {
+        self.0.as_ref()
     }
 }
 
 impl std::fmt::Display for PrefixAs {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "AS{}", self.0)
+        write!(f, "AS{}", u32::from_le_bytes(self.0))
     }
 }
 
@@ -82,6 +102,11 @@ impl Meta for NoMeta {
     fn as_orderable(&self, _tbi: Self::TBI) {}
 }
 
+impl AsRef<[u8]> for NoMeta {
+    fn as_ref(&self) -> &[u8] {
+        &[]
+    }
+}
 
 // impl MergeUpdate for NoMeta {
 //     type UserDataIn = ();
@@ -102,4 +127,4 @@ impl Meta for NoMeta {
 //     ) -> Result<(Self, Self::UserDataOut), Box<dyn std::error::Error>> {
 //         Ok((NoMeta::Empty, ()))
 //     }
-// }
\ No newline at end of file
+// }
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index 890cb41d..5967268a 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -225,7 +225,6 @@ impl<M: Meta> From<(Prefix, M)> for PublicPrefixSingleRecord<M> {
     }
 }
 
-
 //------------ PublicRecord -------------------------------------------
 
 #[derive(Clone, Debug)]
@@ -237,18 +236,27 @@ pub struct PublicRecord<M> {
 }
 
 impl<M> PublicRecord<M> {
-    pub fn new(multi_uniq_id: u32, ltime: u64, status: RouteStatus, meta: M) -> Self {
-        Self { meta, multi_uniq_id, ltime, status }
+    pub fn new(
+        multi_uniq_id: u32,
+        ltime: u64,
+        status: RouteStatus,
+        meta: M,
+    ) -> Self {
+        Self {
+            meta,
+            multi_uniq_id,
+            ltime,
+            status,
+        }
     }
 }
 
 impl<M: std::fmt::Display> std::fmt::Display for PublicRecord<M> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{{ mui: {}, ltime: {}, status: {}, meta: {} }}",
-            self.multi_uniq_id,
-            self.ltime,
-            self.status,
-            self.meta
+        write!(
+            f,
+            "{{ mui: {}, ltime: {}, status: {}, meta: {} }}",
+            self.multi_uniq_id, self.ltime, self.status, self.meta
         )
     }
 }
@@ -264,7 +272,6 @@ impl<M: Clone> From<(u32, MultiMapValue<M>)> for PublicRecord<M> {
     }
 }
 
-
 //------------ PublicPrefixRecord -------------------------------------------
 
 #[derive(Clone, Debug)]
@@ -283,7 +290,8 @@ impl<M: Meta> PublicPrefixRecord<M> {
     }
 }
 
-impl<AF, M> From<(PrefixId<AF>, Vec<PublicRecord<M>>)> for PublicPrefixRecord<M>
+impl<AF, M> From<(PrefixId<AF>, Vec<PublicRecord<M>>)>
+    for PublicPrefixRecord<M>
 where
     AF: AddressFamily,
     M: Meta,
@@ -296,7 +304,9 @@ where
     }
 }
 
-impl<M: Meta + std::fmt::Display> std::fmt::Display for PublicPrefixRecord<M> {
+impl<M: Meta + std::fmt::Display> std::fmt::Display
+    for PublicPrefixRecord<M>
+{
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{}: [", self.prefix)?;
         for rec in &self.meta {
@@ -312,7 +322,6 @@ impl<M: Meta> From<(Prefix, Vec<PublicRecord<M>>)> for PublicPrefixRecord<M> {
     }
 }
 
-
 //------------ RecordSingleSet -----------------------------------------------
 
 #[derive(Clone, Debug)]
@@ -333,7 +342,8 @@ impl<M: Meta> RecordSingleSet<M> {
         match prefix.addr() {
             std::net::IpAddr::V4(_) => &mut self.v4,
             std::net::IpAddr::V6(_) => &mut self.v6,
-        }.push(PublicPrefixSingleRecord::new(prefix, meta));
+        }
+        .push(PublicPrefixSingleRecord::new(prefix, meta));
     }
 
     pub fn is_empty(&self) -> bool {
@@ -385,17 +395,23 @@ impl<M: Meta> fmt::Display for RecordSingleSet<M> {
 }
 
 impl<M: Meta>
-    From<(Vec<PublicPrefixSingleRecord<M>>, Vec<PublicPrefixSingleRecord<M>>)>
-    for RecordSingleSet<M>
+    From<(
+        Vec<PublicPrefixSingleRecord<M>>,
+        Vec<PublicPrefixSingleRecord<M>>,
+    )> for RecordSingleSet<M>
 {
     fn from(
-        (v4, v6): (Vec<PublicPrefixSingleRecord<M>>, Vec<PublicPrefixSingleRecord<M>>),
+        (v4, v6): (
+            Vec<PublicPrefixSingleRecord<M>>,
+            Vec<PublicPrefixSingleRecord<M>>,
+        ),
     ) -> Self {
         Self { v4, v6 }
     }
 }
 
-impl<'a, M: Meta + 'a> std::iter::FromIterator<Arc<PublicPrefixSingleRecord<M>>>
+impl<'a, M: Meta + 'a>
+    std::iter::FromIterator<Arc<PublicPrefixSingleRecord<M>>>
     for RecordSingleSet<M>
 {
     fn from_iter<I: IntoIterator<Item = Arc<PublicPrefixSingleRecord<M>>>>(
@@ -407,10 +423,16 @@ impl<'a, M: Meta + 'a> std::iter::FromIterator<Arc<PublicPrefixSingleRecord<M>>>
             let u_pfx = pfx.prefix;
             match u_pfx.addr() {
                 std::net::IpAddr::V4(_) => {
-                    v4.push(PublicPrefixSingleRecord::new(u_pfx, pfx.meta.clone()));
+                    v4.push(PublicPrefixSingleRecord::new(
+                        u_pfx,
+                        pfx.meta.clone(),
+                    ));
                 }
                 std::net::IpAddr::V6(_) => {
-                    v6.push(PublicPrefixSingleRecord::new(u_pfx, pfx.meta.clone()));
+                    v6.push(PublicPrefixSingleRecord::new(
+                        u_pfx,
+                        pfx.meta.clone(),
+                    ));
                 }
             }
         }
@@ -419,8 +441,7 @@ impl<'a, M: Meta + 'a> std::iter::FromIterator<Arc<PublicPrefixSingleRecord<M>>>
 }
 
 impl<'a, AF: AddressFamily, M: Meta + 'a>
-    std::iter::FromIterator<(PrefixId<AF>, Arc<M>)>
-    for RecordSingleSet<M>
+    std::iter::FromIterator<(PrefixId<AF>, Arc<M>)> for RecordSingleSet<M>
 {
     fn from_iter<I: IntoIterator<Item = (PrefixId<AF>, Arc<M>)>>(
         iter: I,
@@ -431,10 +452,16 @@ impl<'a, AF: AddressFamily, M: Meta + 'a>
             let u_pfx = pfx.0.into_pub();
             match u_pfx.addr() {
                 std::net::IpAddr::V4(_) => {
-                    v4.push(PublicPrefixSingleRecord::new(u_pfx, (*pfx.1).clone()));
+                    v4.push(PublicPrefixSingleRecord::new(
+                        u_pfx,
+                        (*pfx.1).clone(),
+                    ));
                 }
                 std::net::IpAddr::V6(_) => {
-                    v6.push(PublicPrefixSingleRecord::new(u_pfx, (*pfx.1).clone()));
+                    v6.push(PublicPrefixSingleRecord::new(
+                        u_pfx,
+                        (*pfx.1).clone(),
+                    ));
                 }
             }
         }
@@ -455,10 +482,16 @@ impl<'a, AF: AddressFamily, M: Meta + 'a>
             let u_pfx = (*pfx).prefix_into_pub();
             match u_pfx.addr() {
                 std::net::IpAddr::V4(_) => {
-                    v4.push(PublicPrefixSingleRecord::new(u_pfx, pfx.meta.clone()));
+                    v4.push(PublicPrefixSingleRecord::new(
+                        u_pfx,
+                        pfx.meta.clone(),
+                    ));
                 }
                 std::net::IpAddr::V6(_) => {
-                    v6.push(PublicPrefixSingleRecord::new(u_pfx, pfx.meta.clone()));
+                    v6.push(PublicPrefixSingleRecord::new(
+                        u_pfx,
+                        pfx.meta.clone(),
+                    ));
                 }
             }
         }
@@ -498,7 +531,8 @@ impl<M: Meta> RecordSet<M> {
         match prefix.addr() {
             std::net::IpAddr::V4(_) => &mut self.v4,
             std::net::IpAddr::V6(_) => &mut self.v6,
-        }.push(PublicPrefixRecord::new(prefix, meta));
+        }
+        .push(PublicPrefixRecord::new(prefix, meta));
     }
 
     pub fn is_empty(&self) -> bool {
@@ -549,8 +583,7 @@ impl<M: Meta> fmt::Display for RecordSet<M> {
     }
 }
 
-impl<M: Meta>
-    From<(Vec<PublicPrefixRecord<M>>, Vec<PublicPrefixRecord<M>>)>
+impl<M: Meta> From<(Vec<PublicPrefixRecord<M>>, Vec<PublicPrefixRecord<M>>)>
     for RecordSet<M>
 {
     fn from(
@@ -560,8 +593,7 @@ impl<M: Meta>
     }
 }
 
-impl<'a, M: Meta + 'a>
-    std::iter::FromIterator<PublicPrefixRecord<M>>
+impl<'a, M: Meta + 'a> std::iter::FromIterator<PublicPrefixRecord<M>>
     for RecordSet<M>
 {
     fn from_iter<I: IntoIterator<Item = PublicPrefixRecord<M>>>(
@@ -588,7 +620,9 @@ impl<'a, AF: AddressFamily, M: Meta + 'a>
     std::iter::FromIterator<(PrefixId<AF>, Vec<PublicRecord<M>>)>
     for RecordSet<M>
 {
-    fn from_iter<I: IntoIterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)>>(
+    fn from_iter<
+        I: IntoIterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)>,
+    >(
         iter: I,
     ) -> Self {
         let mut v4 = vec![];
@@ -608,8 +642,7 @@ impl<'a, AF: AddressFamily, M: Meta + 'a>
     }
 }
 
-impl<'a, M: Meta + 'a>
-    std::iter::FromIterator<&'a PublicPrefixRecord<M>>
+impl<'a, M: Meta + 'a> std::iter::FromIterator<&'a PublicPrefixRecord<M>>
     for RecordSet<M>
 {
     fn from_iter<I: IntoIterator<Item = &'a PublicPrefixRecord<M>>>(
@@ -644,7 +677,6 @@ impl<M: Meta> std::ops::Index<usize> for RecordSet<M> {
     }
 }
 
-
 //------------ RecordSetSingleIter -------------------------------------------
 
 #[derive(Clone, Debug)]
@@ -693,7 +725,6 @@ impl<'a, M: Meta> Iterator for RecordSetIter<'a, M> {
     }
 }
 
-
 //----------------------- meta-data traits/types-----------------------------
 
 /// Trait that describes how an existing record gets merged
@@ -740,18 +771,22 @@ impl<'a, M: Meta> Iterator for RecordSetIter<'a, M> {
 /// Trait for types that can be used as metadata of a record
 pub trait Meta
 where
-    Self: fmt::Debug + fmt::Display + Clone + Sized + Send + Sync {
-        type Orderable<'a>: Ord where Self: 'a;
-        type TBI: Copy;
+    Self:
+        fmt::Debug + fmt::Display + Clone + Sized + Send + Sync + AsRef<[u8]>,
+{
+    type Orderable<'a>: Ord
+    where
+        Self: 'a;
+    type TBI: Copy;
 
-        fn as_orderable(&self, tbi: Self::TBI) -> Self::Orderable<'_>;
-    }
+    fn as_orderable(&self, tbi: Self::TBI) -> Self::Orderable<'_>;
+}
 
-impl Meta for inetnum::asn::Asn {
-    type Orderable<'a> = inetnum::asn::Asn;
-    type TBI = ();
+// impl Meta for inetnum::asn::Asn {
+//     type Orderable<'a> = inetnum::asn::Asn;
+//     type TBI = ();
 
-    fn as_orderable(&self, _tbi: Self::TBI) -> inetnum::asn::Asn {
-        *self
-    }
-}
\ No newline at end of file
+//     fn as_orderable(&self, _tbi: Self::TBI) -> inetnum::asn::Asn {
+//         *self
+//     }
+// }
diff --git a/tests/best-path.rs b/tests/best-path.rs
index 72625c75..9c47652a 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -1,20 +1,20 @@
 use inetnum::addr::Prefix;
+use inetnum::asn::Asn;
 use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::Record;
 use rotonda_store::prelude::multi::RouteStatus;
 use rotonda_store::MatchOptions;
-use inetnum::asn::Asn;
+use rotonda_store::Meta;
+use rotonda_store::MultiThreadedStore;
 use routecore::bgp::aspath::HopPath;
 use routecore::bgp::path_attributes::BgpIdentifier;
 use routecore::bgp::path_attributes::PaMap;
 use routecore::bgp::path_selection::RouteSource;
+use routecore::bgp::path_selection::{OrdRoute, Rfc4271, TiebreakerInfo};
 use routecore::bgp::types::LocalPref;
 use routecore::bgp::types::Origin;
 use std::net::Ipv4Addr;
 use std::str::FromStr;
-use rotonda_store::Meta;
-use rotonda_store::MultiThreadedStore;
-use routecore::bgp::path_selection::{OrdRoute, Rfc4271, TiebreakerInfo};
 
 #[derive(Clone, Debug)]
 pub struct Ipv4Route(u32, PaMap, TiebreakerInfo);
@@ -32,7 +32,14 @@ impl Meta for Ipv4Route {
     type TBI = ();
 
     fn as_orderable(&self, _tbi: Self::TBI) -> Self::Orderable<'_> {
-        routecore::bgp::path_selection::OrdRoute::rfc4271(&self.1, self.2).unwrap()
+        routecore::bgp::path_selection::OrdRoute::rfc4271(&self.1, self.2)
+            .unwrap()
+    }
+}
+
+impl AsRef<[u8]> for Ipv4Route {
+    fn as_ref(&self) -> &[u8] {
+        todo!()
     }
 }
 
@@ -50,81 +57,147 @@ mod common {
 #[test]
 fn test_best_path_1() -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
-    
-    let tree_bitmap = std::sync::Arc::new(std::sync::Arc::new(MultiThreadedStore::<Ipv4Route>::new()?));
+
+    let tree_bitmap =
+        std::sync::Arc::new(std::sync::Arc::new(MultiThreadedStore::<
+            Ipv4Route,
+        >::new()?));
 
     let pfx = Prefix::from_str("185.34.0.0/16")?;
-    let mut asns = [Asn::from(65400), Asn::from(65401), Asn::from(65402), Asn::from(65403), Asn::from(65404)].into_iter();
+    let mut asns = [
+        Asn::from(65400),
+        Asn::from(65401),
+        Asn::from(65402),
+        Asn::from(65403),
+        Asn::from(65404),
+    ]
+    .into_iter();
     let mut pa_map = PaMap::empty();
 
     pa_map.set::<LocalPref>(routecore::bgp::types::LocalPref(50));
-    pa_map.set::<HopPath>(
-        HopPath::from(vec![Asn::from(65400), Asn::from(65401), Asn::from(65402)])
-    );
-    pa_map.set::<Origin>(routecore::bgp::types::Origin(routecore::bgp::types::OriginType::Egp));
+    pa_map.set::<HopPath>(HopPath::from(vec![
+        Asn::from(65400),
+        Asn::from(65401),
+        Asn::from(65402),
+    ]));
+    pa_map.set::<Origin>(routecore::bgp::types::Origin(
+        routecore::bgp::types::OriginType::Egp,
+    ));
 
     let mut asns_insert = vec![];
-   
-   // Out TiebreakInfo consists of some values that are the same for all of
-   // our routes, and some that are specific to the route.
-    let tbi_modifier = |peer_addr: Ipv4Addr, local_asn: Asn, bgp_identifier: BgpIdentifier| {
-        TiebreakerInfo::new(
-            RouteSource::Ebgp,
-            None,
-            local_asn,
-            bgp_identifier,
-            std::net::IpAddr::V4(peer_addr)
-        )
-    };
+
+    // Out TiebreakInfo consists of some values that are the same for all of
+    // our routes, and some that are specific to the route.
+    let tbi_modifier =
+        |peer_addr: Ipv4Addr,
+         local_asn: Asn,
+         bgp_identifier: BgpIdentifier| {
+            TiebreakerInfo::new(
+                RouteSource::Ebgp,
+                None,
+                local_asn,
+                bgp_identifier,
+                std::net::IpAddr::V4(peer_addr),
+            )
+        };
 
     for (mui, tbi) in [
-        (1, tbi_modifier(std::net::Ipv4Addr::from_str("192.168.12.1")?, Asn::from(65400), BgpIdentifier::from([0; 4]) )),
-        (2, tbi_modifier(std::net::Ipv4Addr::from_str("192.168.12.2")?, Asn::from(65400), BgpIdentifier::from([0; 4]) )),
-        (3, tbi_modifier(std::net::Ipv4Addr::from_str("192.168.12.3")?, Asn::from(65400), BgpIdentifier::from([0; 4]) )),
-        (4, tbi_modifier(std::net::Ipv4Addr::from_str("192.168.12.4")?, Asn::from(65400), BgpIdentifier::from([0; 4]) )),
-        (5, tbi_modifier(std::net::Ipv4Addr::from_str("192.168.12.5")?, Asn::from(65400), BgpIdentifier::from([0; 4]) )),
+        (
+            1,
+            tbi_modifier(
+                std::net::Ipv4Addr::from_str("192.168.12.1")?,
+                Asn::from(65400),
+                BgpIdentifier::from([0; 4]),
+            ),
+        ),
+        (
+            2,
+            tbi_modifier(
+                std::net::Ipv4Addr::from_str("192.168.12.2")?,
+                Asn::from(65400),
+                BgpIdentifier::from([0; 4]),
+            ),
+        ),
+        (
+            3,
+            tbi_modifier(
+                std::net::Ipv4Addr::from_str("192.168.12.3")?,
+                Asn::from(65400),
+                BgpIdentifier::from([0; 4]),
+            ),
+        ),
+        (
+            4,
+            tbi_modifier(
+                std::net::Ipv4Addr::from_str("192.168.12.4")?,
+                Asn::from(65400),
+                BgpIdentifier::from([0; 4]),
+            ),
+        ),
+        (
+            5,
+            tbi_modifier(
+                std::net::Ipv4Addr::from_str("192.168.12.5")?,
+                Asn::from(65400),
+                BgpIdentifier::from([0; 4]),
+            ),
+        ),
     ] {
         asns_insert.push(asns.next().unwrap());
         pa_map.set::<HopPath>(HopPath::from(asns_insert.clone()));
-        let rec = Record::new(mui,0, RouteStatus::Active, Ipv4Route(mui, pa_map.clone(), tbi));
-        tree_bitmap.insert(
-            &pfx, 
-            rec,
-            None
-        )?;
+        let rec = Record::new(
+            mui,
+            0,
+            RouteStatus::Active,
+            Ipv4Route(mui, pa_map.clone(), tbi),
+        );
+        tree_bitmap.insert(&pfx, rec, None)?;
     }
 
     let res = tree_bitmap.match_prefix(
         &pfx,
-        &MatchOptions { 
-            match_type: rotonda_store::MatchType::ExactMatch, 
+        &MatchOptions {
+            match_type: rotonda_store::MatchType::ExactMatch,
             include_withdrawn: false,
             include_less_specifics: false,
             include_more_specifics: false,
-            mui: None
+            mui: None,
         },
-        &rotonda_store::epoch::pin()
+        &rotonda_store::epoch::pin(),
     );
 
     println!("{:?}", res.prefix_meta);
     let best_path = tree_bitmap.best_path(&pfx, &rotonda_store::epoch::pin());
-    println!("ps outdated? {}", tree_bitmap.is_ps_outdated(&pfx, &rotonda_store::epoch::pin()).unwrap());
+    println!(
+        "ps outdated? {}",
+        tree_bitmap
+            .is_ps_outdated(&pfx, &rotonda_store::epoch::pin())
+            .unwrap()
+    );
     println!("{:?}", best_path);
 
     // We didn't calculate the best path yet, but the prefix (and its entries)
     // exists, so this should be `Some(Err(BestPathNotFound))` at this point.
-    assert_eq!(best_path.unwrap().err().unwrap(), PrefixStoreError::BestPathNotFound);
+    assert_eq!(
+        best_path.unwrap().err().unwrap(),
+        PrefixStoreError::BestPathNotFound
+    );
 
     tree_bitmap.calculate_and_store_best_and_backup_path(
         &pfx,
         &(),
-        &rotonda_store::epoch::pin()
+        &rotonda_store::epoch::pin(),
     )?;
 
     let best_path = tree_bitmap.best_path(&pfx, &rotonda_store::epoch::pin());
-    println!("ps outdated? {}", tree_bitmap.is_ps_outdated(&pfx, &rotonda_store::epoch::pin()).unwrap());
+    println!(
+        "ps outdated? {}",
+        tree_bitmap
+            .is_ps_outdated(&pfx, &rotonda_store::epoch::pin())
+            .unwrap()
+    );
     println!("{:?}", best_path);
     assert_eq!(best_path.unwrap().unwrap().multi_uniq_id, 1);
 
     Ok(())
-}
\ No newline at end of file
+}
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index 249dcd72..74a32b08 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -1,25 +1,66 @@
 use std::{str::FromStr, sync::atomic::Ordering};
 
+use common::SimpleAsn;
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
     prelude::multi::{Record, RouteStatus},
-    MatchOptions, MultiThreadedStore,
+    MatchOptions, Meta, MultiThreadedStore,
 };
 
 mod common {
     use std::io::Write;
 
+    use inetnum::asn::Asn;
+    use rotonda_store::Meta;
+
     pub fn init() {
         let _ = env_logger::builder()
             .format(|buf, record| writeln!(buf, "{}", record.args()))
             .is_test(true)
             .try_init();
     }
+
+    #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone)]
+    pub struct SimpleAsn([u8; 4]);
+
+    impl std::fmt::Display for SimpleAsn {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "{}", u32::from_le_bytes(self.0))
+        }
+    }
+
+    impl Meta for SimpleAsn {
+        type Orderable<'a> = SimpleAsn;
+        type TBI = ();
+
+        fn as_orderable(&self, tbi: Self::TBI) -> Self::Orderable<'_> {
+            todo!()
+        }
+    }
+
+    impl From<Asn> for SimpleAsn {
+        fn from(value: Asn) -> Self {
+            Self(u32::from_be_bytes(value.to_raw()).to_le_bytes())
+        }
+    }
+
+    impl From<u32> for SimpleAsn {
+        fn from(value: u32) -> Self {
+            Self(value.to_le_bytes())
+        }
+    }
+
+    impl AsRef<[u8]> for SimpleAsn {
+        fn as_ref(&self) -> &[u8] {
+            &self.0
+        }
+    }
 }
 
 #[test]
 fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
+    use crate::common::SimpleAsn;
 
     let pfx_vec_1 = vec![
         Prefix::from_str("185.34.0.0/16")?,
@@ -48,7 +89,8 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
         pfxs: Vec<Prefix>,
     }
 
-    let tree_bitmap = std::sync::Arc::new(MultiThreadedStore::<Asn>::new()?);
+    let tree_bitmap =
+        std::sync::Arc::new(MultiThreadedStore::<SimpleAsn>::new()?);
 
     let mui_data_1 = MuiData {
         mui: 1,
@@ -89,7 +131,7 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
                                 data.mui,
                                 cur_ltime.load(Ordering::Acquire),
                                 RouteStatus::Active,
-                                data.asn,
+                                data.asn.into(),
                             ),
                             None,
                         ) {
@@ -373,19 +415,10 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
         Prefix::from_str("188.0.0.0/8")?,
     ];
 
-    #[derive(Debug)]
-    struct MuiData {
-        asn: u32,
-    }
+    let tree_bitmap =
+        std::sync::Arc::new(MultiThreadedStore::<common::SimpleAsn>::new()?);
 
-    let tree_bitmap = std::sync::Arc::new(MultiThreadedStore::<Asn>::new()?);
-
-    const MUI_DATA: [MuiData; 4] = [
-        MuiData { asn: 65501 },
-        MuiData { asn: 65502 },
-        MuiData { asn: 65503 },
-        MuiData { asn: 65504 },
-    ];
+    const MUI_DATA: [u32; 4] = [65501, 65502, 65503, 65504];
 
     let cur_ltime = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
 
@@ -410,7 +443,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
                                     i as u32 + 1,
                                     cur_ltime.load(Ordering::Acquire),
                                     RouteStatus::Active,
-                                    MUI_DATA[i].asn.into(),
+                                    Asn::from(MUI_DATA[i]).into(),
                                 ),
                                 None,
                             ) {
@@ -500,7 +533,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
     );
     assert_eq!(
         all_pfxs_iter.iter().find(|p| p.prefix == pfx).unwrap().meta[0].meta,
-        Asn::from_u32(65503)
+        Asn::from_u32(65503).into()
     );
 
     let pfx = Prefix::from_str("187.0.0.0/8").unwrap();
@@ -515,7 +548,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
     );
     assert_eq!(
         all_pfxs_iter.iter().find(|p| p.prefix == pfx).unwrap().meta[0].meta,
-        Asn::from_u32(65504)
+        Asn::from_u32(65504).into()
     );
 
     let pfx = Prefix::from_str("185.35.0.0/16").unwrap();
@@ -530,7 +563,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
     );
     assert_eq!(
         all_pfxs_iter.iter().find(|p| p.prefix == pfx).unwrap().meta[0].meta,
-        Asn::from_u32(65501)
+        Asn::from_u32(65501).into()
     );
 
     let pfx = Prefix::from_str("185.34.15.0/24").unwrap();
@@ -570,7 +603,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
     );
     assert_eq!(
         all_pfxs_iter.iter().find(|p| p.prefix == pfx).unwrap().meta[0].meta,
-        Asn::from_u32(65504)
+        Asn::from_u32(65504).into()
     );
 
     // Create Withdrawals
diff --git a/tests/full-table.rs b/tests/full-table.rs
index 72f4c38f..4b84f73f 100644
--- a/tests/full-table.rs
+++ b/tests/full-table.rs
@@ -1,19 +1,18 @@
 #![cfg(feature = "csv")]
 #[cfg(test)]
 mod tests {
-    use inetnum::asn::Asn;
     use inetnum::addr::Prefix;
-    use rotonda_store::{
-        prelude::*, 
-        prelude::multi::*,
-    };
+    use inetnum::asn::Asn;
+    use rotonda_store::{prelude::multi::*, prelude::*};
 
     use std::error::Error;
     use std::fs::File;
     use std::process;
 
     #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
-    pub struct ComplexPrefixAs(pub Vec<u32>);
+    pub struct ComplexPrefixAs(Vec<u8>);
+
+    // pub struct ComplexPrefixAs(pub Vec<u32>);
 
     impl std::fmt::Display for ComplexPrefixAs {
         fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
@@ -26,7 +25,25 @@ mod tests {
         type TBI = ();
 
         fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
-            Asn::from(self.0[0])
+            Asn::from(u32::from_le_bytes(*self.0.first_chunk::<4>().unwrap()))
+        }
+    }
+
+    impl AsRef<[u8]> for ComplexPrefixAs {
+        fn as_ref(&self) -> &[u8] {
+            &self.0
+        }
+    }
+
+    impl From<Vec<u32>> for ComplexPrefixAs {
+        fn from(value: Vec<u32>) -> Self {
+            ComplexPrefixAs(
+                value
+                    .into_iter()
+                    .map(|v| v.to_le_bytes())
+                    .flatten()
+                    .collect::<Vec<u8>>(),
+            )
         }
     }
 
@@ -64,7 +81,7 @@ mod tests {
                         0,
                         0,
                         RouteStatus::Active,
-                        ComplexPrefixAs(vec![asn])
+                        vec![asn].into(),
                     )],
                 );
                 pfxs.push(pfx);
@@ -81,7 +98,7 @@ mod tests {
         for _strides in strides_vec.iter().enumerate() {
             let mut pfxs: Vec<PrefixRecord<ComplexPrefixAs>> = vec![];
             let tree_bitmap = MultiThreadedStore::<ComplexPrefixAs>::new()?;
-                // .with_user_data("Testing".to_string());
+            // .with_user_data("Testing".to_string());
 
             if let Err(err) = load_prefixes(&mut pfxs) {
                 println!("error running example: {}", err);
@@ -90,7 +107,11 @@ mod tests {
 
             let inserts_num = pfxs.len();
             for pfx in pfxs.into_iter() {
-                match tree_bitmap.insert(&pfx.prefix, pfx.meta[0].clone(), None) {
+                match tree_bitmap.insert(
+                    &pfx.prefix,
+                    pfx.meta[0].clone(),
+                    None,
+                ) {
                     Ok(_) => {}
                     Err(e) => {
                         println!("{}", e);
@@ -98,26 +119,27 @@ mod tests {
                     }
                 };
 
-                let query = tree_bitmap.match_prefix(&pfx.prefix,
-                        &MatchOptions {
+                let query = tree_bitmap.match_prefix(
+                    &pfx.prefix,
+                    &MatchOptions {
                         match_type: MatchType::LongestMatch,
                         include_withdrawn: false,
                         include_less_specifics: false,
                         include_more_specifics: false,
-                        mui: None
+                        mui: None,
                     },
-                    guard
+                    guard,
                 );
 
-                if query.prefix.is_none() { panic!("STOPSTOPSTOPST"); }
-                else { 
+                if query.prefix.is_none() {
+                    panic!("STOPSTOPSTOPST");
+                } else {
                     assert_eq!(query.prefix.unwrap(), pfx.prefix);
                 }
             }
 
             println!("done inserting {} prefixes", inserts_num);
 
-
             let inet_max = 255;
             let len_max = 32;
 
@@ -128,7 +150,6 @@ mod tests {
             (0..inet_max).for_each(|i_net| {
                 len_count = 0;
                 (0..len_max).for_each(|s_len| {
-
                     (0..inet_max).for_each(|ii_net| {
                         let pfx = Prefix::new_relaxed(
                             std::net::Ipv4Addr::new(i_net, ii_net, 0, 0)
@@ -143,7 +164,7 @@ mod tests {
                                 include_withdrawn: false,
                                 include_less_specifics: false,
                                 include_more_specifics: false,
-                                mui: None
+                                mui: None,
                             },
                             guard,
                         );
@@ -180,7 +201,10 @@ mod tests {
 
             assert_eq!(searches_num, SEARCHES_NUM as u128);
             assert_eq!(inserts_num, INSERTS_NUM);
-            assert_eq!(tree_bitmap.prefixes_count(), GLOBAL_PREFIXES_VEC_SIZE);
+            assert_eq!(
+                tree_bitmap.prefixes_count(),
+                GLOBAL_PREFIXES_VEC_SIZE
+            );
             assert_eq!(found_counter, FOUND_PREFIXES);
             assert_eq!(not_found_counter, SEARCHES_NUM - FOUND_PREFIXES);
         }
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index 2c790bcb..8c52128b 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -1,10 +1,8 @@
 // type Prefix4<'a> = Prefix<u32, PrefixAs>;
 mod tests {
     use inetnum::addr::Prefix;
-    use rotonda_store::{ 
-        meta_examples::PrefixAs,
-        prelude::*,
-        prelude::multi::*,
+    use rotonda_store::{
+        meta_examples::PrefixAs, prelude::multi::*, prelude::*,
     };
 
     use std::error::Error;
@@ -32,7 +30,14 @@ mod tests {
 
         for pfx in pfxs.iter() {
             tree_bitmap.insert(
-                pfx, Record::new(0, 0, RouteStatus::Active, PrefixAs(666)), None
+                pfx,
+                Record::new(
+                    0,
+                    0,
+                    RouteStatus::Active,
+                    PrefixAs::new_from_u32(666),
+                ),
+                None,
             )?;
         }
         println!("------ end of inserts\n");
@@ -61,7 +66,7 @@ mod tests {
                     include_more_specifics: true,
                     mui: None,
                 },
-                guard
+                guard,
             );
             println!("em/m-s: {:#?}", found_result);
 
@@ -106,11 +111,15 @@ mod tests {
         let ltime = 0;
         let status = RouteStatus::Active;
         for pfx in pfxs.iter() {
-            tree_bitmap.insert(&pfx.unwrap(), Record::new(0, ltime, status, PrefixAs(666)), None)?;
+            tree_bitmap.insert(
+                &pfx.unwrap(),
+                Record::new(0, ltime, status, PrefixAs::new_from_u32(666)),
+                None,
+            )?;
         }
         println!("------ end of inserts\n");
         let guard = &epoch::pin();
-        
+
         for spfx in &[
             (
                 &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9),
@@ -137,9 +146,9 @@ mod tests {
                     include_withdrawn: false,
                     include_less_specifics: false,
                     include_more_specifics: true,
-                    mui: None
+                    mui: None,
                 },
-                guard
+                guard,
             );
             println!("em/m-s: {}", found_result);
 
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index bf2c8eeb..59807bf6 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -2,9 +2,7 @@
 mod tests {
     use inetnum::addr::Prefix;
     use rotonda_store::meta_examples::PrefixAs;
-    use rotonda_store::{
-        prelude::*, prelude::multi::*
-    };
+    use rotonda_store::{prelude::multi::*, prelude::*};
 
     use std::error::Error;
 
@@ -96,8 +94,13 @@ mod tests {
         for pfx in pfxs.iter().flatten() {
             tree_bitmap.insert(
                 pfx,
-                Record::new(0, 0, RouteStatus::Active, PrefixAs(666)),
-                None
+                Record::new(
+                    0,
+                    0,
+                    RouteStatus::Active,
+                    PrefixAs::new_from_u32(666),
+                ),
+                None,
             )?;
         }
         println!("------ end of inserts\n");
@@ -168,10 +171,7 @@ mod tests {
                 vec![12, 13, 24, 25, 26, 27],
             ),
             (
-                &Prefix::new(
-                    std::net::Ipv4Addr::new(0,0,0,0).into(),
-                    0,
-                ),
+                &Prefix::new(std::net::Ipv4Addr::new(0, 0, 0, 0).into(), 0),
                 None,
                 // These are the indexes to pfxs.2 vec.
                 // These are all supposed to show up in the result.
@@ -189,9 +189,9 @@ mod tests {
                     include_withdrawn: false,
                     include_less_specifics: false,
                     include_more_specifics: true,
-                    mui: None
+                    mui: None,
                 },
-                guard
+                guard,
             );
             println!("em/m-s: {:#?}", found_result);
 
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 3d1632e4..3b51028a 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -321,7 +321,12 @@ mod tests {
         for pfx in pfxs.into_iter() {
             tree_bitmap.insert(
                 &pfx?,
-                Record::new(0, 0, RouteStatus::Active, PrefixAs(666)),
+                Record::new(
+                    0,
+                    0,
+                    RouteStatus::Active,
+                    PrefixAs::new_from_u32(666),
+                ),
                 None,
             )?;
         }
diff --git a/tests/treebitmap_v6.rs b/tests/treebitmap_v6.rs
index 5606ff30..6adcecb4 100644
--- a/tests/treebitmap_v6.rs
+++ b/tests/treebitmap_v6.rs
@@ -274,7 +274,12 @@ mod tests {
         for pfx in pfxs.into_iter() {
             tree_bitmap.insert(
                 &pfx?,
-                Record::new(0, 0, RouteStatus::Active, PrefixAs(666)),
+                Record::new(
+                    0,
+                    0,
+                    RouteStatus::Active,
+                    PrefixAs::new_from_u32(666),
+                ),
                 None,
             )?;
         }
@@ -535,7 +540,12 @@ mod tests {
         for pfx in pfxs.into_iter() {
             tree_bitmap.insert(
                 &pfx?,
-                Record::new(0, 0, RouteStatus::Active, PrefixAs(666)),
+                Record::new(
+                    0,
+                    0,
+                    RouteStatus::Active,
+                    PrefixAs::new_from_u32(666),
+                ),
                 None,
             )?;
         }

From 207061989a68b07722e505cbfc38ea4cabd236bb Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 28 Nov 2024 11:09:01 +0100
Subject: [PATCH 005/147] weird typo

---
 proc_macros/src/lib.rs                | 1 -
 src/local_array/store/custom_alloc.rs | 4 +++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 49acac5a..6a0c015f 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -47,7 +47,6 @@ pub fn stride_sizes(
     let prefixes_all_len;
     let all_len;
     let prefixes_buckets_name: syn::Ident;
-    // let prefix_store_bits;
     let get_root_prefix_set;
 
     // The name of the Struct that we're going to generate
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 0bfbdc9b..80b55d2d 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -304,7 +304,8 @@ pub struct PersistTree<
     const PREFIX_SIZE: usize,
     // The size in bytes of the complete key in the persisted storage, this
     // is PREFIX_SIZE bytes (4; 16) + mui size (4) + ltime (8)
-    const KEY_SIZE: usize, (lsm_tree::Tree, PhantomData<AF>);
+    const KEY_SIZE: usize,
+>(lsm_tree::Tree, PhantomData<AF>);
 
 impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
@@ -317,6 +318,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         self.0.flush_active_memtable(0).unwrap();
     }
 
+    #[cfg(feature = "persist")]
     pub fn persistence_key(
         // PREFIX_SIZE bytes
         prefix_id: PrefixId<AF>,

From 463b6452bc1e91fdb6c5b6740926473d28737aea Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 28 Nov 2024 19:18:54 +0100
Subject: [PATCH 006/147] Add approx items fn for persistence

---
 proc_macros/src/lib.rs                | 12 +++++++++++-
 src/local_array/store/custom_alloc.rs | 11 +++++++----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 6a0c015f..4155024a 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -1625,11 +1625,21 @@ pub fn create_store(
                 }
             }
 
-            /// Persist to disk
+            /// Persist all the non-unique (prefix, mui, ltime) tuples
+            /// with their values to disk
             pub fn flush_to_disk(&self) {
                 self.v4.store.persistence.flush_to_disk();
                 self.v6.store.persistence.flush_to_disk();
             }
+
+            /// Return the approximate number of items that are persisted
+            /// to disk, for IPv4 and IPv6 respectively.
+            pub fn approx_persisted_items(&self) -> (usize, usize) {
+                (
+                    self.v4.store.persistence.approximate_len(),
+                    self.v6.store.persistence.approximate_len()
+                )
+            }
         }
     };
 
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 80b55d2d..23641ac5 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -318,6 +318,10 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         self.0.flush_active_memtable(0).unwrap();
     }
 
+    pub fn approximate_len(&self) -> usize {
+        self.0.approximate_len()
+    }
+
     #[cfg(feature = "persist")]
     pub fn persistence_key(
         // PREFIX_SIZE bytes
@@ -327,7 +331,6 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         // 8 bytes
         ltime: u64,
     ) -> [u8; KEY_SIZE] {
-        eprintln!("prefix_size {} key_size {}", PREFIX_SIZE, KEY_SIZE);
         assert!(KEY_SIZE > PREFIX_SIZE);
         let key = &mut [0_u8; KEY_SIZE];
         *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
@@ -703,7 +706,7 @@ impl<
         update_path_selections: Option<M::TBI>,
         guard: &Guard,
     ) -> Result<UpsertReport, PrefixStoreError> {
-        let mut prefix_new = true;
+        let mut prefix_is_new = true;
 
         let (mui_is_new, insert_retry_count) =
             match self.non_recursive_retrieve_prefix_mut(prefix) {
@@ -746,7 +749,7 @@ impl<
                         prefix.get_len()
                     );
                     }
-                    prefix_new = false;
+                    prefix_is_new = false;
 
                     // Update the already existing record_map with our
                     // caller's record.
@@ -773,7 +776,7 @@ impl<
             };
 
         Ok(UpsertReport {
-            prefix_new,
+            prefix_new: prefix_is_new,
             cas_count: insert_retry_count,
             mui_new: mui_is_new.is_none(),
             mui_count: mui_is_new.unwrap_or(1),

From 7cbc6e9bab06f0506a559a5fcecc1830ab18db79 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 3 Dec 2024 16:11:50 +0100
Subject: [PATCH 007/147] improved load_mrt and new prefix/mui reporting

---
 src/bin/load_mrt.rs                   | 509 +++++++++++++++++---------
 src/local_array/store/atomic_types.rs |   2 +-
 src/local_array/store/custom_alloc.rs |  25 +-
 3 files changed, 362 insertions(+), 174 deletions(-)

diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index 1e20f19a..7dfd8dd8 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -16,24 +16,26 @@ use rotonda_store::PublicRecord;
 use routecore::mrt::MrtFile;
 
 use rand::seq::SliceRandom;
+use routecore::mrt::RibEntryIterator;
+use routecore::mrt::TableDumpIterator;
 
 #[derive(Clone, Debug)]
 //struct MyPaMap(PaMap);
-struct MyPaMap(Vec<u8>);
+struct PaBytes(Vec<u8>);
 
-impl std::fmt::Display for MyPaMap {
+impl std::fmt::Display for PaBytes {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{:?}", self.0)
     }
 }
 
-impl AsRef<[u8]> for MyPaMap {
+impl AsRef<[u8]> for PaBytes {
     fn as_ref(&self) -> &[u8] {
         self.0.as_ref()
     }
 }
 
-impl rotonda_store::Meta for MyPaMap {
+impl rotonda_store::Meta for PaBytes {
     type Orderable<'a> = u32;
 
     type TBI = u32;
@@ -43,6 +45,75 @@ impl rotonda_store::Meta for MyPaMap {
     }
 }
 
+#[derive(Copy, Clone, Default)]
+struct UpsertCounters {
+    unique_prefixes: usize,
+    unique_routes: usize,
+    total_routes: usize,
+}
+
+impl std::ops::AddAssign for UpsertCounters {
+    fn add_assign(&mut self, rhs: Self) {
+        self.unique_prefixes += rhs.unique_prefixes;
+        self.unique_routes += rhs.unique_routes;
+        self.total_routes += rhs.total_routes;
+    }
+}
+
+impl std::ops::Add for UpsertCounters {
+    type Output = UpsertCounters;
+
+    fn add(self, rhs: Self) -> Self::Output {
+        Self {
+            unique_prefixes: self.unique_prefixes + rhs.unique_prefixes,
+            unique_routes: self.unique_routes + rhs.unique_routes,
+            total_routes: self.total_routes + rhs.total_routes,
+        }
+    }
+}
+
+impl std::fmt::Display for UpsertCounters {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        writeln!(f, "inserted unique prefixes:\t{}", self.unique_prefixes)?;
+        writeln!(f, "inserted unique routes:\t\t{}", self.unique_routes)?;
+        writeln!(f, "total routes:\t\t\t{}", self.total_routes)?;
+        writeln!(
+            f,
+            "overwritten routes:\t\t{}",
+            self.total_routes - self.unique_routes
+        )
+    }
+}
+
+fn counter_update(
+    counters: &mut UpsertCounters,
+) -> impl FnMut(UpsertReport) -> UpsertCounters + '_ {
+    move |r| match (r.prefix_new, r.mui_new) {
+        // new prefix, new mui
+        (true, true) => {
+            counters.unique_prefixes += 1;
+            counters.unique_routes += 1;
+            counters.total_routes += 1;
+            *counters
+        }
+        // old prefix, new mui
+        (false, true) => {
+            counters.unique_routes += 1;
+            counters.total_routes += 1;
+            *counters
+        }
+        // old prefix, old mui
+        (false, false) => {
+            counters.total_routes += 1;
+            *counters
+        }
+        // new prefix, old mui
+        (true, false) => {
+            panic!("THIS DOESN'T MEAN ANYTHING!");
+        }
+    }
+}
+
 #[derive(Parser)]
 #[command(version, about, long_about = None)]
 struct Cli {
@@ -89,191 +160,301 @@ fn insert<T: rotonda_store::Meta>(
         .inspect_err(|e| eprintln!("Error in test_store: {e}"))
 }
 
-fn main() {
-    let args = Cli::parse();
-
-    let mut store = MultiThreadedStore::<MyPaMap>::new().unwrap();
-    let t_total = Instant::now();
-    let mut routes_total: usize = 0;
-    let mut mib_total: usize = 0;
-    for mrtfile in &args.mrt_files {
-        if !args.single_store {
-            store = MultiThreadedStore::<MyPaMap>::new().unwrap();
-        }
-
-        let file = File::open(mrtfile).unwrap();
-        let mmap = unsafe { Mmap::map(&file).unwrap() };
-        println!("{}: {}MiB", mrtfile.to_string_lossy(), mmap.len() >> 20);
-        mib_total += mmap.len() >> 20;
-
-        let t_0 = Instant::now();
-        let t_prev = t_0;
-
-        let mrt_file = MrtFile::new(&mmap[..]);
-        let tables = mrt_file.tables().unwrap();
-
-        if !args.parse_only && (args.shuffle || args.prime || args.mt_prime) {
-            let mut prefixes = mrt_file
-                .tables()
-                .unwrap()
-                .par_bridge()
-                .map(|(_fam, reh)| {
-                    let iter = routecore::mrt::SingleEntryIterator::new(reh);
-                    iter.map(|(prefix, _, _)| prefix)
-                })
-                .flatten_iter()
-                .collect::<Vec<_>>();
+fn par_load_prefixes(
+    mrt_file: &MrtFile,
+    shuffle: bool,
+) -> Vec<(Prefix, u16)> {
+    let t0 = std::time::Instant::now();
+    let mut prefixes = mrt_file
+        .tables()
+        .unwrap()
+        .par_bridge()
+        .map(|(_fam, reh)| {
+            let iter = routecore::mrt::SingleEntryIterator::new(reh);
+            iter.map(|(prefix, peer_idx, _)| (prefix, peer_idx))
+        })
+        .flatten_iter()
+        .collect::<Vec<_>>();
 
-            eprintln!(
-                "collected {} prefixes to prime store, took {}ms",
-                prefixes.len(),
-                t_prev.elapsed().as_millis(),
-            );
+    eprintln!(
+        "loaded file with {} prefixes in {}ms",
+        prefixes.len(),
+        t0.elapsed().as_millis()
+    );
 
-            if args.shuffle {
-                let t_s = Instant::now();
-                eprint!("shuffling before priming... ");
-                prefixes.shuffle(&mut rand::thread_rng());
-                eprintln!("done! took {}ms", t_s.elapsed().as_millis());
-            }
+    if shuffle {
+        let t_s = Instant::now();
+        eprint!("shuffling prefixes... ");
+        prefixes.shuffle(&mut rand::thread_rng());
+        eprintln!("done! took {}ms", t_s.elapsed().as_millis());
+    }
 
-            let t_prev = Instant::now();
-
-            if args.mt_prime {
-                if prefixes
-                    .par_iter()
-                    .try_for_each(|p| {
-                        insert(
-                            &store,
-                            p,
-                            0,
-                            0,
-                            RouteStatus::InActive,
-                            MyPaMap(vec![]),
-                        )
-                        .map(|_| ())
-                    })
-                    .is_err()
-                {
-                    return;
-                }
-            } else {
-                for p in &prefixes {
-                    if insert(
-                        &store,
-                        p,
-                        0,
-                        0,
-                        RouteStatus::InActive,
-                        MyPaMap(vec![]),
-                    )
-                    .is_err()
-                    {
-                        return;
-                    }
-                }
-            }
-            eprintln!(
-                "primed store with {} prefixes, took {}ms",
-                prefixes.len(),
-                t_prev.elapsed().as_millis(),
-            );
-        }
+    prefixes
+}
 
-        let t_prev = Instant::now();
-
-        let mut num_routes = 0;
-        if args.mt {
-            // TODO turn into a try_fold or something, allowing to return
-            // on the first error
-            num_routes = tables
-                .par_bridge()
-                .map(|(_fam, reh)| {
-                    let iter = routecore::mrt::SingleEntryIterator::new(reh);
-
-                    let mut cnt = 0;
-                    for e in iter {
-                        cnt += 1;
-                        let (prefix, peer_idx, pamap) = e;
-                        let mui = peer_idx.into();
-                        let ltime = 0;
-                        let val = MyPaMap(pamap);
-
-                        if !args.parse_only {
-                            let _ = insert(
-                                &store,
-                                &prefix,
-                                mui,
-                                ltime,
-                                RouteStatus::Active,
-                                val,
-                            );
-                        }
-                    }
-                    cnt
-                })
-                .fold(|| 0, |sum, e| sum + e)
-                .sum::<usize>();
-        } else {
-            // single threaded
-            let rib_entries = mrt_file.rib_entries().unwrap();
-
-            for e in rib_entries {
-                num_routes += 1;
-                let (_, peer_idx, _, prefix, pamap) = e;
+fn mt_parse_and_insert_table(
+    tables: TableDumpIterator<&[u8]>,
+    store: Option<&MultiThreadedStore<PaBytes>>,
+) -> UpsertCounters {
+    let counters = tables
+        .par_bridge()
+        .map(|(_fam, reh)| {
+            let mut local_counters = UpsertCounters::default();
+            let iter = routecore::mrt::SingleEntryIterator::new(reh);
+            // let mut cnt = 0;
+            for (prefix, peer_idx, pa_bytes) in iter {
+                // cnt += 1;
+                // let (prefix, peer_idx, pa_bytes) = e;
                 let mui = peer_idx.into();
                 let ltime = 0;
-                let val = MyPaMap(pamap);
+                let val = PaBytes(pa_bytes);
 
-                if !args.parse_only {
-                    let _ = insert(
-                        &store,
+                if let Some(store) = store {
+                    let counters = insert(
+                        store,
                         &prefix,
                         mui,
                         ltime,
                         RouteStatus::Active,
                         val,
-                    );
+                    )
+                    .map(move |r| match (r.prefix_new, r.mui_new) {
+                        // new prefix, new mui
+                        (true, true) => UpsertCounters {
+                            unique_prefixes: 1,
+                            unique_routes: 1,
+                            total_routes: 1,
+                        },
+                        // old prefix, new mui
+                        (false, true) => UpsertCounters {
+                            unique_prefixes: 0,
+                            unique_routes: 1,
+                            total_routes: 1,
+                        },
+                        // old prefix, old mui
+                        (false, false) => UpsertCounters {
+                            unique_prefixes: 0,
+                            unique_routes: 0,
+                            total_routes: 1,
+                        },
+                        // new prefix, old mui
+                        (true, false) => {
+                            panic!("THIS DOESN'T MEAN ANYTHING!");
+                        }
+                    })
+                    .unwrap();
+
+                    local_counters += counters;
                 }
             }
+            local_counters
+        })
+        .fold(UpsertCounters::default, |acc, c| acc + c)
+        .reduce(UpsertCounters::default, |acc, c| acc + c);
+
+    println!("{}", counters);
+
+    counters
+}
+
+fn st_parse_and_insert_table(
+    entries: RibEntryIterator<&[u8]>,
+    store: Option<&MultiThreadedStore<PaBytes>>,
+) -> UpsertCounters {
+    let mut counters = UpsertCounters::default();
+    let mut cnt = 0;
+    let t0 = std::time::Instant::now();
+
+    for (_, peer_idx, _, prefix, pamap) in entries {
+        cnt += 1;
+        let mui = peer_idx.into();
+        let ltime = 0;
+        let val = PaBytes(pamap);
+
+        if let Some(store) = store {
+            insert(store, &prefix, mui, ltime, RouteStatus::Active, val)
+                .map(counter_update(&mut counters))
+                .unwrap();
         }
-        if !args.parse_only {
-            let threading = if args.mt {
-                " (multi-threaded)"
-            } else {
-                " (single-threaded)"
-            };
-            eprintln!(
-                "inserted {} routes{}, took {}ms, total for this file {}ms",
-                num_routes,
-                threading,
-                t_prev.elapsed().as_millis(),
-                t_0.elapsed().as_millis(),
-            );
-        } else {
+    }
+
+    println!("primed {} prefixes in {}ms", cnt, t0.elapsed().as_millis());
+    println!("{}", counters);
+
+    counters
+}
+
+fn mt_prime_store(
+    prefixes: &Vec<(Prefix, u16)>,
+    store: &MultiThreadedStore<PaBytes>,
+) -> UpsertCounters {
+    let t0 = std::time::Instant::now();
+
+    let counters = prefixes
+        .par_iter()
+        .fold(UpsertCounters::default, |mut acc, p| {
+            insert(
+                store,
+                &p.0,
+                p.1 as u32,
+                0,
+                RouteStatus::InActive,
+                PaBytes(vec![]),
+            )
+            .map(counter_update(&mut acc))
+            .unwrap()
+        })
+        .reduce(UpsertCounters::default, |c1, c2| c1 + c2);
+
+    println!(
+        "primed {} prefixes in {}ms",
+        prefixes.len(),
+        t0.elapsed().as_millis()
+    );
+
+    // println!("{}", counters);
+
+    counters
+}
+
+fn st_prime_store(
+    prefixes: &Vec<(Prefix, u16)>,
+    store: &MultiThreadedStore<PaBytes>,
+) -> UpsertCounters {
+    let mut counters = UpsertCounters::default();
+
+    for p in prefixes {
+        insert(
+            store,
+            &p.0,
+            p.1 as u32,
+            0,
+            RouteStatus::InActive,
+            PaBytes(vec![]),
+        )
+        .map(counter_update(&mut counters))
+        .unwrap();
+    }
+
+    counters
+}
+
+fn main() {
+    let args = Cli::parse();
+
+    let t_total = Instant::now();
+
+    let mut global_counters = UpsertCounters::default();
+    let mut mib_total: usize = 0;
+    let mut inner_stores = vec![];
+
+    // Create all the stores necessary, and if at least one is created, create
+    // a reference to the first one.
+    let mut store = match &args {
+        a if a.single_store && a.parse_only => {
             eprintln!(
-                "parsed {}, no insertions,  total for this file {}ms",
-                num_routes,
-                t_0.elapsed().as_millis(),
+                "Can't combine --parse-only and --single-store. 
+                Make up your mind."
             );
+            return;
+        }
+        a if a.single_store => {
+            inner_stores.push(MultiThreadedStore::<PaBytes>::default());
+            println!("Created a single-store");
+            Some(&inner_stores[0])
+        }
+        a if a.parse_only => {
+            println!("No store created (parse only)");
+            None
+        }
+        _ => {
+            for _ in &args.mrt_files {
+                inner_stores.push(MultiThreadedStore::<PaBytes>::default());
+            }
+            println!("Number of created stores: {}", inner_stores.len());
+            Some(&inner_stores[0])
+        }
+    };
+
+    // Loop over all the mrt-files specified as arguments
+    for (f_index, mrtfile) in args.mrt_files.iter().enumerate() {
+        print!("file #{} ", f_index);
+
+        let file = File::open(mrtfile).unwrap();
+        let mmap = unsafe { Mmap::map(&file).unwrap() };
+        println!("{} ({}MiB)", mrtfile.to_string_lossy(), mmap.len() >> 20);
+        mib_total += mmap.len() >> 20;
+
+        let mrt_file = MrtFile::new(&mmap[..]);
+
+        if !args.single_store && !args.parse_only {
+            println!("use store #{}", f_index);
+            store = Some(&inner_stores[f_index]);
         }
-        eprintln!("--------");
+        // Load the mrt file, maybe shuffle, and maybe prime the store
+        match &args {
+            a if a.mt_prime && a.prime => {
+                eprintln!(
+                    "--prime and --mt-prime can't be combined.
+                    Make up your mind."
+                );
+                return;
+            }
+            a if a.prime => {
+                let prefixes = par_load_prefixes(&mrt_file, a.shuffle);
+                st_prime_store(&prefixes, store.unwrap());
+            }
+            a if a.mt_prime => {
+                let prefixes = par_load_prefixes(&mrt_file, a.shuffle);
+                mt_prime_store(&prefixes, store.unwrap());
+            }
+            _ => {}
+        };
+
+        // Parse the prefixes in the file, and maybe insert them into the
+        // Store
+        global_counters += match &args {
+            a if a.mt => {
+                let tables = mrt_file.tables().unwrap();
+                mt_parse_and_insert_table(tables, store)
+            }
+            _ => {
+                let entries = mrt_file.rib_entries().unwrap();
+                st_parse_and_insert_table(entries, store)
+            }
+        };
+    }
 
-        routes_total += num_routes;
+    if let Some(store) = store {
+        store.flush_to_disk()
     }
 
-    store.flush_to_disk();
+    // eprintln!(
+    //     "processed {} routes in {} files in {:.2}s",
+    //     routes_count,
+    //     args.mrt_files.len(),
+    //     t_total.elapsed().as_millis() as f64 / 1000.0
+    // );
+
+    eprintln!("upsert counters");
+    eprintln!("---------------");
+    eprintln!("{}", global_counters);
+
+    if let Some(store) = store {
+        eprintln!(
+            "Approximate number of items persisted {} + {} = {}",
+            store.approx_persisted_items().0,
+            store.approx_persisted_items().1,
+            store.approx_persisted_items().0
+                + store.approx_persisted_items().1
+        );
+    }
 
-    eprintln!(
-        "Processed {} routes in {} files in {:.2}s",
-        routes_total,
-        args.mrt_files.len(),
-        t_total.elapsed().as_millis() as f64 / 1000.0
-    );
     eprintln!(
         "{:.0} routes per second\n\
             {:.0} MiB per second",
-        routes_total as f64 / t_total.elapsed().as_secs() as f64,
+        global_counters.total_routes as f64
+            / t_total.elapsed().as_secs() as f64,
         mib_total as f64 / t_total.elapsed().as_secs() as f64
     );
 }
diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index c26b0a13..9b424056 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -8,7 +8,7 @@ use std::{
 use crossbeam_epoch::{self as epoch, Atomic};
 
 use crossbeam_utils::Backoff;
-use log::{debug, info, log_enabled, trace};
+use log::{debug, log_enabled, trace};
 
 use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 23641ac5..7b893588 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -707,8 +707,10 @@ impl<
         guard: &Guard,
     ) -> Result<UpsertReport, PrefixStoreError> {
         let mut prefix_is_new = true;
+        let mut mui_is_new = true;
+        let mut mui_count = 0;
 
-        let (mui_is_new, insert_retry_count) =
+        let insert_retry_count =
             match self.non_recursive_retrieve_prefix_mut(prefix) {
                 // There's no StoredPrefix at this location yet. Create a new
                 // PrefixRecord and try to store it in the empty slot.
@@ -724,17 +726,22 @@ impl<
 
                     let mui = record.multi_uniq_id;
                     let res = locked_prefix.record_map.upsert_record(record);
-                    let mut mui_is_new = None;
+
+                    if res.0.is_some() {
+                        mui_is_new = false;
+                        prefix_is_new = false;
+                    }
+
                     let retry_count = res.1;
 
                     #[cfg(feature = "persist")]
                     if let Some((p_rec, min)) = res.0 {
+                        mui_count = min;
                         self.persist_record(prefix, mui, p_rec);
-                        mui_is_new = Some(min);
                     }
 
                     self.counters.inc_prefixes_count(prefix.get_len());
-                    (mui_is_new, retry_count)
+                    retry_count
                 }
                 // There already is a StoredPrefix with a record at this
                 // location.
@@ -757,13 +764,13 @@ impl<
 
                     let mui = record.multi_uniq_id;
                     let res = stored_prefix.record_map.upsert_record(record);
+                    mui_is_new = res.0.is_none();
                     let retry_count = res.1;
-                    let mut mui_is_new = None;
 
                     #[cfg(feature = "persist")]
                     if let Some((p_rec, min)) = res.0 {
                         self.persist_record(prefix, mui, p_rec);
-                        mui_is_new = Some(min);
+                        mui_count = min;
                     }
 
                     if let Some(tbi) = update_path_selections {
@@ -771,15 +778,15 @@ impl<
                             .calculate_and_store_best_backup(&tbi, guard)?;
                     }
 
-                    (mui_is_new, retry_count)
+                    retry_count
                 }
             };
 
         Ok(UpsertReport {
             prefix_new: prefix_is_new,
             cas_count: insert_retry_count,
-            mui_new: mui_is_new.is_none(),
-            mui_count: mui_is_new.unwrap_or(1),
+            mui_new: mui_is_new,
+            mui_count,
         })
     }
 

From 2b805d61f9988185043604f73e693861181ad1ab Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 4 Dec 2024 13:56:21 +0100
Subject: [PATCH 008/147] add verifcation to test_store; add status to
 persistance key

---
 proc_macros/src/lib.rs                 |  39 +++++++-
 src/bin/load_mrt.rs                    | 123 +++++++++++++++++++------
 src/local_array/store/atomic_types.rs  |  74 ++++++++++++---
 src/local_array/store/custom_alloc.rs  | 110 ++++++++++++++++++----
 src/local_array/store/default_store.rs |   4 +-
 src/prefix_record.rs                   |  20 ++--
 6 files changed, 300 insertions(+), 70 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 4155024a..2abdcf46 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -1625,11 +1625,37 @@ pub fn create_store(
                 }
             }
 
+            // Disk Persistence
+
+            pub fn get_values_for_prefix(&self, prefix: &Prefix) ->
+                Vec<(u32, u64, u8, Vec<u8>)> {
+                match prefix.is_v4() {
+                    true => self.v4.store.persistence
+                        .get_values_for_prefix(
+                            PrefixId::<IPv4>::from(*prefix)
+                        ),
+                    false => self.v6.store.persistence
+                        .get_values_for_prefix(
+                            PrefixId::<IPv6>::from(*prefix)
+                        )
+                }
+            }
+
             /// Persist all the non-unique (prefix, mui, ltime) tuples
             /// with their values to disk
-            pub fn flush_to_disk(&self) {
-                self.v4.store.persistence.flush_to_disk();
-                self.v6.store.persistence.flush_to_disk();
+            pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
+                if let Some(segment) =
+                    self.v4.store.persistence.flush_to_disk()? {
+                    self.v4.store.persistence.register_segments(&[segment])?
+                }
+
+                if let Some(segment) =
+                    self.v6.store.persistence.flush_to_disk()?
+                {
+                    self.v6.store.persistence.register_segments(&[segment])?
+                }
+
+                Ok(())
             }
 
             /// Return the approximate number of items that are persisted
@@ -1640,6 +1666,13 @@ pub fn create_store(
                     self.v6.store.persistence.approximate_len()
                 )
             }
+
+            /// Return an estimation of the disk space currently used by the
+            /// store in bytes.
+            pub fn disk_space(&self) -> u64 {
+                self.v4.store.persistence.disk_space() +
+                self.v6.store.persistence.disk_space()
+            }
         }
     };
 
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index 7dfd8dd8..7905c016 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -1,3 +1,4 @@
+use std::collections::BTreeSet;
 use std::fmt;
 use std::fs::File;
 use std::path::PathBuf;
@@ -144,6 +145,10 @@ struct Cli {
     /// Don't insert in store, only parse MRT_FILES
     #[arg(long, default_value_t = false)]
     parse_only: bool,
+
+    /// Verify the persisted entries
+    #[arg(long, default_value_t = false)]
+    verify: bool,
 }
 
 fn insert<T: rotonda_store::Meta>(
@@ -195,18 +200,19 @@ fn par_load_prefixes(
 fn mt_parse_and_insert_table(
     tables: TableDumpIterator<&[u8]>,
     store: Option<&MultiThreadedStore<PaBytes>>,
-) -> UpsertCounters {
+    ltime: u64,
+) -> (UpsertCounters, Vec<Prefix>) {
     let counters = tables
         .par_bridge()
         .map(|(_fam, reh)| {
             let mut local_counters = UpsertCounters::default();
             let iter = routecore::mrt::SingleEntryIterator::new(reh);
+            let overwritten_prefixes = &mut vec![];
             // let mut cnt = 0;
             for (prefix, peer_idx, pa_bytes) in iter {
                 // cnt += 1;
                 // let (prefix, peer_idx, pa_bytes) = e;
                 let mui = peer_idx.into();
-                let ltime = 0;
                 let val = PaBytes(pa_bytes);
 
                 if let Some(store) = store {
@@ -218,7 +224,7 @@ fn mt_parse_and_insert_table(
                         RouteStatus::Active,
                         val,
                     )
-                    .map(move |r| match (r.prefix_new, r.mui_new) {
+                    .map(|r| match (r.prefix_new, r.mui_new) {
                         // new prefix, new mui
                         (true, true) => UpsertCounters {
                             unique_prefixes: 1,
@@ -232,11 +238,14 @@ fn mt_parse_and_insert_table(
                             total_routes: 1,
                         },
                         // old prefix, old mui
-                        (false, false) => UpsertCounters {
-                            unique_prefixes: 0,
-                            unique_routes: 0,
-                            total_routes: 1,
-                        },
+                        (false, false) => {
+                            overwritten_prefixes.push(prefix);
+                            UpsertCounters {
+                                unique_prefixes: 0,
+                                unique_routes: 0,
+                                total_routes: 1,
+                            }
+                        }
                         // new prefix, old mui
                         (true, false) => {
                             panic!("THIS DOESN'T MEAN ANYTHING!");
@@ -247,12 +256,24 @@ fn mt_parse_and_insert_table(
                     local_counters += counters;
                 }
             }
-            local_counters
+            (local_counters, overwritten_prefixes.clone())
         })
-        .fold(UpsertCounters::default, |acc, c| acc + c)
-        .reduce(UpsertCounters::default, |acc, c| acc + c);
+        .fold(
+            || (UpsertCounters::default(), vec![]),
+            |mut acc, c| {
+                acc.1.extend(c.1);
+                (acc.0 + c.0, acc.1)
+            },
+        )
+        .reduce(
+            || (UpsertCounters::default(), vec![]),
+            |mut acc, c| {
+                acc.1.extend(c.1);
+                (acc.0 + c.0, acc.1)
+            },
+        );
 
-    println!("{}", counters);
+    println!("{}", counters.0);
 
     counters
 }
@@ -260,6 +281,7 @@ fn mt_parse_and_insert_table(
 fn st_parse_and_insert_table(
     entries: RibEntryIterator<&[u8]>,
     store: Option<&MultiThreadedStore<PaBytes>>,
+    ltime: u64,
 ) -> UpsertCounters {
     let mut counters = UpsertCounters::default();
     let mut cnt = 0;
@@ -268,7 +290,6 @@ fn st_parse_and_insert_table(
     for (_, peer_idx, _, prefix, pamap) in entries {
         cnt += 1;
         let mui = peer_idx.into();
-        let ltime = 0;
         let val = PaBytes(pamap);
 
         if let Some(store) = store {
@@ -278,7 +299,11 @@ fn st_parse_and_insert_table(
         }
     }
 
-    println!("primed {} prefixes in {}ms", cnt, t0.elapsed().as_millis());
+    println!(
+        "parsed & inserted {} prefixes in {}ms",
+        cnt,
+        t0.elapsed().as_millis()
+    );
     println!("{}", counters);
 
     counters
@@ -347,6 +372,7 @@ fn main() {
     let mut global_counters = UpsertCounters::default();
     let mut mib_total: usize = 0;
     let mut inner_stores = vec![];
+    let mut overwritten_prefixes = BTreeSet::new();
 
     // Create all the stores necessary, and if at least one is created, create
     // a reference to the first one.
@@ -359,8 +385,8 @@ fn main() {
             return;
         }
         a if a.single_store => {
-            inner_stores.push(MultiThreadedStore::<PaBytes>::default());
-            println!("Created a single-store");
+            inner_stores.push(MultiThreadedStore::<PaBytes>::new().unwrap());
+            println!("created a single-store\n");
             Some(&inner_stores[0])
         }
         a if a.parse_only => {
@@ -369,7 +395,8 @@ fn main() {
         }
         _ => {
             for _ in &args.mrt_files {
-                inner_stores.push(MultiThreadedStore::<PaBytes>::default());
+                inner_stores
+                    .push(MultiThreadedStore::<PaBytes>::new().unwrap());
             }
             println!("Number of created stores: {}", inner_stores.len());
             Some(&inner_stores[0])
@@ -416,17 +443,25 @@ fn main() {
         global_counters += match &args {
             a if a.mt => {
                 let tables = mrt_file.tables().unwrap();
-                mt_parse_and_insert_table(tables, store)
+                let (counters, ow_pfxs) =
+                    mt_parse_and_insert_table(tables, store, f_index as u64);
+                if args.verify {
+                    overwritten_prefixes.extend(&ow_pfxs)
+                }
+                counters
             }
             _ => {
                 let entries = mrt_file.rib_entries().unwrap();
-                st_parse_and_insert_table(entries, store)
+                st_parse_and_insert_table(entries, store, f_index as u64)
             }
         };
     }
 
     if let Some(store) = store {
-        store.flush_to_disk()
+        let res = store.flush_to_disk();
+        if res.is_err() {
+            eprintln!("Persistence Error: {:?}", res);
+        }
     }
 
     // eprintln!(
@@ -436,25 +471,61 @@ fn main() {
     //     t_total.elapsed().as_millis() as f64 / 1000.0
     // );
 
-    eprintln!("upsert counters");
-    eprintln!("---------------");
-    eprintln!("{}", global_counters);
+    println!("upsert counters");
+    println!("---------------");
+    println!("{}", global_counters);
 
     if let Some(store) = store {
-        eprintln!(
-            "Approximate number of items persisted {} + {} = {}",
+        println!("store in-memory counters");
+        println!("------------------------");
+        println!("prefixes:\t\t\t{:?}\n", store.prefixes_count());
+
+        println!("store persistence counters");
+        println!("--------------------------");
+        println!(
+            "approx. prefixes:\t\t{} + {} = {}",
             store.approx_persisted_items().0,
             store.approx_persisted_items().1,
             store.approx_persisted_items().0
                 + store.approx_persisted_items().1
         );
+        println!(
+            "disk size of persisted store:\t{}MiB\n",
+            store.disk_space() / (1024 * 1024)
+        );
     }
 
-    eprintln!(
+    println!(
         "{:.0} routes per second\n\
             {:.0} MiB per second",
         global_counters.total_routes as f64
             / t_total.elapsed().as_secs() as f64,
         mib_total as f64 / t_total.elapsed().as_secs() as f64
     );
+
+    if args.verify {
+        println!("\nverifying disk persistence...");
+        let mut max_len = 0;
+        for pfx in overwritten_prefixes {
+            let values = store.unwrap().get_values_for_prefix(&pfx);
+            if values.is_empty() {
+                eprintln!("Found empty prefix on disk");
+                eprintln!("prefix: {}", pfx);
+                return;
+            }
+            if values.len() > max_len {
+                max_len = values.len();
+                println!(
+                    "len {}: {} -> {:?}",
+                    max_len,
+                    pfx,
+                    store.unwrap().get_values_for_prefix(&pfx)
+                );
+            }
+            values
+                .iter()
+                .filter(|v| v.3.is_empty())
+                .for_each(|v| println!("withdraw for {}, mui {}", pfx, v.0))
+        }
+    }
 }
diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index 9b424056..b18baa13 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -273,20 +273,34 @@ impl std::fmt::Display for RouteStatus {
     }
 }
 
+impl From<RouteStatus> for u8 {
+    fn from(value: RouteStatus) -> Self {
+        match value {
+            RouteStatus::Active => 1,
+            RouteStatus::InActive => 2,
+            RouteStatus::Withdrawn => 3,
+        }
+    }
+}
+
 #[derive(Clone, Debug)]
 pub(crate) struct MultiMapValue<M> {
-    pub meta: M,
-    pub ltime: u64,
-    pub status: RouteStatus,
+    meta: M,
+    ltime: u64,
+    status: RouteStatus,
 }
 
 impl<M: Clone + AsRef<[u8]>> MultiMapValue<M> {
-    pub(crate) fn _new(meta: M, ltime: u64, status: RouteStatus) -> Self {
-        Self {
-            meta,
-            ltime,
-            status,
-        }
+    pub(crate) fn logical_time(&self) -> u64 {
+        self.ltime
+    }
+
+    pub(crate) fn meta(&self) -> &M {
+        &self.meta
+    }
+
+    pub(crate) fn status(&self) -> RouteStatus {
+        self.status
     }
 }
 
@@ -298,6 +312,12 @@ impl<M: crate::prefix_record::Meta + AsRef<[u8]>> std::fmt::Display
     }
 }
 
+// impl<M: Clone + AsRef<[u8]>> AsRef<[u8]> for MultiMapValue<M> {
+//     fn as_ref(&self) -> &[u8] {
+//         self.meta.as_ref()
+//     }
+// }
+
 impl<M: Meta + AsRef<[u8]>> From<PublicRecord<M>> for MultiMapValue<M> {
     fn from(value: PublicRecord<M>) -> Self {
         Self {
@@ -308,19 +328,47 @@ impl<M: Meta + AsRef<[u8]>> From<PublicRecord<M>> for MultiMapValue<M> {
     }
 }
 
-impl<M: Clone + AsRef<[u8]>> AsRef<[u8]> for MultiMapValue<M> {
-    fn as_ref(&self) -> &[u8] {
-        self.meta.as_ref()
+impl<M: Clone> From<(u32, MultiMapValue<M>)> for PublicRecord<M> {
+    fn from(value: (u32, MultiMapValue<M>)) -> Self {
+        Self {
+            multi_uniq_id: value.0,
+            meta: value.1.meta,
+            ltime: value.1.ltime,
+            status: value.1.status,
+        }
     }
 }
 
+// #[derive(Clone, Debug)]
+// pub(crate) enum MultiMapValueSwip<M> {
+//     Swizzled(MultiMapValue<M>),
+//     Unswizzled(u64),
+// }
+
+// impl<M: Clone + AsRef<[u8]>> MultiMapValueSwip<M> {
+//     pub(crate) fn logical_time(&self) -> u64 {
+//         match self {
+//             Self::Swizzled(v) => v.ltime,
+//             Self::Unswizzled(ltime) => *ltime,
+//         }
+//     }
+// }
+
+// impl<M: Clone + AsRef<[u8]>> AsRef<[u8]> for MultiMapValueSwip<M> {
+//     fn as_ref(&self) -> &[u8] {
+//         match self {
+//             Swizzled(v) => v.meta.as_ref(),
+//             Self::Unswizzled(_) =>
+//     }
+// }
+
 // ----------- MultiMap ------------------------------------------------------
 // This is the record that holds the aggregates at the top-level for a given
 // prefix.
 
 #[derive(Debug)]
 pub struct MultiMap<M: Meta>(
-    pub(crate) Arc<Mutex<std::collections::HashMap<u32, MultiMapValue<M>>>>,
+    Arc<Mutex<std::collections::HashMap<u32, MultiMapValue<M>>>>,
 );
 
 impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 7b893588..18741b5c 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -190,7 +190,7 @@ use log::{debug, info, log_enabled, trace};
 use crossbeam_epoch::{self as epoch, Atomic};
 use crossbeam_utils::Backoff;
 use epoch::{Guard, Owned};
-use lsm_tree::AbstractTree;
+use lsm_tree::{AbstractTree, Segment};
 use roaring::RoaringBitmap;
 
 use std::marker::PhantomData;
@@ -314,14 +314,40 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         self.0.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
     }
 
-    pub fn flush_to_disk(&self) {
-        self.0.flush_active_memtable(0).unwrap();
+    pub fn get_values_for_prefix(
+        &self,
+        prefix: PrefixId<AF>,
+    ) -> Vec<(u32, u64, u8, Vec<u8>)> {
+        let prefix_b = &prefix.as_bytes::<PREFIX_SIZE>();
+        (*self.0.prefix(prefix_b))
+            .into_iter()
+            .map(|kv| {
+                let kv = kv.unwrap();
+                let (_, mui, ltime, status) = Self::parse_key(kv.0.as_ref());
+                (mui, ltime, status, kv.1.as_ref().to_vec())
+            })
+            .collect::<Vec<_>>()
+    }
+
+    pub fn flush_to_disk(&self) -> Result<Option<Segment>, lsm_tree::Error> {
+        self.0.flush_active_memtable(0)
     }
 
     pub fn approximate_len(&self) -> usize {
         self.0.approximate_len()
     }
 
+    pub fn disk_space(&self) -> u64 {
+        self.0.disk_space()
+    }
+
+    pub fn register_segments(
+        &self,
+        segments: &[lsm_tree::Segment],
+    ) -> Result<(), lsm_tree::Error> {
+        self.0.register_segments(segments)
+    }
+
     #[cfg(feature = "persist")]
     pub fn persistence_key(
         // PREFIX_SIZE bytes
@@ -330,18 +356,57 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         mui: u32,
         // 8 bytes
         ltime: u64,
+        // 1 byte
+        status: RouteStatus,
     ) -> [u8; KEY_SIZE] {
         assert!(KEY_SIZE > PREFIX_SIZE);
         let key = &mut [0_u8; KEY_SIZE];
+
+        // prefix 5 or 17 bytes
         *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
+
+        // mui 4 bytes
         *key[PREFIX_SIZE..PREFIX_SIZE + 4]
             .first_chunk_mut::<4>()
             .unwrap() = mui.to_le_bytes();
-        *key[PREFIX_SIZE + 4..].first_chunk_mut::<8>().unwrap() =
-            ltime.to_le_bytes();
+
+        // ltime 8 bytes
+        *key[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
+            .first_chunk_mut::<8>()
+            .unwrap() = ltime.to_le_bytes();
+
+        // status 1 byte
+        key[PREFIX_SIZE + 12] = status.into();
 
         *key
     }
+
+    #[cfg(feature = "persist")]
+    pub fn parse_key(bytes: &[u8]) -> ([u8; PREFIX_SIZE], u32, u64, u8) {
+        (
+            // prefix 5 or 17 bytes
+            *bytes.first_chunk::<PREFIX_SIZE>().unwrap(),
+            // mui 4 bytes
+            u32::from_le_bytes(
+                *bytes[PREFIX_SIZE..PREFIX_SIZE + 4]
+                    .first_chunk::<4>()
+                    .unwrap(),
+            ),
+            // ltime 8 bytes
+            u64::from_le_bytes(
+                *bytes[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
+                    .first_chunk::<8>()
+                    .unwrap(),
+            ),
+            // status 1 byte
+            bytes[PREFIX_SIZE + 12],
+        )
+    }
+
+    #[cfg(feature = "persist")]
+    pub fn parse_prefix(bytes: &[u8]) -> [u8; PREFIX_SIZE] {
+        *bytes.first_chunk::<PREFIX_SIZE>().unwrap()
+    }
 }
 
 impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
@@ -370,7 +435,7 @@ pub struct CustomAllocStorage<
     #[cfg(feature = "persist")]
     pub persistence: PersistTree<AF, PREFIX_SIZE, KEY_SIZE>,
     pub default_route_prefix_serial: AtomicUsize,
-    // Global Roaring Bitmap INdex that stores MUIs.
+    // Global Roaring BitMap INdex that stores MUIs.
     pub withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     pub counters: Counters,
     _m: PhantomData<M>,
@@ -727,9 +792,13 @@ impl<
                     let mui = record.multi_uniq_id;
                     let res = locked_prefix.record_map.upsert_record(record);
 
+                    // See if someone beat us to creating the record.
                     if res.0.is_some() {
                         mui_is_new = false;
                         prefix_is_new = false;
+                    } else {
+                        // No, we were the first, we created a new prefix
+                        self.counters.inc_prefixes_count(prefix.get_len());
                     }
 
                     let retry_count = res.1;
@@ -740,7 +809,6 @@ impl<
                         self.persist_record(prefix, mui, p_rec);
                     }
 
-                    self.counters.inc_prefixes_count(prefix.get_len());
                     retry_count
                 }
                 // There already is a StoredPrefix with a record at this
@@ -749,12 +817,12 @@ impl<
                     if log_enabled!(log::Level::Debug) {
                         debug!(
                         "{} store: Found existing prefix record for {}/{}",
-                        std::thread::current()
-                            .name()
-                            .unwrap_or("unnamed-thread"),
-                        prefix.get_net(),
-                        prefix.get_len()
-                    );
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread"),
+                            prefix.get_net(),
+                            prefix.get_len()
+                        );
                     }
                     prefix_is_new = false;
 
@@ -950,7 +1018,8 @@ impl<
                     // search_prefix_id anywhere. Return the end-of-the-chain
                     // StoredPrefix, so the caller can attach a new one.
                     trace!(
-                        "no record. returning last found record in level {}, with index {}.",
+                        "no record. returning last found record in level
+                        {}, with index {}.",
                         level,
                         index
                     );
@@ -1265,11 +1334,20 @@ impl<
             PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
                 prefix,
                 mui,
-                record.ltime,
+                record.logical_time(),
+                record.status(),
             ),
-            record.as_ref(),
+            record.meta().as_ref(),
         );
     }
+
+    // #[cfg(feature = "persist")]
+    // fn get_prefix(
+    //     &self,
+    //     prefix: PrefixId<AF>,
+    // ) -> Vec<(([u8; PREFIX_SIZE], u32, u64), Vec<u8>)> {
+    //     self.persistence.prefix(prefix)
+    // }
 }
 
 //------------ Upsert -------------------------------------------------------
diff --git a/src/local_array/store/default_store.rs b/src/local_array/store/default_store.rs
index a9893765..155e2b1d 100644
--- a/src/local_array/store/default_store.rs
+++ b/src/local_array/store/default_store.rs
@@ -4,9 +4,9 @@ use std::fmt;
 
 // The default stride sizes for IPv4, IPv6, resp.
 #[create_store((
-    ([5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 5, 17),
+    ([5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 5, 18),
     ([4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-    4, 4, 4, 4, 4, 4, 4, 4, 4], 17, 29)
+    4, 4, 4, 4, 4, 4, 4, 4, 4], 17, 30)
 ))]
 struct DefaultStore;
 
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index 5967268a..752f8fee 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -261,16 +261,16 @@ impl<M: std::fmt::Display> std::fmt::Display for PublicRecord<M> {
     }
 }
 
-impl<M: Clone> From<(u32, MultiMapValue<M>)> for PublicRecord<M> {
-    fn from(value: (u32, MultiMapValue<M>)) -> Self {
-        Self {
-            multi_uniq_id: value.0,
-            meta: value.1.meta,
-            ltime: value.1.ltime,
-            status: value.1.status,
-        }
-    }
-}
+// impl<M: Clone> From<(u32, MultiMapValue<M>)> for PublicRecord<M> {
+//     fn from(value: (u32, MultiMapValue<M>)) -> Self {
+//         Self {
+//             multi_uniq_id: value.0,
+//             meta: value.1.meta,
+//             ltime: value.1.ltime,
+//             status: value.1.status,
+//         }
+//     }
+// }
 
 //------------ PublicPrefixRecord -------------------------------------------
 

From a67362fd6c2fae19604ab60c01bcdeb678a887f0 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 5 Dec 2024 14:22:23 +0100
Subject: [PATCH 009/147] Swip impl

---
 src/local_array/store/atomic_types.rs | 176 +++++++++++++++++---------
 src/local_array/store/custom_alloc.rs |   2 +-
 src/local_array/store/errors.rs       |  15 ++-
 3 files changed, 127 insertions(+), 66 deletions(-)

diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index b18baa13..92b4c8c9 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -284,23 +284,31 @@ impl From<RouteStatus> for u8 {
 }
 
 #[derive(Clone, Debug)]
-pub(crate) struct MultiMapValue<M> {
-    meta: M,
-    ltime: u64,
-    status: RouteStatus,
-}
+pub(crate) struct MultiMapValue<M>(Swip<RouteStatus, M>);
 
-impl<M: Clone + AsRef<[u8]>> MultiMapValue<M> {
+impl<M: Clone + AsRef<[u8]> + Debug> MultiMapValue<M> {
     pub(crate) fn logical_time(&self) -> u64 {
-        self.ltime
+        self.0.logical_time()
     }
 
-    pub(crate) fn meta(&self) -> &M {
-        &self.meta
+    pub(crate) fn meta(&self) -> Result<&M, PrefixStoreError> {
+        if let Swip::InMemory(_, _, meta) = &self.0 {
+            Ok(meta)
+        } else {
+            Err(PrefixStoreError::RecordNotInMemory)
+        }
     }
 
     pub(crate) fn status(&self) -> RouteStatus {
-        self.status
+        self.0.status()
+    }
+
+    pub(crate) fn set_status(&mut self, status: RouteStatus) {
+        self.0.set_status(status);
+    }
+
+    pub(crate) fn unswizzle(&mut self) {
+        self.0.unswizzle();
     }
 }
 
@@ -308,59 +316,96 @@ impl<M: crate::prefix_record::Meta + AsRef<[u8]>> std::fmt::Display
     for MultiMapValue<M>
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{} {} {}", self.meta, self.ltime, self.status)
+        write!(
+            f,
+            "{} {}",
+            // self.meta(),
+            self.0.logical_time(),
+            self.0.status()
+        )
     }
 }
 
-// impl<M: Clone + AsRef<[u8]>> AsRef<[u8]> for MultiMapValue<M> {
-//     fn as_ref(&self) -> &[u8] {
-//         self.meta.as_ref()
-//     }
-// }
-
 impl<M: Meta + AsRef<[u8]>> From<PublicRecord<M>> for MultiMapValue<M> {
     fn from(value: PublicRecord<M>) -> Self {
-        Self {
-            meta: value.meta,
-            ltime: value.ltime,
-            status: value.status,
-        }
+        Self(Swip::InMemory(value.ltime, value.status, value.meta))
     }
 }
 
-impl<M: Clone> From<(u32, MultiMapValue<M>)> for PublicRecord<M> {
-    fn from(value: (u32, MultiMapValue<M>)) -> Self {
-        Self {
-            multi_uniq_id: value.0,
-            meta: value.1.meta,
-            ltime: value.1.ltime,
-            status: value.1.status,
+impl<M: Clone + AsRef<[u8]> + Debug> TryFrom<(u32, &MultiMapValue<M>)>
+    for PublicRecord<M>
+{
+    type Error = PrefixStoreError;
+
+    fn try_from(
+        value: (u32, &MultiMapValue<M>),
+    ) -> Result<Self, Self::Error> {
+        match value.1 .0 {
+            Swip::InMemory(ltime, status, _) => Ok(Self {
+                multi_uniq_id: value.0,
+                meta: value.1.meta()?.clone(),
+                ltime,
+                status,
+            }),
+            Swip::HistoricalOnDisk(ltime, status) => Ok(Self {
+                multi_uniq_id: value.0,
+                meta: value.1.meta()?.clone(),
+                ltime,
+                status,
+            }),
+            Swip::OnDisk(_ltime, _status) => {
+                Err(PrefixStoreError::RecordNotInMemory)
+            }
         }
     }
 }
 
-// #[derive(Clone, Debug)]
-// pub(crate) enum MultiMapValueSwip<M> {
-//     Swizzled(MultiMapValue<M>),
-//     Unswizzled(u64),
-// }
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum Swip<T, M> {
+    // There's one ((prefix, mui), value) pair in the store and it only lives
+    // in memory
+    InMemory(u64, T, M),
+    // There are multiple ((prefix, mui), value) pairs, and the current one
+    // lives in memory, but there is/are more on disk
+    HistoricalOnDisk(u64, T),
+    // There is at least one ((prefix, mui), value) pair, it/they live(s)
+    // on disk only
+    OnDisk(u64, T),
+}
 
-// impl<M: Clone + AsRef<[u8]>> MultiMapValueSwip<M> {
-//     pub(crate) fn logical_time(&self) -> u64 {
-//         match self {
-//             Self::Swizzled(v) => v.ltime,
-//             Self::Unswizzled(ltime) => *ltime,
-//         }
-//     }
-// }
+impl<M: Debug> Swip<RouteStatus, M> {
+    fn logical_time(&self) -> u64 {
+        match self {
+            Swip::InMemory(ltime, _, _) => *ltime,
+            Swip::HistoricalOnDisk(ltime, _) => *ltime,
+            Swip::OnDisk(ltime, _) => *ltime,
+        }
+    }
 
-// impl<M: Clone + AsRef<[u8]>> AsRef<[u8]> for MultiMapValueSwip<M> {
-//     fn as_ref(&self) -> &[u8] {
-//         match self {
-//             Swizzled(v) => v.meta.as_ref(),
-//             Self::Unswizzled(_) =>
-//     }
-// }
+    fn status(&self) -> RouteStatus {
+        match self {
+            Swip::InMemory(_, status, _) => *status,
+            Swip::HistoricalOnDisk(_, status) => *status,
+            Swip::OnDisk(_, status) => *status,
+        }
+    }
+
+    fn set_status(&mut self, status: RouteStatus) {
+        match self {
+            Swip::InMemory(_, s, _) => {
+                *s = status;
+            }
+            Swip::HistoricalOnDisk(_, s) => *s = status,
+            Swip::OnDisk(_, s) => *s = status,
+        };
+    }
+
+    fn unswizzle(&mut self) {
+        if let Swip::InMemory(l, s, _) = self {
+            *self = Self::HistoricalOnDisk(*l, *s);
+        }
+    }
+}
 
 // ----------- MultiMap ------------------------------------------------------
 // This is the record that holds the aggregates at the top-level for a given
@@ -406,8 +451,8 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let record_map = c_map.lock().unwrap();
 
         record_map.get(&mui).and_then(|r| {
-            if r.status == RouteStatus::Active {
-                Some(PublicRecord::from((mui, r.clone())))
+            if r.status() == RouteStatus::Active {
+                Some(PublicRecord::try_from((mui, r)).unwrap())
             } else {
                 None
             }
@@ -417,11 +462,12 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     pub fn best_backup(&self, tbi: M::TBI) -> (Option<u32>, Option<u32>) {
         let c_map = Arc::clone(&self.0);
         let record_map = c_map.lock().unwrap();
-        let ord_routes =
-            record_map.iter().map(|r| (r.1.meta.as_orderable(tbi), r.0));
+        let ord_routes = record_map
+            .iter()
+            .map(|r| (r.1.meta().unwrap().as_orderable(tbi), *r.0));
         let (best, bckup) =
             routecore::bgp::path_selection::best_backup_generic(ord_routes);
-        (best.map(|b| *b.1), bckup.map(|b| *b.1))
+        (best.map(|b| b.1), bckup.map(|b| b.1))
     }
 
     pub(crate) fn get_record_for_mui_with_rewritten_status(
@@ -437,9 +483,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
             // untouched.
             let mut r = r.clone();
             if bmin.contains(mui) {
-                r.status = rewrite_status;
+                r.set_status(rewrite_status);
             }
-            PublicRecord::from((mui, r))
+            PublicRecord::try_from((mui, &r)).unwrap()
         })
     }
 
@@ -473,9 +519,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
             .map(move |r| {
                 let mut rec = r.1.clone();
                 if bmin.contains(*r.0) {
-                    rec.status = rewrite_status;
+                    rec.set_status(rewrite_status);
                 }
-                PublicRecord::from((*r.0, rec))
+                PublicRecord::try_from((*r.0, &rec)).unwrap()
             })
             .collect::<Vec<_>>()
     }
@@ -485,7 +531,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let record_map = c_map.lock().unwrap();
         record_map
             .iter()
-            .map(|r| PublicRecord::from((*r.0, r.1.clone())))
+            .map(|r| PublicRecord::try_from((*r.0, r.1)).unwrap())
             .collect::<Vec<_>>()
     }
 
@@ -501,8 +547,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         record_map
             .iter()
             .filter_map(|r| {
-                if r.1.status == RouteStatus::Active && !bmin.contains(*r.0) {
-                    Some(PublicRecord::from((*r.0, r.1.clone())))
+                if r.1.status() == RouteStatus::Active && !bmin.contains(*r.0)
+                {
+                    Some(PublicRecord::try_from((*r.0, r.1)).unwrap())
                 } else {
                     None
                 }
@@ -515,7 +562,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let c_map = Arc::clone(&self.0);
         let mut record_map = c_map.lock().unwrap();
         if let Some(rec) = record_map.get_mut(&mui) {
-            rec.status = RouteStatus::Withdrawn;
+            rec.set_status(RouteStatus::Withdrawn);
         }
     }
 
@@ -524,7 +571,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let record_map = Arc::clone(&self.0);
         let mut r_map = record_map.lock().unwrap();
         if let Some(rec) = r_map.get_mut(&mui) {
-            rec.status = RouteStatus::Active;
+            rec.set_status(RouteStatus::Active);
         }
     }
 
@@ -538,11 +585,14 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     ) -> (Option<(MultiMapValue<M>, usize)>, usize) {
         // let c_map = self.clone();
         let (mut record_map, retry_count) = self.guard_with_retry(0);
+        let key = record.multi_uniq_id;
 
         match record_map
             .insert(record.multi_uniq_id, MultiMapValue::from(record))
         {
             Some(exist_rec) => {
+                record_map.get_mut(&key).unwrap().unswizzle();
+                println!("{:?}", record_map.get(&key));
                 (Some((exist_rec, record_map.len())), retry_count)
             }
             _ => (None, retry_count),
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 18741b5c..193f426e 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -1337,7 +1337,7 @@ impl<
                 record.logical_time(),
                 record.status(),
             ),
-            record.meta().as_ref(),
+            record.meta().unwrap().as_ref(),
         );
     }
 
diff --git a/src/local_array/store/errors.rs b/src/local_array/store/errors.rs
index b83a96f4..c08fe609 100644
--- a/src/local_array/store/errors.rs
+++ b/src/local_array/store/errors.rs
@@ -7,7 +7,8 @@ pub enum PrefixStoreError {
     StoreNotReadyError,
     PathSelectionOutdated,
     PrefixNotFound,
-    BestPathNotFound
+    BestPathNotFound,
+    RecordNotInMemory,
 }
 
 impl std::error::Error for PrefixStoreError {}
@@ -32,7 +33,17 @@ impl fmt::Display for PrefixStoreError {
                 write!(f, "Error: The Prefix cannot be found.")
             }
             PrefixStoreError::BestPathNotFound => {
-                write!(f, "Error: The Prefix does not have a stored best path.")
+                write!(
+                    f,
+                    "Error: The Prefix does not have a stored best path."
+                )
+            }
+            PrefixStoreError::RecordNotInMemory => {
+                write!(
+                    f,
+                    "Error: The Record for this (prefix, mui) is not in \
+                    memory."
+                )
             }
         }
     }

From 78d8a9f0d9d2315cea766b5961b232f7474b9848 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 6 Dec 2024 22:20:48 +0100
Subject: [PATCH 010/147] PersistStrategy

---
 src/local_array/store/atomic_types.rs | 131 ++++++++++++++-------
 src/local_array/store/custom_alloc.rs | 157 +++++++++++++-------------
 src/local_array/tree.rs               |   6 +-
 src/prefix_record.rs                  |  19 +---
 4 files changed, 181 insertions(+), 132 deletions(-)

diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index 92b4c8c9..c5163514 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -13,6 +13,7 @@ use log::{debug, log_enabled, trace};
 use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
 
+use crate::custom_alloc::{PersistStrategy, PersistTree};
 use crate::local_array::tree::*;
 use crate::prefix_record::PublicRecord;
 use crate::prelude::Meta;
@@ -286,7 +287,7 @@ impl From<RouteStatus> for u8 {
 #[derive(Clone, Debug)]
 pub(crate) struct MultiMapValue<M>(Swip<RouteStatus, M>);
 
-impl<M: Clone + AsRef<[u8]> + Debug> MultiMapValue<M> {
+impl<M: Meta> MultiMapValue<M> {
     pub(crate) fn logical_time(&self) -> u64 {
         self.0.logical_time()
     }
@@ -310,11 +311,25 @@ impl<M: Clone + AsRef<[u8]> + Debug> MultiMapValue<M> {
     pub(crate) fn unswizzle(&mut self) {
         self.0.unswizzle();
     }
+
+    pub(crate) fn in_memory_record(value: PublicRecord<M>) -> Self {
+        Self(Swip::InMemory(value.ltime, value.status, value.meta))
+    }
+
+    pub(crate) fn partially_persisted_record(value: PublicRecord<M>) -> Self {
+        Self(Swip::PartiallyPersisted(
+            value.ltime,
+            value.status,
+            value.meta,
+        ))
+    }
+
+    pub(crate) fn persisted_record(value: PublicRecord<M>) -> Self {
+        Self(Swip::Persisted(value.ltime, value.status))
+    }
 }
 
-impl<M: crate::prefix_record::Meta + AsRef<[u8]>> std::fmt::Display
-    for MultiMapValue<M>
-{
+impl<M: Meta> std::fmt::Display for MultiMapValue<M> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
@@ -326,15 +341,13 @@ impl<M: crate::prefix_record::Meta + AsRef<[u8]>> std::fmt::Display
     }
 }
 
-impl<M: Meta + AsRef<[u8]>> From<PublicRecord<M>> for MultiMapValue<M> {
-    fn from(value: PublicRecord<M>) -> Self {
-        Self(Swip::InMemory(value.ltime, value.status, value.meta))
-    }
-}
+// impl<M: Meta> From<PublicRecord<M>> for MultiMapValue<M> {
+//     fn from(value: PublicRecord<M>) -> Self {
+//         Self(Swip::InMemory(value.ltime, value.status, value.meta))
+//     }
+// }
 
-impl<M: Clone + AsRef<[u8]> + Debug> TryFrom<(u32, &MultiMapValue<M>)>
-    for PublicRecord<M>
-{
+impl<M: Meta> TryFrom<(u32, &MultiMapValue<M>)> for PublicRecord<M> {
     type Error = PrefixStoreError;
 
     fn try_from(
@@ -347,13 +360,13 @@ impl<M: Clone + AsRef<[u8]> + Debug> TryFrom<(u32, &MultiMapValue<M>)>
                 ltime,
                 status,
             }),
-            Swip::HistoricalOnDisk(ltime, status) => Ok(Self {
+            Swip::PartiallyPersisted(ltime, status, _) => Ok(Self {
                 multi_uniq_id: value.0,
                 meta: value.1.meta()?.clone(),
                 ltime,
                 status,
             }),
-            Swip::OnDisk(_ltime, _status) => {
+            Swip::Persisted(_ltime, _status) => {
                 Err(PrefixStoreError::RecordNotInMemory)
             }
         }
@@ -366,27 +379,26 @@ pub(crate) enum Swip<T, M> {
     // in memory
     InMemory(u64, T, M),
     // There are multiple ((prefix, mui), value) pairs, and the current one
-    // lives in memory, but there is/are more on disk
-    HistoricalOnDisk(u64, T),
-    // There is at least one ((prefix, mui), value) pair, it/they live(s)
-    // on disk only
-    OnDisk(u64, T),
+    // lives in memory, but there is/are older persisted ones
+    PartiallyPersisted(u64, T, M),
+    // There is at least one persisted ((prefix, mui), value) pair
+    Persisted(u64, T),
 }
 
-impl<M: Debug> Swip<RouteStatus, M> {
+impl<M: Meta> Swip<RouteStatus, M> {
     fn logical_time(&self) -> u64 {
         match self {
             Swip::InMemory(ltime, _, _) => *ltime,
-            Swip::HistoricalOnDisk(ltime, _) => *ltime,
-            Swip::OnDisk(ltime, _) => *ltime,
+            Swip::PartiallyPersisted(ltime, _, _) => *ltime,
+            Swip::Persisted(ltime, _) => *ltime,
         }
     }
 
     fn status(&self) -> RouteStatus {
         match self {
             Swip::InMemory(_, status, _) => *status,
-            Swip::HistoricalOnDisk(_, status) => *status,
-            Swip::OnDisk(_, status) => *status,
+            Swip::PartiallyPersisted(_, status, _) => *status,
+            Swip::Persisted(_, status) => *status,
         }
     }
 
@@ -395,14 +407,14 @@ impl<M: Debug> Swip<RouteStatus, M> {
             Swip::InMemory(_, s, _) => {
                 *s = status;
             }
-            Swip::HistoricalOnDisk(_, s) => *s = status,
-            Swip::OnDisk(_, s) => *s = status,
+            Swip::PartiallyPersisted(_, s, _) => *s = status,
+            Swip::Persisted(_, s) => *s = status,
         };
     }
 
     fn unswizzle(&mut self) {
-        if let Swip::InMemory(l, s, _) = self {
-            *self = Self::HistoricalOnDisk(*l, *s);
+        if let Swip::InMemory(l, s, m) = self {
+            *self = Self::PartiallyPersisted(*l, *s, m.clone());
         }
     }
 }
@@ -579,23 +591,64 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     // record.multi_uniq_id. Returns the number of entries in the HashMap
     // after updating it, if it's more than 1. Returns None if this is the
     // first entry.
-    pub(crate) fn upsert_record(
+    pub(crate) fn upsert_record<
+        AF: AddressFamily,
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    >(
         &self,
+        prefix: PrefixId<AF>,
         record: PublicRecord<M>,
-    ) -> (Option<(MultiMapValue<M>, usize)>, usize) {
-        // let c_map = self.clone();
+        persistence: &PersistTree<AF, PREFIX_SIZE, KEY_SIZE>,
+        strategy: PersistStrategy,
+    ) -> (Option<usize>, usize) {
         let (mut record_map, retry_count) = self.guard_with_retry(0);
         let key = record.multi_uniq_id;
 
-        match record_map
-            .insert(record.multi_uniq_id, MultiMapValue::from(record))
-        {
-            Some(exist_rec) => {
-                record_map.get_mut(&key).unwrap().unswizzle();
-                println!("{:?}", record_map.get(&key));
-                (Some((exist_rec, record_map.len())), retry_count)
+        match (strategy, record_map.get_mut(&key)) {
+            (PersistStrategy::Historical, Some(exist_rec)) => {
+                persistence.persist_record(prefix, key, exist_rec);
+                *exist_rec =
+                    MultiMapValue::partially_persisted_record(record);
+
+                (Some(record_map.len()), retry_count)
+            }
+            (
+                PersistStrategy::Historical | PersistStrategy::NoPersist,
+                None,
+            ) => {
+                record_map
+                    .insert(key, MultiMapValue::in_memory_record(record));
+
+                (None, retry_count)
+            }
+            (
+                PersistStrategy::WriteAhead | PersistStrategy::NoPersist,
+                Some(_),
+            ) => {
+                panic!("Encountered illegally stored record");
+            }
+            (PersistStrategy::WriteAhead, None) => {
+                let mmv =
+                    &mut MultiMapValue::in_memory_record(record.clone());
+                persistence.persist_record(prefix, key, mmv);
+                record_map
+                    .insert(key, MultiMapValue::in_memory_record(record));
+
+                (None, retry_count)
+            }
+            (PersistStrategy::NoMemory, None) => {
+                persistence.persist_record(
+                    prefix,
+                    key,
+                    &mut MultiMapValue::persisted_record(record),
+                );
+
+                (None, retry_count)
+            }
+            (PersistStrategy::NoMemory, Some(_)) => {
+                panic!("Encountered illegally stored record");
             }
-            _ => (None, retry_count),
         }
     }
 }
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 193f426e..f2e72fe0 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -183,6 +183,7 @@
 // an actual memory leak in the mt-prefix-store (and even more if they can
 // produce a fix for it).
 
+use std::path::Path;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use log::{debug, info, log_enabled, trace};
@@ -203,7 +204,7 @@ use crate::{
 
 use crate::{
     impl_search_level, impl_search_level_for_mui, retrieve_node_mut_closure,
-    store_node_closure,
+    store_node_closure, Meta,
 };
 
 use super::atomic_types::*;
@@ -407,6 +408,24 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     pub fn parse_prefix(bytes: &[u8]) -> [u8; PREFIX_SIZE] {
         *bytes.first_chunk::<PREFIX_SIZE>().unwrap()
     }
+
+    #[cfg(feature = "persist")]
+    pub(crate) fn persist_record<M: Meta>(
+        &self,
+        prefix: PrefixId<AF>,
+        mui: u32,
+        record: &mut MultiMapValue<M>,
+    ) {
+        self.insert(
+            PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
+                prefix,
+                mui,
+                record.logical_time(),
+                record.status(),
+            ),
+            record.meta().unwrap().as_ref(),
+        );
+    }
 }
 
 impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
@@ -417,6 +436,29 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     }
 }
 
+#[derive(Clone, Debug)]
+pub enum PersistStrategy {
+    WriteAhead,
+    Historical,
+    NoPersist,
+    NoMemory,
+}
+
+#[derive(Debug, Clone)]
+pub struct StoreConfig {
+    persist_strategy: PersistStrategy,
+    persist_path: String,
+}
+
+impl Default for StoreConfig {
+    fn default() -> Self {
+        StoreConfig {
+            persist_strategy: PersistStrategy::Historical,
+            persist_path: "/tmp/rotonda/".into(),
+        }
+    }
+}
+
 // ----------- CustomAllocStorage -------------------------------------------
 //
 // CustomAllocStorage is a storage backend that uses a custom allocator, that
@@ -424,12 +466,13 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
 #[derive(Debug)]
 pub struct CustomAllocStorage<
     AF: AddressFamily,
-    M: crate::prefix_record::Meta,
+    M: Meta,
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
     const PREFIX_SIZE: usize,
     const KEY_SIZE: usize,
 > {
+    config: StoreConfig,
     pub(crate) buckets: NB,
     pub prefixes: PB,
     #[cfg(feature = "persist")]
@@ -454,28 +497,24 @@ impl<
 {
     pub(crate) fn init(
         root_node: SizedStrideNode<AF>,
-        // A node always gets created as an intermediary to create an actual
-        // meta-data record. A meta-data record has an id that is unique in
-        // the collection of Records, that is stored as a value in the tree.
-        // This unique id is used to be able to decide to replace or add a
-        // record to the meta-data collection in a multi-map. It is also added
-        // to a bitmap index on each node that has children where the unique
-        // id appears on a Record.
-        // multi_uniq_id: u32,
+        config: StoreConfig,
     ) -> Result<Self, Box<dyn std::error::Error>> {
         info!("store: initialize store {}", AF::BITS);
 
+        let persist_path = Path::new(&config.persist_path);
+        let persistence = PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>(
+            lsm_tree::Config::new(persist_path).open()?,
+            PhantomData,
+        );
+
         let store = CustomAllocStorage {
+            config,
             buckets: NodeBuckets::<AF>::init(),
             prefixes: PrefixBuckets::<AF, M>::init(),
-            persistence: PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>(
-                lsm_tree::Config::new("/tmp/rotonda/").open()?,
-                PhantomData,
-            ),
+            persistence,
             default_route_prefix_serial: AtomicUsize::new(0),
             withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
-            _af: PhantomData,
             _m: PhantomData,
         };
 
@@ -773,9 +812,8 @@ impl<
     ) -> Result<UpsertReport, PrefixStoreError> {
         let mut prefix_is_new = true;
         let mut mui_is_new = true;
-        let mut mui_count = 0;
 
-        let insert_retry_count =
+        let (mui_count, cas_count) =
             match self.non_recursive_retrieve_prefix_mut(prefix) {
                 // There's no StoredPrefix at this location yet. Create a new
                 // PrefixRecord and try to store it in the empty slot.
@@ -789,11 +827,16 @@ impl<
                         );
                     }
 
-                    let mui = record.multi_uniq_id;
-                    let res = locked_prefix.record_map.upsert_record(record);
+                    let (mui_count, retry_count) =
+                        locked_prefix.record_map.upsert_record(
+                            prefix,
+                            record,
+                            &self.persistence,
+                            PersistStrategy::Historical,
+                        );
 
                     // See if someone beat us to creating the record.
-                    if res.0.is_some() {
+                    if mui_count.is_some() {
                         mui_is_new = false;
                         prefix_is_new = false;
                     } else {
@@ -801,15 +844,7 @@ impl<
                         self.counters.inc_prefixes_count(prefix.get_len());
                     }
 
-                    let retry_count = res.1;
-
-                    #[cfg(feature = "persist")]
-                    if let Some((p_rec, min)) = res.0 {
-                        mui_count = min;
-                        self.persist_record(prefix, mui, p_rec);
-                    }
-
-                    retry_count
+                    (mui_count, retry_count)
                 }
                 // There already is a StoredPrefix with a record at this
                 // location.
@@ -817,12 +852,12 @@ impl<
                     if log_enabled!(log::Level::Debug) {
                         debug!(
                         "{} store: Found existing prefix record for {}/{}",
-                            std::thread::current()
-                                .name()
-                                .unwrap_or("unnamed-thread"),
-                            prefix.get_net(),
-                            prefix.get_len()
-                        );
+                        std::thread::current()
+                            .name()
+                            .unwrap_or("unnamed-thread"),
+                        prefix.get_net(),
+                        prefix.get_len()
+                    );
                     }
                     prefix_is_new = false;
 
@@ -830,31 +865,29 @@ impl<
                     // caller's record.
                     stored_prefix.set_ps_outdated(guard)?;
 
-                    let mui = record.multi_uniq_id;
-                    let res = stored_prefix.record_map.upsert_record(record);
-                    mui_is_new = res.0.is_none();
-                    let retry_count = res.1;
-
-                    #[cfg(feature = "persist")]
-                    if let Some((p_rec, min)) = res.0 {
-                        self.persist_record(prefix, mui, p_rec);
-                        mui_count = min;
-                    }
+                    let (mui_count, retry_count) =
+                        stored_prefix.record_map.upsert_record(
+                            prefix,
+                            record,
+                            &self.persistence,
+                            PersistStrategy::Historical,
+                        );
+                    mui_is_new = mui_count.is_none();
 
                     if let Some(tbi) = update_path_selections {
                         stored_prefix
                             .calculate_and_store_best_backup(&tbi, guard)?;
                     }
 
-                    retry_count
+                    (mui_count, retry_count)
                 }
             };
 
         Ok(UpsertReport {
             prefix_new: prefix_is_new,
-            cas_count: insert_retry_count,
+            cas_count,
             mui_new: mui_is_new,
-            mui_count,
+            mui_count: mui_count.unwrap_or(1),
         })
     }
 
@@ -1320,34 +1353,6 @@ impl<
             >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
             .dangerously_truncate_to_u32() as usize
     }
-
-    // persistance features
-
-    #[cfg(feature = "persist")]
-    fn persist_record(
-        &self,
-        prefix: PrefixId<AF>,
-        mui: u32,
-        record: MultiMapValue<M>,
-    ) {
-        self.persistence.insert(
-            PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
-                prefix,
-                mui,
-                record.logical_time(),
-                record.status(),
-            ),
-            record.meta().unwrap().as_ref(),
-        );
-    }
-
-    // #[cfg(feature = "persist")]
-    // fn get_prefix(
-    //     &self,
-    //     prefix: PrefixId<AF>,
-    // ) -> Vec<(([u8; PREFIX_SIZE], u32, u64), Vec<u8>)> {
-    //     self.persistence.prefix(prefix)
-    // }
 }
 
 //------------ Upsert -------------------------------------------------------
diff --git a/src/local_array/tree.rs b/src/local_array/tree.rs
index f457e5a8..a9b22e20 100644
--- a/src/local_array/tree.rs
+++ b/src/local_array/tree.rs
@@ -9,7 +9,7 @@ use std::sync::atomic::{
 use std::{fmt::Debug, marker::PhantomData};
 
 use crate::af::AddressFamily;
-use crate::custom_alloc::{CustomAllocStorage, UpsertReport};
+use crate::custom_alloc::{CustomAllocStorage, StoreConfig, UpsertReport};
 use crate::insert_match;
 use crate::local_array::store::atomic_types::{NodeBuckets, PrefixBuckets};
 
@@ -440,7 +440,9 @@ impl<
                     PB,
                     PREFIX_SIZE,
                     KEY_SIZE,
-                >::init(root_node)?,
+                >::init(
+                    root_node, StoreConfig::default()
+                )?,
             },
         )
     }
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index 752f8fee..b079dafc 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -225,7 +225,7 @@ impl<M: Meta> From<(Prefix, M)> for PublicPrefixSingleRecord<M> {
     }
 }
 
-//------------ PublicRecord -------------------------------------------
+//------------ PublicRecord --------------------------------------------------
 
 #[derive(Clone, Debug)]
 pub struct PublicRecord<M> {
@@ -261,18 +261,7 @@ impl<M: std::fmt::Display> std::fmt::Display for PublicRecord<M> {
     }
 }
 
-// impl<M: Clone> From<(u32, MultiMapValue<M>)> for PublicRecord<M> {
-//     fn from(value: (u32, MultiMapValue<M>)) -> Self {
-//         Self {
-//             multi_uniq_id: value.0,
-//             meta: value.1.meta,
-//             ltime: value.1.ltime,
-//             status: value.1.status,
-//         }
-//     }
-// }
-
-//------------ PublicPrefixRecord -------------------------------------------
+//------------ PublicPrefixRecord --------------------------------------------
 
 #[derive(Clone, Debug)]
 pub struct PublicPrefixRecord<M: Meta> {
@@ -511,7 +500,7 @@ impl<M: Meta> std::ops::Index<usize> for RecordSingleSet<M> {
     }
 }
 
-//------------ RecordSet ----------------------------------------------------
+//------------ RecordSet -----------------------------------------------------
 
 #[derive(Clone, Debug)]
 pub struct RecordSet<M: Meta> {
@@ -701,7 +690,7 @@ impl<'a, M: Meta> Iterator for RecordSetSingleIter<'a, M> {
     }
 }
 
-//------------ RecordSetIter -------------------------------------------
+//------------ RecordSetIter -------------------------------------------------
 
 #[derive(Clone, Debug)]
 pub struct RecordSetIter<'a, M: Meta> {

From 808384417f47b605cbd46da2c0d376f0c3f0d377 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 6 Dec 2024 22:20:57 +0100
Subject: [PATCH 011/147] remove outcommented code

---
 src/local_array/store/atomic_types.rs | 100 --------------------------
 src/local_array/store/custom_alloc.rs |   1 -
 src/prefix_record.rs                  |  50 -------------
 3 files changed, 151 deletions(-)

diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index c5163514..0a8b601a 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -659,106 +659,6 @@ impl<M: Meta> Clone for MultiMap<M> {
     }
 }
 
-// ----------- AtomicStoredPrefix -------------------------------------------
-// Unlike StoredNode, we don't need an Empty variant, since we're using
-// serial == 0 as the empty value. We're not using an Option here, to
-// avoid going outside our atomic procedure.
-// #[allow(clippy::type_complexity)]
-// #[derive(Debug)]
-// pub struct AtomicStoredPrefix<
-//     AF: AddressFamily,
-//     M: crate::prefix_record::Meta,
-// >(pub Atomic<StoredPrefix<AF, M>>);
-
-// impl<AF: AddressFamily, Meta: crate::prefix_record::Meta>
-//     AtomicStoredPrefix<AF, Meta>
-// {
-//     pub(crate) fn empty() -> Self {
-//         AtomicStoredPrefix(Atomic::null())
-//     }
-
-//     // pub(crate) fn is_empty(&self, guard: &Guard) -> bool {
-//     //     let pfx = self.0.load(Ordering::SeqCst, guard);
-//     //     pfx.is_null()
-//     // }
-
-//     pub(crate) fn get_stored_prefix<'a>(
-//         &'a self,
-//         guard: &'a Guard,
-//     ) -> Option<&'a StoredPrefix<AF, Meta>> {
-//         let pfx = self.0.load(Ordering::Acquire, guard);
-//         match pfx.is_null() {
-//             true => None,
-//             false => Some(unsafe { pfx.deref() }),
-//         }
-//     }
-
-//     pub(crate) fn _get_stored_prefix_with_tag<'a>(
-//         &'a self,
-//         guard: &'a Guard,
-//     ) -> Option<(&'a StoredPrefix<AF, Meta>, usize)> {
-//         let pfx = self.0.load(Ordering::Acquire, guard);
-//         match pfx.is_null() {
-//             true => None,
-//             false => Some((unsafe { pfx.deref() }, pfx.tag())),
-//         }
-//     }
-
-//     pub(crate) fn get_stored_prefix_mut<'a>(
-//         &'a self,
-//         guard: &'a Guard,
-//     ) -> Option<&'a StoredPrefix<AF, Meta>> {
-//         let pfx = self.0.load(Ordering::SeqCst, guard);
-
-//         match pfx.is_null() {
-//             true => None,
-//             false => Some(unsafe { pfx.deref() }),
-//         }
-//     }
-
-//     #[allow(dead_code)]
-//     pub(crate) fn get_serial(&self) -> usize {
-//         let guard = &epoch::pin();
-//         unsafe { self.0.load(Ordering::Acquire, guard).into_owned() }.tag()
-//     }
-
-//     pub(crate) fn get_prefix_id(&self) -> PrefixId<AF> {
-//         let guard = &epoch::pin();
-//         match self.get_stored_prefix(guard) {
-//             None => {
-//                 panic!("AtomicStoredPrefix::get_prefix_id: empty prefix");
-//             }
-//             Some(pfx) => pfx.prefix,
-//         }
-//     }
-
-//     // PrefixSet is an Atomic that might be a null pointer, which is
-//     // UB! Therefore we keep the prefix record in an Option: If
-//     // that Option is None, then the PrefixSet is a null pointer and
-//     // we'll return None
-//     pub(crate) fn get_next_bucket<'a>(
-//         &'a self,
-//         guard: &'a Guard,
-//     ) -> Option<&PrefixSet<AF, Meta>> {
-//         // let guard = &epoch::pin();
-//         if let Some(stored_prefix) = self.get_stored_prefix(guard) {
-//             // if stored_prefix.super_agg_record.is_some() {
-//             if !&stored_prefix
-//                 .next_bucket
-//                 .0
-//                 .load(Ordering::SeqCst, guard)
-//                 .is_null()
-//             {
-//                 Some(&stored_prefix.next_bucket)
-//             } else {
-//                 None
-//             }
-//         } else {
-//             None
-//         }
-//     }
-// }
-
 // ----------- FamilyBuckets Trait ------------------------------------------
 //
 // Implementations of this trait are done by a proc-macro called
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index f2e72fe0..016ad295 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -482,7 +482,6 @@ pub struct CustomAllocStorage<
     pub withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     pub counters: Counters,
     _m: PhantomData<M>,
-    _af: PhantomData<AF>,
 }
 
 impl<
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index b079dafc..8adea019 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -716,47 +716,6 @@ impl<'a, M: Meta> Iterator for RecordSetIter<'a, M> {
 
 //----------------------- meta-data traits/types-----------------------------
 
-/// Trait that describes how an existing record gets merged
-///
-/// MergeUpdate must be implemented by a type that implements Meta if it
-/// wants to be able to be stored. It should describe how the metadata for an
-/// existing record should be merged with newly arriving records for the same
-/// key.
-// pub trait MergeUpdate: Send + Sync {
-//     /// User-defined data to be passed in to the merge implementation.
-//     type UserDataIn: Debug + Sync + Send;
-
-//     /// User-defined data returned by the users implementation of the merge
-//     /// operations. Set to () if not needed.
-//     /// TODO: Define () as the default when the 'associated_type_defaults'
-//     /// Rust feature is stabilized. See:
-//     ///   https://github.com/rust-lang/rust/issues/29661
-//     type UserDataOut;
-
-//     fn merge_update(
-//         &mut self,
-//         update_meta: Self,
-//         user_data: Option<&Self::UserDataIn>,
-//     ) -> Result<Self::UserDataOut, Box<dyn std::error::Error>>;
-
-//     // This is part of the Read-Copy-Update pattern for updating a record
-//     // concurrently. The Read part should be done by the caller and then
-//     // the result should be passed in into this function together with
-//     // the new meta-data that updates it. This function will then create
-//     // a copy (in the pattern lingo, but in Rust that would be a Clone,
-//     // since we're not requiring Copy for Meta) and update that with a
-//     // copy of the new meta-data. It then returns the result of that merge.
-//     // The caller should then proceed to insert that as a new entry
-//     // in the global store.
-//     fn clone_merge_update(
-//         &self,
-//         update_meta: &Self,
-//         user_data: Option<&Self::UserDataIn>,
-//     ) -> Result<(Self, Self::UserDataOut), Box<dyn std::error::Error>>
-//     where
-//         Self: std::marker::Sized;
-// }
-
 /// Trait for types that can be used as metadata of a record
 pub trait Meta
 where
@@ -770,12 +729,3 @@ where
 
     fn as_orderable(&self, tbi: Self::TBI) -> Self::Orderable<'_>;
 }
-
-// impl Meta for inetnum::asn::Asn {
-//     type Orderable<'a> = inetnum::asn::Asn;
-//     type TBI = ();
-
-//     fn as_orderable(&self, _tbi: Self::TBI) -> inetnum::asn::Asn {
-//         *self
-//     }
-// }

From 9b9b9382dbf28e4d9a562c5a3d2e9e4563486452 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 11 Dec 2024 20:32:45 +0100
Subject: [PATCH 012/147] match_prefix for all strategies

---
 Cargo.toml                                 |   1 +
 examples/exact_matches.rs                  |  16 +-
 examples/full_table_multiple_trees_json.rs |   2 +-
 examples/more_specifics.rs                 |   2 +-
 examples/multi_no_thread.rs                |   2 +-
 examples/multi_single_thread.rs            |   2 +-
 examples/multi_thread_1.rs                 |  49 ++--
 examples/multi_thread_2.rs                 |   3 +-
 examples/multi_thread_3.rs                 |   3 +-
 examples/multi_thread_4.rs                 |  78 +++---
 examples/multi_thread_multi_prefix.rs      |   3 +-
 examples/multi_thread_single_prefix.rs     |   3 +-
 examples/numbers_treebitmap.rs             |   3 +-
 examples/single_thread_24.rs               |   2 +-
 examples/treebitmap.rs                     |  14 +-
 proc_macros/src/lib.rs                     | 115 ++++++---
 src/bin/cli.rs                             |   2 +-
 src/bin/load_mrt.rs                        |  78 ++++--
 src/lib.rs                                 |   4 +-
 src/local_array/query.rs                   |  68 +++--
 src/local_array/store/atomic_types.rs      | 278 ++++++++++-----------
 src/local_array/store/custom_alloc.rs      | 120 +++++----
 src/local_array/store/default_store.rs     |  12 +
 src/local_array/store/errors.rs            |   8 +
 src/local_array/tree.rs                    |  34 +--
 src/local_vec/tests/full_table_single.rs   |  68 +----
 src/meta_examples.rs                       |  12 +
 src/mod.rs                                 |   1 +
 src/prefix_record.rs                       |  10 +-
 src/prelude/mod.rs                         |   3 +-
 src/test_types.rs                          |  45 ++++
 tests/best-path.rs                         |   8 +-
 tests/concurrency.rs                       |  49 +---
 tests/full-table.rs                        |  29 ++-
 tests/more-more-specifics.rs               |   4 +-
 tests/more-specifics.rs                    |   2 +-
 tests/treebitmap.rs                        |   8 +-
 tests/treebitmap_v6.rs                     |  11 +-
 38 files changed, 637 insertions(+), 515 deletions(-)
 create mode 100644 src/mod.rs
 create mode 100644 src/test_types.rs

diff --git a/Cargo.toml b/Cargo.toml
index 41e8e1a8..2f2a22f1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -41,6 +41,7 @@ rayon = { version = "1.10", optional = true }
 memmap2 = { version = "0.9", optional = true }
 rand = { version = "^0.8", optional = true }
 lsm-tree = { version = "2.4.0", optional = true }
+uuid = { version = "1.11.0", features = ["v4", "fast-rng"] }
 
 [dev-dependencies]
 csv         = { version = "1" }
diff --git a/examples/exact_matches.rs b/examples/exact_matches.rs
index df65d45e..02e0dcdd 100644
--- a/examples/exact_matches.rs
+++ b/examples/exact_matches.rs
@@ -1,10 +1,10 @@
-use rotonda_store::prelude::*;
-use rotonda_store::prelude::multi::*;
 use rotonda_store::meta_examples::NoMeta;
+use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let guard = &epoch::pin();
-    let tree_bitmap = MultiThreadedStore::<NoMeta>::new()?;
+    let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
     let pfxs = vec![
         Prefix::new_relaxed(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
@@ -258,7 +258,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     for pfx in pfxs.into_iter() {
         println!("insert {}", pfx?);
         // let p : rotonda_store::Prefix<u32, PrefixAs> = pfx.into();
-        tree_bitmap.insert(&pfx.unwrap(), Record::new(0, 0, RouteStatus::Active, NoMeta::Empty), None)?;
+        tree_bitmap.insert(
+            &pfx.unwrap(),
+            Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
+            None,
+        )?;
     }
     println!("------ end of inserts\n");
     // println!(
@@ -343,9 +347,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_withdrawn: false,
                 include_less_specifics: false,
                 include_more_specifics: false,
-                mui: None
+                mui: None,
             },
-            guard
+            guard,
         );
         println!("exact match: {:?}", s_spfx);
         println!("-----------");
diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index bbcd3dbb..05d186cc 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -51,7 +51,7 @@ fn main() -> Result<(), Box<dyn Error>> {
         println!("[");
         for n in 1..6 {
             let mut rec_vec: Vec<PrefixRecord<PrefixAs>> = vec![];
-            let tree_bitmap = MyStore::<PrefixAs>::new()?;
+            let tree_bitmap = MyStore::<PrefixAs>::try_default()?;
 
             if let Err(err) = load_prefixes(&mut rec_vec) {
                 println!("error running example: {}", err);
diff --git a/examples/more_specifics.rs b/examples/more_specifics.rs
index 8190ca4f..12cef348 100644
--- a/examples/more_specifics.rs
+++ b/examples/more_specifics.rs
@@ -7,7 +7,7 @@ use rotonda_store::AddressFamily;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // type StoreType = InMemStorage<u32, PrefixAs>;
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
     let pfxs = vec![
         Prefix::new_relaxed(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
diff --git a/examples/multi_no_thread.rs b/examples/multi_no_thread.rs
index 0263de77..300adfe6 100644
--- a/examples/multi_no_thread.rs
+++ b/examples/multi_no_thread.rs
@@ -9,7 +9,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
     // let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_single_thread.rs b/examples/multi_single_thread.rs
index f49d4b02..c697ecdc 100644
--- a/examples/multi_single_thread.rs
+++ b/examples/multi_single_thread.rs
@@ -14,7 +14,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
     // let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_thread_1.rs b/examples/multi_thread_1.rs
index 515aaeb5..7b992f26 100644
--- a/examples/multi_thread_1.rs
+++ b/examples/multi_thread_1.rs
@@ -1,35 +1,42 @@
 use std::{sync::Arc, thread};
 
-use rotonda_store::prelude::*;
-use rotonda_store::prelude::multi::*;
 use rotonda_store::meta_examples::NoMeta;
+use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let tree_bitmap = Arc::new(MultiThreadedStore::<NoMeta>::new()?);
+    let tree_bitmap = Arc::new(MultiThreadedStore::<NoMeta>::try_default()?);
 
     let _: Vec<_> = (0..16)
         .map(|i: i32| {
             let tree_bitmap = tree_bitmap.clone();
 
-            thread::Builder::new().name(i.to_string()).spawn(move || {
-                let pfxs = get_pfx();
+            thread::Builder::new()
+                .name(i.to_string())
+                .spawn(move || {
+                    let pfxs = get_pfx();
 
-                for pfx in pfxs.into_iter() {
-                    println!("insert {}", pfx.unwrap());
+                    for pfx in pfxs.into_iter() {
+                        println!("insert {}", pfx.unwrap());
 
-                    match tree_bitmap
-                        .insert(
-                            &pfx.unwrap(), 
-                            Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
-                            None
-                    ) {
-                        Ok(_) => {}
-                        Err(e) => {
-                            println!("{}", e);
-                        }
-                    };
-                }
-            }).unwrap()
+                        match tree_bitmap.insert(
+                            &pfx.unwrap(),
+                            Record::new(
+                                0,
+                                0,
+                                RouteStatus::Active,
+                                NoMeta::Empty,
+                            ),
+                            None,
+                        ) {
+                            Ok(_) => {}
+                            Err(e) => {
+                                println!("{}", e);
+                            }
+                        };
+                    }
+                })
+                .unwrap()
         })
         .map(|t| t.join())
         .collect();
@@ -47,7 +54,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_withdrawn: false,
                 include_less_specifics: true,
                 include_more_specifics: true,
-                mui: None
+                mui: None,
             },
             guard,
         );
diff --git a/examples/multi_thread_2.rs b/examples/multi_thread_2.rs
index 666740ec..510cfc00 100644
--- a/examples/multi_thread_2.rs
+++ b/examples/multi_thread_2.rs
@@ -12,7 +12,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = Arc::new(MultiThreadedStore::<PrefixAs>::new()?);
+    let tree_bitmap =
+        Arc::new(MultiThreadedStore::<PrefixAs>::try_default()?);
     let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_thread_3.rs b/examples/multi_thread_3.rs
index 0c2165c4..13885b71 100644
--- a/examples/multi_thread_3.rs
+++ b/examples/multi_thread_3.rs
@@ -12,7 +12,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = Arc::new(MultiThreadedStore::<PrefixAs>::new()?);
+    let tree_bitmap =
+        Arc::new(MultiThreadedStore::<PrefixAs>::try_default()?);
     let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_thread_4.rs b/examples/multi_thread_4.rs
index 342019d5..ac04db46 100644
--- a/examples/multi_thread_4.rs
+++ b/examples/multi_thread_4.rs
@@ -1,3 +1,4 @@
+use inetnum::asn::Asn;
 use log::trace;
 use std::time::Duration;
 use std::{sync::Arc, thread};
@@ -6,63 +7,43 @@ use std::{sync::Arc, thread};
 use rotonda_store::prelude::*;
 use rotonda_store::prelude::multi::*;
 
-#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
-pub struct ComplexPrefixAs(pub Vec<u32>);
-
-// impl MergeUpdate for ComplexPrefixAs {
-//     type UserDataIn = ();
-//     type UserDataOut = ();
-
-//     fn merge_update(
-//         &mut self,
-//         update_record: ComplexPrefixAs,
-//         _: Option<&Self::UserDataIn>,
-//     ) -> Result<(), Box<dyn std::error::Error>> {
-//         self.0 = update_record.0;
-//         Ok(())
-//     }
-
-//     fn clone_merge_update(
-//         &self,
-//         update_meta: &Self,
-//         _: Option<&Self::UserDataIn>,
-//     ) -> Result<(Self, Self::UserDataOut), Box<dyn std::error::Error>>
-//     where
-//         Self: std::marker::Sized,
-//     {
-//         let mut new_meta = update_meta.0.clone();
-//         new_meta.push(self.0[0]);
-//         Ok((ComplexPrefixAs(new_meta), ()))
-//     }
-// }
-
-impl Meta for ComplexPrefixAs {
-    type Orderable<'a> = ComplexPrefixAs;
-    type TBI = ();
-    
-    fn as_orderable(&self, _tbi: Self::TBI) -> ComplexPrefixAs {
-        self.clone()
+
+    #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
+    pub struct BytesPrefixAs(pub [u8; 4]);
+
+    impl AsRef<[u8]> for BytesPrefixAs {
+        fn as_ref(&self) -> &[u8] {
+            self.0.as_ref()
+        }
     }
-}
 
-impl std::fmt::Display for ComplexPrefixAs {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "AS{:?}", self.0)
+    impl From<Vec<u8>> for BytesPrefixAs {
+        fn from(value: Vec<u8>) -> Self {
+            Self(*value.first_chunk::<4>().unwrap())
+        }
     }
-}
 
-impl AsRef<[u8]> for ComplexPrefixAs {
-    fn as_ref(&self) -> &[u8] {
-        todo!()
+    impl Meta for BytesPrefixAs {
+        type Orderable<'a> = Asn;
+        type TBI = ();
+
+        fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
+            u32::from_be_bytes(self.0).into()
+        }
+    }
+
+    impl std::fmt::Display for BytesPrefixAs {
+        fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+            write!(f, "AS{:?}", self.0)
+        }
     }
-}
 
-fn main() -> Result<(), Box<dyn std::error::Error>> {
+fn main() -> Result<(), Box<dyn std::error::Error>> {    
     #[cfg(feature = "cli")]
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = Arc::new(MultiThreadedStore::<ComplexPrefixAs>::new()?);
+    let tree_bitmap = Arc::new(MultiThreadedStore::<BytesPrefixAs>::try_default()?);
     let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
@@ -94,7 +75,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                 0,
                                 0,
                                 RouteStatus::Active,
-                                ComplexPrefixAs([i as u32].to_vec()),
+                                BytesPrefixAs((i as u32).to_be_bytes()),
+
                             ),
                             None
                         ) {
diff --git a/examples/multi_thread_multi_prefix.rs b/examples/multi_thread_multi_prefix.rs
index 251bfdac..08000fb0 100644
--- a/examples/multi_thread_multi_prefix.rs
+++ b/examples/multi_thread_multi_prefix.rs
@@ -17,7 +17,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = Arc::new(MultiThreadedStore::<PrefixAs>::new()?);
+    let tree_bitmap =
+        Arc::new(MultiThreadedStore::<PrefixAs>::try_default()?);
     // let pfx = Prefix::new_relaxed(
     //     0b1111_1111_1111_1111_1111_1111_1111_1111_u32.into_ipaddr(),
     //     32,
diff --git a/examples/multi_thread_single_prefix.rs b/examples/multi_thread_single_prefix.rs
index 07cc79ec..e9dcf5e3 100644
--- a/examples/multi_thread_single_prefix.rs
+++ b/examples/multi_thread_single_prefix.rs
@@ -16,7 +16,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = Arc::new(MultiThreadedStore::<PrefixAs>::new()?);
+    let tree_bitmap =
+        Arc::new(MultiThreadedStore::<PrefixAs>::try_default()?);
 
     let pfx = Prefix::new_relaxed(
         0b1111_1111_1111_1111_1111_1111_1111_1111_u32.into_ipaddr(),
diff --git a/examples/numbers_treebitmap.rs b/examples/numbers_treebitmap.rs
index 0d6a5859..32aeac28 100644
--- a/examples/numbers_treebitmap.rs
+++ b/examples/numbers_treebitmap.rs
@@ -63,7 +63,8 @@ fn main() -> Result<(), Box<dyn Error>> {
 
     for _strides in strides_vec.iter() {
         let mut pfxs: Vec<PrefixRecord<PrefixAs>> = vec![];
-        let tree_bitmap: MyStore<PrefixAs> = MyStore::<PrefixAs>::new()?;
+        let tree_bitmap: MyStore<PrefixAs> =
+            MyStore::<PrefixAs>::try_default()?;
 
         if let Err(err) = load_prefixes(&mut pfxs) {
             println!("error running example: {}", err);
diff --git a/examples/single_thread_24.rs b/examples/single_thread_24.rs
index 71c03466..80a67801 100644
--- a/examples/single_thread_24.rs
+++ b/examples/single_thread_24.rs
@@ -14,7 +14,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
     // let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let mut pfx_int = 0_u32;
diff --git a/examples/treebitmap.rs b/examples/treebitmap.rs
index 5d483ee9..f7389de6 100644
--- a/examples/treebitmap.rs
+++ b/examples/treebitmap.rs
@@ -1,12 +1,12 @@
 use inetnum::addr::Prefix;
-use rotonda_store::prelude::*;
-use rotonda_store::prelude::multi::*;
 use rotonda_store::meta_examples::NoMeta;
+use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
 type Prefix4<'a> = Prefix;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let tree_bitmap = MultiThreadedStore::new()?;
+    let tree_bitmap = MultiThreadedStore::try_default()?;
     let pfxs = vec![
         Prefix::new(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
@@ -185,9 +185,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     for pfx in pfxs.into_iter() {
         // println!("insert {:?}", pfx);
         tree_bitmap.insert(
-            &pfx.unwrap(), 
+            &pfx.unwrap(),
             Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
-            None
+            None,
         )?;
     }
     println!("------ end of inserts\n");
@@ -297,9 +297,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_withdrawn: false,
                 include_less_specifics: true,
                 include_more_specifics: false,
-                mui: None
+                mui: None,
             },
-            guard
+            guard,
         );
         println!("lmp: {:?}", s_spfx);
         println!("-----------");
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 2abdcf46..2e3daccf 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -498,14 +498,18 @@ pub fn create_store(
             v6: #strides6_name<M>,
         }
 
-        impl<
-                M: Meta
-            > Default for #store_name<M>
-        {
-            fn default() -> Self {
-                Self::new().expect("failed to create store")
-            }
-        }
+        // impl<
+        //         M: Meta
+        //     > Default for #store_name<M>
+        // {
+        //     fn default() -> Self {
+        //         let config = StoreConfig {
+        //             persist_strategy: PersistStrategy::PersistHistory,
+        //             persist_path: #persist_path.to_string()
+        //         };
+        //         Self::new_with_config(config).expect("failed to create store")
+        //     }
+        // }
 
         impl<
                 M: Meta
@@ -561,10 +565,22 @@ pub fn create_store(
             ///          })
             ///      }).map(|t| t.join()).collect();
             /// ```
-            pub fn new() -> Result<Self, Box<dyn std::error::Error>> {
+            pub fn new_with_config(
+                    mut config: StoreConfig
+                ) -> Result<Self, Box<dyn std::error::Error>> {
+
+                let uuid = Uuid::new_v4();
+                let mut config_v4 = config.clone();
+
+                config_v4.persist_path = format!(
+                    "{}/{}/ipv4/", config_v4.persist_path, uuid);
+
+                config.persist_path = format!(
+                    "{}/{}/ipv6/", config.persist_path, uuid);
+
                 Ok(Self {
-                    v4: #strides4_name::new()?,
-                    v6: #strides6_name::new()?,
+                    v4: #strides4_name::new(config_v4)?,
+                    v6: #strides6_name::new(config)?,
                 })
             }
         }
@@ -680,25 +696,37 @@ pub fn create_store(
 
                 match search_pfx.addr() {
                     std::net::IpAddr::V4(addr) => {
-                        self.v4.match_prefix_by_store_direct(
-                            PrefixId::<IPv4>::new(
+                        match self.v4.store.config.persist_strategy() {
+                            PersistStrategy::PersistOnly =>
+                                self.v4.match_prefix_in_persisted_store(
+                                    PrefixId::<IPv4>::new(
+                                        addr.into(),
+                                        search_pfx.len(),
+                                    ),
+                                    options.mui,
+                                ),
+                            _ => self.v4.match_prefix_by_store_direct(
+                                    PrefixId::<IPv4>::new(
+                                        addr.into(),
+                                        search_pfx.len(),
+                                    ),
+                                    options,
+                                    // options.mui,
+                                    guard
+                                )
+                        }
+                    },
+
+                    std::net::IpAddr::V6(addr) =>
+                        self.v6.match_prefix_by_store_direct(
+                            PrefixId::<IPv6>::new(
                                 addr.into(),
                                 search_pfx.len(),
                             ),
                             options,
-                            options.mui,
+                            // options.mui,
                             guard
-                        )
-                    },
-                    std::net::IpAddr::V6(addr) => self.v6.match_prefix_by_store_direct(
-                        PrefixId::<IPv6>::new(
-                            addr.into(),
-                            search_pfx.len(),
                         ),
-                        options,
-                        options.mui,
-                        guard
-                    ),
                 }
             }
 
@@ -1627,32 +1655,37 @@ pub fn create_store(
 
             // Disk Persistence
 
+            pub fn persist_strategy(&self) -> PersistStrategy {
+                self.v4.store.config.persist_strategy()
+            }
+
             pub fn get_values_for_prefix(&self, prefix: &Prefix) ->
                 Vec<(u32, u64, u8, Vec<u8>)> {
                 match prefix.is_v4() {
-                    true => self.v4.store.persistence
-                        .get_values_for_prefix(
+                    true => self.v4.store.persistence.as_ref().map_or(vec![],
+                        |p| p.get_values_for_prefix(
                             PrefixId::<IPv4>::from(*prefix)
-                        ),
-                    false => self.v6.store.persistence
-                        .get_values_for_prefix(
+                        )),
+                    false => self.v6.store.persistence.as_ref().map_or(vec![],
+                        |p| p.get_values_for_prefix(
                             PrefixId::<IPv6>::from(*prefix)
-                        )
+                        ))
                 }
             }
 
             /// Persist all the non-unique (prefix, mui, ltime) tuples
             /// with their values to disk
             pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
-                if let Some(segment) =
-                    self.v4.store.persistence.flush_to_disk()? {
-                    self.v4.store.persistence.register_segments(&[segment])?
+                if let Some(persistence) = self.v4.store.persistence.as_ref() {
+                    if let Some(segment) = persistence.flush_to_disk()? {
+                        persistence.register_segments(&[segment])?;
+                    }
                 }
 
-                if let Some(segment) =
-                    self.v6.store.persistence.flush_to_disk()?
-                {
-                    self.v6.store.persistence.register_segments(&[segment])?
+                if let Some(persistence) = self.v6.store.persistence.as_ref() {
+                    if let Some(segment) = persistence.flush_to_disk()? {
+                        persistence.register_segments(&[segment])?;
+                    }
                 }
 
                 Ok(())
@@ -1662,16 +1695,16 @@ pub fn create_store(
             /// to disk, for IPv4 and IPv6 respectively.
             pub fn approx_persisted_items(&self) -> (usize, usize) {
                 (
-                    self.v4.store.persistence.approximate_len(),
-                    self.v6.store.persistence.approximate_len()
+                    self.v4.store.persistence.as_ref().map_or(0, |p| p.approximate_len()),
+                    self.v6.store.persistence.as_ref().map_or(0, |p| p.approximate_len())
                 )
             }
 
             /// Return an estimation of the disk space currently used by the
             /// store in bytes.
             pub fn disk_space(&self) -> u64 {
-                self.v4.store.persistence.disk_space() +
-                self.v6.store.persistence.disk_space()
+                self.v4.store.persistence.as_ref().map_or(0, |p| p.disk_space()) +
+                self.v6.store.persistence.as_ref().map_or(0, |p| p.disk_space())
             }
         }
     };
diff --git a/src/bin/cli.rs b/src/bin/cli.rs
index edb71951..eac5661b 100644
--- a/src/bin/cli.rs
+++ b/src/bin/cli.rs
@@ -66,7 +66,7 @@ fn load_prefixes(
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut pfxs: Vec<PrefixRecord<PrefixAs>> = vec![];
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
 
     if let Err(err) = load_prefixes(&mut pfxs) {
         println!("error running example: {}", err);
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index 7905c016..381f4b62 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -10,6 +10,8 @@ use memmap2::Mmap;
 use rayon::iter::ParallelBridge;
 use rayon::iter::ParallelIterator;
 use rayon::prelude::*;
+use rotonda_store::custom_alloc::PersistStrategy;
+use rotonda_store::custom_alloc::PersistTree;
 use rotonda_store::custom_alloc::UpsertReport;
 use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::{MultiThreadedStore, RouteStatus};
@@ -36,6 +38,12 @@ impl AsRef<[u8]> for PaBytes {
     }
 }
 
+impl From<Vec<u8>> for PaBytes {
+    fn from(value: Vec<u8>) -> Self {
+        Self(value)
+    }
+}
+
 impl rotonda_store::Meta for PaBytes {
     type Orderable<'a> = u32;
 
@@ -50,6 +58,7 @@ impl rotonda_store::Meta for PaBytes {
 struct UpsertCounters {
     unique_prefixes: usize,
     unique_routes: usize,
+    persisted_routes: usize,
     total_routes: usize,
 }
 
@@ -68,6 +77,7 @@ impl std::ops::Add for UpsertCounters {
         Self {
             unique_prefixes: self.unique_prefixes + rhs.unique_prefixes,
             unique_routes: self.unique_routes + rhs.unique_routes,
+            persisted_routes: self.persisted_routes + rhs.persisted_routes,
             total_routes: self.total_routes + rhs.total_routes,
         }
     }
@@ -80,7 +90,7 @@ impl std::fmt::Display for UpsertCounters {
         writeln!(f, "total routes:\t\t\t{}", self.total_routes)?;
         writeln!(
             f,
-            "overwritten routes:\t\t{}",
+            "persisted routes:\t\t{}",
             self.total_routes - self.unique_routes
         )
     }
@@ -202,12 +212,14 @@ fn mt_parse_and_insert_table(
     store: Option<&MultiThreadedStore<PaBytes>>,
     ltime: u64,
 ) -> (UpsertCounters, Vec<Prefix>) {
+    let persist_strategy =
+        store.map_or(PersistStrategy::MemoryOnly, |p| p.persist_strategy());
     let counters = tables
         .par_bridge()
         .map(|(_fam, reh)| {
             let mut local_counters = UpsertCounters::default();
             let iter = routecore::mrt::SingleEntryIterator::new(reh);
-            let overwritten_prefixes = &mut vec![];
+            let persisted_prefixes = &mut vec![];
             // let mut cnt = 0;
             for (prefix, peer_idx, pa_bytes) in iter {
                 // cnt += 1;
@@ -226,20 +238,42 @@ fn mt_parse_and_insert_table(
                     )
                     .map(|r| match (r.prefix_new, r.mui_new) {
                         // new prefix, new mui
-                        (true, true) => UpsertCounters {
-                            unique_prefixes: 1,
-                            unique_routes: 1,
-                            total_routes: 1,
-                        },
+                        (true, true) => {
+                            match persist_strategy {
+                                PersistStrategy::WriteAhead
+                                | PersistStrategy::PersistOnly => {
+                                    persisted_prefixes.push(prefix);
+                                }
+                                _ => {}
+                            };
+                            UpsertCounters {
+                                unique_prefixes: 1,
+                                unique_routes: 1,
+                                total_routes: 1,
+                            }
+                        }
                         // old prefix, new mui
-                        (false, true) => UpsertCounters {
-                            unique_prefixes: 0,
-                            unique_routes: 1,
-                            total_routes: 1,
-                        },
+                        (false, true) => {
+                            match persist_strategy {
+                                PersistStrategy::WriteAhead
+                                | PersistStrategy::PersistOnly => {
+                                    persisted_prefixes.push(prefix);
+                                }
+                                _ => {}
+                            };
+
+                            UpsertCounters {
+                                unique_prefixes: 0,
+                                unique_routes: 1,
+                                total_routes: 1,
+                            }
+                        }
                         // old prefix, old mui
                         (false, false) => {
-                            overwritten_prefixes.push(prefix);
+                            if persist_strategy == PersistStrategy::MemoryOnly
+                            {
+                                persisted_prefixes.push(prefix);
+                            }
                             UpsertCounters {
                                 unique_prefixes: 0,
                                 unique_routes: 0,
@@ -256,7 +290,7 @@ fn mt_parse_and_insert_table(
                     local_counters += counters;
                 }
             }
-            (local_counters, overwritten_prefixes.clone())
+            (local_counters, persisted_prefixes.clone())
         })
         .fold(
             || (UpsertCounters::default(), vec![]),
@@ -372,7 +406,7 @@ fn main() {
     let mut global_counters = UpsertCounters::default();
     let mut mib_total: usize = 0;
     let mut inner_stores = vec![];
-    let mut overwritten_prefixes = BTreeSet::new();
+    let mut persisted_prefixes = BTreeSet::new();
 
     // Create all the stores necessary, and if at least one is created, create
     // a reference to the first one.
@@ -385,7 +419,8 @@ fn main() {
             return;
         }
         a if a.single_store => {
-            inner_stores.push(MultiThreadedStore::<PaBytes>::new().unwrap());
+            inner_stores
+                .push(MultiThreadedStore::<PaBytes>::try_default().unwrap());
             println!("created a single-store\n");
             Some(&inner_stores[0])
         }
@@ -395,8 +430,9 @@ fn main() {
         }
         _ => {
             for _ in &args.mrt_files {
-                inner_stores
-                    .push(MultiThreadedStore::<PaBytes>::new().unwrap());
+                inner_stores.push(
+                    MultiThreadedStore::<PaBytes>::try_default().unwrap(),
+                );
             }
             println!("Number of created stores: {}", inner_stores.len());
             Some(&inner_stores[0])
@@ -443,10 +479,10 @@ fn main() {
         global_counters += match &args {
             a if a.mt => {
                 let tables = mrt_file.tables().unwrap();
-                let (counters, ow_pfxs) =
+                let (counters, per_pfxs) =
                     mt_parse_and_insert_table(tables, store, f_index as u64);
                 if args.verify {
-                    overwritten_prefixes.extend(&ow_pfxs)
+                    persisted_prefixes.extend(&per_pfxs)
                 }
                 counters
             }
@@ -506,7 +542,7 @@ fn main() {
     if args.verify {
         println!("\nverifying disk persistence...");
         let mut max_len = 0;
-        for pfx in overwritten_prefixes {
+        for pfx in persisted_prefixes {
             let values = store.unwrap().get_values_for_prefix(&pfx);
             if values.is_empty() {
                 eprintln!("Found empty prefix on disk");
diff --git a/src/lib.rs b/src/lib.rs
index 4bfdc1c0..1ee6b348 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -18,6 +18,8 @@ mod prefix_record;
 mod stride;
 mod synth_int;
 
+pub mod test_types;
+
 #[macro_use]
 mod macros;
 
@@ -36,7 +38,7 @@ pub mod meta_examples;
 pub use crate::rotonda_store::*;
 
 // re-exports
-pub use inetnum::addr;
 pub use crossbeam_epoch::{self as epoch, Guard};
+pub use inetnum::addr;
 
 pub use prefix_record::{PublicRecord, RecordSet};
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index dd16a130..3a6bcd33 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -4,6 +4,7 @@ use crossbeam_epoch::{self as epoch};
 use epoch::Guard;
 
 use crate::af::AddressFamily;
+use crate::custom_alloc::PersistTree;
 use crate::local_array::store::atomic_types::{NodeBuckets, PrefixBuckets};
 use crate::prefix_record::{Meta, PublicRecord};
 use inetnum::addr::Prefix;
@@ -109,7 +110,7 @@ where
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> Result<
-        impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + '_,
+        impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a,
         std::io::Error,
     > {
         Ok(self.store.more_specific_prefix_iter_from(
@@ -124,27 +125,28 @@ where
         &'a self,
         search_pfx: PrefixId<AF>,
         options: &MatchOptions,
-        mui: Option<u32>,
+        // mui: Option<u32>,
         guard: &'a Guard,
     ) -> QueryResult<M> {
-        // `non_recursive_retrieve_prefix` returns an exact match
-        // only, so no longest matching prefix!
+        // `non_recursive_retrieve_prefix` returns an exact match only, so no
+        // longest matching prefix!
         let mut stored_prefix =
             self.store.non_recursive_retrieve_prefix(search_pfx).0.map(
                 |pfx| {
                     (
                         pfx.prefix,
                         if !options.include_withdrawn {
-                            // Filter out all the withdrawn records, both with
-                            // globally withdrawn muis, and with local statuses
+                            // Filter out all the withdrawn records, both
+                            // with globally withdrawn muis, and with local
+                            // statuses
                             // set to Withdrawn.
-                            self.get_filtered_records(pfx, mui, guard)
+                            self.get_filtered_records(pfx, options.mui, guard)
                                 .into_iter()
                                 .collect()
                         } else {
-                            // Do no filter out any records, but do rewrite the
-                            // local statuses of the records with muis that
-                            // appear in the specified bitmap index.
+                            // Do no filter out any records, but do rewrite
+                            // the local statuses of the records with muis
+                            // that appear in the specified bitmap index.
                             pfx.record_map.as_records_with_rewritten_status(
                                 unsafe {
                                     self.store
@@ -175,7 +177,7 @@ where
                     .store
                     .less_specific_prefix_iter(
                         search_pfx,
-                        mui,
+                        options.mui,
                         options.include_withdrawn,
                         guard,
                     )
@@ -207,7 +209,7 @@ where
                             } else {
                                 search_pfx
                             },
-                            mui,
+                            options.mui,
                             options.include_withdrawn,
                             guard,
                         )
@@ -225,15 +227,14 @@ where
                             } else {
                                 search_pfx
                             },
-                            mui,
+                            options.mui,
                             options.include_withdrawn,
                             guard,
                         )
-                        // .map(|p| (p.prefix_into_pub(), p))
                         .collect(),
                 )
-                // The user requested more specifics, but there aren't any, so we
-                // need to return an empty vec, not a None.
+                // The user requested more specifics, but there aren't any, so
+                // we need to return an empty vec, not a None.
             } else {
                 None
             },
@@ -241,6 +242,41 @@ where
         }
     }
 
+    pub fn match_prefix_in_persisted_store(
+        &'a self,
+        search_pfx: PrefixId<AF>,
+        mui: Option<u32>,
+    ) -> QueryResult<M> {
+        let key: Vec<u8> = if let Some(mui) = mui {
+            PersistTree::<AF,
+        PREFIX_SIZE, KEY_SIZE>::prefix_mui_persistence_key(search_pfx, mui)
+        } else {
+            search_pfx.as_bytes::<PREFIX_SIZE>().to_vec()
+        };
+
+        if let Some(persist) = &self.store.persistence {
+            QueryResult {
+                prefix: Some(search_pfx.into_pub()),
+                match_type: MatchType::ExactMatch,
+                prefix_meta: persist
+                    .get_records_for_key(&key)
+                    .into_iter()
+                    .map(|(_, rec)| rec)
+                    .collect::<Vec<_>>(),
+                less_specifics: None,
+                more_specifics: None,
+            }
+        } else {
+            QueryResult {
+                prefix: None,
+                match_type: MatchType::EmptyMatch,
+                prefix_meta: vec![],
+                less_specifics: None,
+                more_specifics: None,
+            }
+        }
+    }
+
     // In a LMP search we have to go over all the nibble lengths in the
     // stride up until the value of the actual nibble length were looking for
     // (until we reach stride length for all strides that aren't the last)
diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index 0a8b601a..eeb07131 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -284,48 +284,59 @@ impl From<RouteStatus> for u8 {
     }
 }
 
-#[derive(Clone, Debug)]
-pub(crate) struct MultiMapValue<M>(Swip<RouteStatus, M>);
+impl TryFrom<u8> for RouteStatus {
+    type Error = PrefixStoreError;
 
-impl<M: Meta> MultiMapValue<M> {
-    pub(crate) fn logical_time(&self) -> u64 {
-        self.0.logical_time()
+    fn try_from(value: u8) -> Result<Self, Self::Error> {
+        match value {
+            1 => Ok(RouteStatus::Active),
+            2 => Ok(RouteStatus::InActive),
+            3 => Ok(RouteStatus::Withdrawn),
+            _ => Err(PrefixStoreError::StoreNotReadyError),
+        }
     }
+}
 
-    pub(crate) fn meta(&self) -> Result<&M, PrefixStoreError> {
-        if let Swip::InMemory(_, _, meta) = &self.0 {
-            Ok(meta)
-        } else {
-            Err(PrefixStoreError::RecordNotInMemory)
-        }
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct PersistStatus(bool);
+
+impl PersistStatus {
+    pub(crate) fn persisted() -> Self {
+        Self(true)
     }
 
-    pub(crate) fn status(&self) -> RouteStatus {
-        self.0.status()
+    pub(crate) fn not_persisted() -> Self {
+        Self(false)
     }
+}
 
-    pub(crate) fn set_status(&mut self, status: RouteStatus) {
-        self.0.set_status(status);
+#[derive(Clone, Debug)]
+pub(crate) struct MultiMapValue<M> {
+    meta: M,
+    ltime: u64,
+    route_status: RouteStatus,
+    persist_status: PersistStatus,
+}
+
+impl<M: Meta> MultiMapValue<M> {
+    pub(crate) fn logical_time(&self) -> u64 {
+        self.ltime
     }
 
-    pub(crate) fn unswizzle(&mut self) {
-        self.0.unswizzle();
+    pub(crate) fn meta(&self) -> &M {
+        &self.meta
     }
 
-    pub(crate) fn in_memory_record(value: PublicRecord<M>) -> Self {
-        Self(Swip::InMemory(value.ltime, value.status, value.meta))
+    pub(crate) fn route_status(&self) -> RouteStatus {
+        self.route_status
     }
 
-    pub(crate) fn partially_persisted_record(value: PublicRecord<M>) -> Self {
-        Self(Swip::PartiallyPersisted(
-            value.ltime,
-            value.status,
-            value.meta,
-        ))
+    pub(crate) fn set_route_status(&mut self, status: RouteStatus) {
+        self.route_status = status;
     }
 
-    pub(crate) fn persisted_record(value: PublicRecord<M>) -> Self {
-        Self(Swip::Persisted(value.ltime, value.status))
+    pub(crate) fn has_persisted_history(&self) -> bool {
+        self.persist_status.0
     }
 }
 
@@ -335,86 +346,30 @@ impl<M: Meta> std::fmt::Display for MultiMapValue<M> {
             f,
             "{} {}",
             // self.meta(),
-            self.0.logical_time(),
-            self.0.status()
+            self.logical_time(),
+            self.route_status()
         )
     }
 }
 
-// impl<M: Meta> From<PublicRecord<M>> for MultiMapValue<M> {
-//     fn from(value: PublicRecord<M>) -> Self {
-//         Self(Swip::InMemory(value.ltime, value.status, value.meta))
-//     }
-// }
-
-impl<M: Meta> TryFrom<(u32, &MultiMapValue<M>)> for PublicRecord<M> {
-    type Error = PrefixStoreError;
-
-    fn try_from(
-        value: (u32, &MultiMapValue<M>),
-    ) -> Result<Self, Self::Error> {
-        match value.1 .0 {
-            Swip::InMemory(ltime, status, _) => Ok(Self {
-                multi_uniq_id: value.0,
-                meta: value.1.meta()?.clone(),
-                ltime,
-                status,
-            }),
-            Swip::PartiallyPersisted(ltime, status, _) => Ok(Self {
-                multi_uniq_id: value.0,
-                meta: value.1.meta()?.clone(),
-                ltime,
-                status,
-            }),
-            Swip::Persisted(_ltime, _status) => {
-                Err(PrefixStoreError::RecordNotInMemory)
-            }
+impl<M: Meta> From<(PublicRecord<M>, PersistStatus)> for MultiMapValue<M> {
+    fn from(value: (PublicRecord<M>, PersistStatus)) -> Self {
+        Self {
+            ltime: value.0.ltime,
+            route_status: value.0.status,
+            meta: value.0.meta,
+            persist_status: value.1,
         }
     }
 }
 
-#[derive(Clone, Copy, Debug)]
-pub(crate) enum Swip<T, M> {
-    // There's one ((prefix, mui), value) pair in the store and it only lives
-    // in memory
-    InMemory(u64, T, M),
-    // There are multiple ((prefix, mui), value) pairs, and the current one
-    // lives in memory, but there is/are older persisted ones
-    PartiallyPersisted(u64, T, M),
-    // There is at least one persisted ((prefix, mui), value) pair
-    Persisted(u64, T),
-}
-
-impl<M: Meta> Swip<RouteStatus, M> {
-    fn logical_time(&self) -> u64 {
-        match self {
-            Swip::InMemory(ltime, _, _) => *ltime,
-            Swip::PartiallyPersisted(ltime, _, _) => *ltime,
-            Swip::Persisted(ltime, _) => *ltime,
-        }
-    }
-
-    fn status(&self) -> RouteStatus {
-        match self {
-            Swip::InMemory(_, status, _) => *status,
-            Swip::PartiallyPersisted(_, status, _) => *status,
-            Swip::Persisted(_, status) => *status,
-        }
-    }
-
-    fn set_status(&mut self, status: RouteStatus) {
-        match self {
-            Swip::InMemory(_, s, _) => {
-                *s = status;
-            }
-            Swip::PartiallyPersisted(_, s, _) => *s = status,
-            Swip::Persisted(_, s) => *s = status,
-        };
-    }
-
-    fn unswizzle(&mut self) {
-        if let Swip::InMemory(l, s, m) = self {
-            *self = Self::PartiallyPersisted(*l, *s, m.clone());
+impl<M: Meta> From<(u32, &MultiMapValue<M>)> for PublicRecord<M> {
+    fn from(value: (u32, &MultiMapValue<M>)) -> Self {
+        Self {
+            multi_uniq_id: value.0,
+            meta: value.1.meta().clone(),
+            ltime: value.1.ltime,
+            status: value.1.route_status,
         }
     }
 }
@@ -463,8 +418,8 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let record_map = c_map.lock().unwrap();
 
         record_map.get(&mui).and_then(|r| {
-            if r.status() == RouteStatus::Active {
-                Some(PublicRecord::try_from((mui, r)).unwrap())
+            if r.route_status() == RouteStatus::Active {
+                Some(PublicRecord::from((mui, r)))
             } else {
                 None
             }
@@ -476,7 +431,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let record_map = c_map.lock().unwrap();
         let ord_routes = record_map
             .iter()
-            .map(|r| (r.1.meta().unwrap().as_orderable(tbi), *r.0));
+            .map(|r| (r.1.meta().as_orderable(tbi), *r.0));
         let (best, bckup) =
             routecore::bgp::path_selection::best_backup_generic(ord_routes);
         (best.map(|b| b.1), bckup.map(|b| b.1))
@@ -495,9 +450,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
             // untouched.
             let mut r = r.clone();
             if bmin.contains(mui) {
-                r.set_status(rewrite_status);
+                r.set_route_status(rewrite_status);
             }
-            PublicRecord::try_from((mui, &r)).unwrap()
+            PublicRecord::from((mui, &r))
         })
     }
 
@@ -531,9 +486,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
             .map(move |r| {
                 let mut rec = r.1.clone();
                 if bmin.contains(*r.0) {
-                    rec.set_status(rewrite_status);
+                    rec.set_route_status(rewrite_status);
                 }
-                PublicRecord::try_from((*r.0, &rec)).unwrap()
+                PublicRecord::from((*r.0, &rec))
             })
             .collect::<Vec<_>>()
     }
@@ -543,7 +498,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let record_map = c_map.lock().unwrap();
         record_map
             .iter()
-            .map(|r| PublicRecord::try_from((*r.0, r.1)).unwrap())
+            .map(|r| PublicRecord::from((*r.0, r.1)))
             .collect::<Vec<_>>()
     }
 
@@ -559,9 +514,10 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         record_map
             .iter()
             .filter_map(|r| {
-                if r.1.status() == RouteStatus::Active && !bmin.contains(*r.0)
+                if r.1.route_status() == RouteStatus::Active
+                    && !bmin.contains(*r.0)
                 {
-                    Some(PublicRecord::try_from((*r.0, r.1)).unwrap())
+                    Some(PublicRecord::from((*r.0, r.1)))
                 } else {
                     None
                 }
@@ -574,7 +530,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let c_map = Arc::clone(&self.0);
         let mut record_map = c_map.lock().unwrap();
         if let Some(rec) = record_map.get_mut(&mui) {
-            rec.set_status(RouteStatus::Withdrawn);
+            rec.set_route_status(RouteStatus::Withdrawn);
         }
     }
 
@@ -583,7 +539,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let record_map = Arc::clone(&self.0);
         let mut r_map = record_map.lock().unwrap();
         if let Some(rec) = r_map.get_mut(&mui) {
-            rec.set_status(RouteStatus::Active);
+            rec.set_route_status(RouteStatus::Active);
         }
     }
 
@@ -598,57 +554,89 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     >(
         &self,
         prefix: PrefixId<AF>,
-        record: PublicRecord<M>,
-        persistence: &PersistTree<AF, PREFIX_SIZE, KEY_SIZE>,
+        new_rec: PublicRecord<M>,
+        persistence: &Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
         strategy: PersistStrategy,
-    ) -> (Option<usize>, usize) {
+    ) -> Result<(Option<usize>, usize), PrefixStoreError> {
         let (mut record_map, retry_count) = self.guard_with_retry(0);
-        let key = record.multi_uniq_id;
+        let key = new_rec.multi_uniq_id;
 
         match (strategy, record_map.get_mut(&key)) {
-            (PersistStrategy::Historical, Some(exist_rec)) => {
-                persistence.persist_record(prefix, key, exist_rec);
-                *exist_rec =
-                    MultiMapValue::partially_persisted_record(record);
+            // New record for (prefix, mui) in memory.
 
-                (Some(record_map.len()), retry_count)
-            }
+            // We store the record in memory only.
             (
-                PersistStrategy::Historical | PersistStrategy::NoPersist,
+                PersistStrategy::PersistHistory | PersistStrategy::MemoryOnly,
                 None,
             ) => {
-                record_map
-                    .insert(key, MultiMapValue::in_memory_record(record));
+                record_map.insert(
+                    key,
+                    MultiMapValue::from((
+                        new_rec,
+                        PersistStatus::not_persisted(),
+                    )),
+                );
 
-                (None, retry_count)
+                Ok((None, retry_count))
             }
-            (
-                PersistStrategy::WriteAhead | PersistStrategy::NoPersist,
-                Some(_),
-            ) => {
-                panic!("Encountered illegally stored record");
+            // We only persist the record.
+            (PersistStrategy::PersistOnly, None) => {
+                if let Some(persistence) = persistence {
+                    persistence.persist_record(prefix, key, &new_rec);
+                    Ok((None, retry_count))
+                } else {
+                    Err(PrefixStoreError::PersistFailed)
+                }
             }
+            // We store both in memory and persist it.
             (PersistStrategy::WriteAhead, None) => {
-                let mmv =
-                    &mut MultiMapValue::in_memory_record(record.clone());
-                persistence.persist_record(prefix, key, mmv);
-                record_map
-                    .insert(key, MultiMapValue::in_memory_record(record));
-
-                (None, retry_count)
+                if let Some(persistence) = persistence {
+                    persistence.persist_record(prefix, key, &new_rec);
+                    let mmv = MultiMapValue::from((
+                        new_rec,
+                        PersistStatus::persisted(),
+                    ));
+                    record_map.insert(key, mmv);
+
+                    Ok((None, retry_count))
+                } else {
+                    Err(PrefixStoreError::PersistFailed)
+                }
             }
-            (PersistStrategy::NoMemory, None) => {
-                persistence.persist_record(
-                    prefix,
-                    key,
-                    &mut MultiMapValue::persisted_record(record),
-                );
 
-                (None, retry_count)
+            // Existing record for (prefix, mui) in memory.
+
+            // We store the record in memory only, and discard the old record.
+            (PersistStrategy::MemoryOnly, Some(exist_rec)) => {
+                *exist_rec = MultiMapValue::from((
+                    new_rec,
+                    PersistStatus::not_persisted(),
+                ));
+
+                Ok((Some(record_map.len()), retry_count))
             }
-            (PersistStrategy::NoMemory, Some(_)) => {
+            // We only persist record, so how come there's one in memory?
+            // Should not happen.
+            (PersistStrategy::PersistOnly, Some(_)) => {
                 panic!("Encountered illegally stored record");
             }
+            // We store the new record in memory and persist the old record.
+            (
+                PersistStrategy::PersistHistory | PersistStrategy::WriteAhead,
+                Some(exist_rec),
+            ) => {
+                if let Some(persistence) = persistence {
+                    persistence.persist_record(prefix, key, &new_rec);
+                    *exist_rec = MultiMapValue::from((
+                        new_rec,
+                        PersistStatus::persisted(),
+                    ));
+
+                    Ok((Some(record_map.len()), retry_count))
+                } else {
+                    Err(PrefixStoreError::PersistFailed)
+                }
+            }
         }
     }
 }
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 016ad295..91eb3028 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -330,6 +330,30 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             .collect::<Vec<_>>()
     }
 
+    pub fn get_records_for_key<M: Meta + From<Vec<u8>>>(
+        &self,
+        key: &[u8],
+    ) -> Vec<(inetnum::addr::Prefix, PublicRecord<M>)> {
+        (*self.0.prefix(key))
+            .into_iter()
+            .map(|kv| {
+                let kv = kv.unwrap();
+                let (pfx, mui, ltime, status) =
+                    Self::parse_key(kv.0.as_ref());
+
+                (
+                    PrefixId::<AF>::from(pfx).into_pub(),
+                    PublicRecord::new(
+                        mui,
+                        ltime,
+                        status.try_into().unwrap(),
+                        kv.1.as_ref().to_vec().into(),
+                    ),
+                )
+            })
+            .collect::<Vec<_>>()
+    }
+
     pub fn flush_to_disk(&self) -> Result<Option<Segment>, lsm_tree::Error> {
         self.0.flush_active_memtable(0)
     }
@@ -382,6 +406,23 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         *key
     }
 
+    #[cfg(feature = "persist")]
+    pub fn prefix_mui_persistence_key(
+        prefix_id: PrefixId<AF>,
+        mui: u32,
+    ) -> Vec<u8> {
+        let mut key = vec![];
+        // prefix 5 or 17 bytes
+        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
+
+        // mui 4 bytes
+        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
+            .first_chunk_mut::<4>()
+            .unwrap() = mui.to_le_bytes();
+
+        key
+    }
+
     #[cfg(feature = "persist")]
     pub fn parse_key(bytes: &[u8]) -> ([u8; PREFIX_SIZE], u32, u64, u8) {
         (
@@ -414,16 +455,16 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         prefix: PrefixId<AF>,
         mui: u32,
-        record: &mut MultiMapValue<M>,
+        record: &PublicRecord<M>,
     ) {
         self.insert(
             PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
                 prefix,
                 mui,
-                record.logical_time(),
-                record.status(),
+                record.ltime,
+                record.status,
             ),
-            record.meta().unwrap().as_ref(),
+            record.meta.as_ref(),
         );
     }
 }
@@ -436,26 +477,23 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum PersistStrategy {
     WriteAhead,
-    Historical,
-    NoPersist,
-    NoMemory,
+    PersistHistory,
+    MemoryOnly,
+    PersistOnly,
 }
 
 #[derive(Debug, Clone)]
 pub struct StoreConfig {
-    persist_strategy: PersistStrategy,
-    persist_path: String,
+    pub(crate) persist_strategy: PersistStrategy,
+    pub(crate) persist_path: String,
 }
 
-impl Default for StoreConfig {
-    fn default() -> Self {
-        StoreConfig {
-            persist_strategy: PersistStrategy::Historical,
-            persist_path: "/tmp/rotonda/".into(),
-        }
+impl StoreConfig {
+    pub(crate) fn persist_strategy(&self) -> PersistStrategy {
+        self.persist_strategy
     }
 }
 
@@ -472,11 +510,11 @@ pub struct CustomAllocStorage<
     const PREFIX_SIZE: usize,
     const KEY_SIZE: usize,
 > {
-    config: StoreConfig,
+    pub(crate) config: StoreConfig,
     pub(crate) buckets: NB,
     pub prefixes: PB,
     #[cfg(feature = "persist")]
-    pub persistence: PersistTree<AF, PREFIX_SIZE, KEY_SIZE>,
+    pub persistence: Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
     pub default_route_prefix_serial: AtomicUsize,
     // Global Roaring BitMap INdex that stores MUIs.
     pub withdrawn_muis_bmin: Atomic<RoaringBitmap>,
@@ -500,11 +538,16 @@ impl<
     ) -> Result<Self, Box<dyn std::error::Error>> {
         info!("store: initialize store {}", AF::BITS);
 
-        let persist_path = Path::new(&config.persist_path);
-        let persistence = PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>(
-            lsm_tree::Config::new(persist_path).open()?,
-            PhantomData,
-        );
+        let persistence = match config.persist_strategy {
+            PersistStrategy::MemoryOnly => None,
+            _ => {
+                let persist_path = Path::new(&config.persist_path);
+                Some(PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>(
+                    lsm_tree::Config::new(persist_path).open()?,
+                    PhantomData,
+                ))
+            }
+        };
 
         let store = CustomAllocStorage {
             config,
@@ -709,7 +752,7 @@ impl<
         &'a self,
         id: StrideNodeId<AF>,
         multi_uniq_id: u32,
-    ) -> Option<SizedStrideRef<AF>> {
+    ) -> Option<SizedStrideRef<'a, AF>> {
         struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
             f: &'s dyn for<'a> Fn(
                 &SearchLevel<AF, S>,
@@ -831,8 +874,8 @@ impl<
                             prefix,
                             record,
                             &self.persistence,
-                            PersistStrategy::Historical,
-                        );
+                            self.config.persist_strategy,
+                        )?;
 
                     // See if someone beat us to creating the record.
                     if mui_count.is_some() {
@@ -869,8 +912,8 @@ impl<
                             prefix,
                             record,
                             &self.persistence,
-                            PersistStrategy::Historical,
-                        );
+                            self.config.persist_strategy,
+                        )?;
                     mui_is_new = mui_count.is_none();
 
                     if let Some(tbi) = update_path_selections {
@@ -1090,12 +1133,14 @@ impl<
         }
     }
 
+    // This function is used by the match_prefix, and [more|less]_specifics
+    // public methods on the TreeBitMap (indirectly).
     #[allow(clippy::type_complexity)]
     pub fn non_recursive_retrieve_prefix(
         &'a self,
         id: PrefixId<AF>,
     ) -> (
-        Option<&StoredPrefix<AF, M>>,
+        Option<&'a StoredPrefix<AF, M>>,
         Option<(
             PrefixId<AF>,
             u8,
@@ -1144,7 +1189,7 @@ impl<
     pub(crate) fn retrieve_prefix(
         &'a self,
         prefix_id: PrefixId<AF>,
-    ) -> Option<(&StoredPrefix<AF, M>, usize)> {
+    ) -> Option<(&'a StoredPrefix<AF, M>, usize)> {
         struct SearchLevel<
             's,
             AF: AddressFamily,
@@ -1353,18 +1398,3 @@ impl<
             .dangerously_truncate_to_u32() as usize
     }
 }
-
-//------------ Upsert -------------------------------------------------------
-pub enum Upsert<T> {
-    Insert,
-    Update(T),
-}
-
-impl<T> std::fmt::Display for Upsert<T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Upsert::Insert => write!(f, "Insert"),
-            Upsert::Update(_) => write!(f, "Update"),
-        }
-    }
-}
diff --git a/src/local_array/store/default_store.rs b/src/local_array/store/default_store.rs
index 155e2b1d..05eac48f 100644
--- a/src/local_array/store/default_store.rs
+++ b/src/local_array/store/default_store.rs
@@ -10,6 +10,18 @@ use std::fmt;
 ))]
 struct DefaultStore;
 
+/// Try some
+impl<M: Meta> DefaultStore<M> {
+    pub fn try_default() -> Result<Self, PrefixStoreError> {
+        let config = StoreConfig {
+            persist_strategy: PersistStrategy::PersistOnly,
+            persist_path: "/tmp/rotonda/".to_string(),
+        };
+        Self::new_with_config(config)
+            .map_err(|_| PrefixStoreError::StoreNotReadyError)
+    }
+}
+
 impl<
         M: Meta,
         NB: NodeBuckets<IPv4>,
diff --git a/src/local_array/store/errors.rs b/src/local_array/store/errors.rs
index c08fe609..f38c9298 100644
--- a/src/local_array/store/errors.rs
+++ b/src/local_array/store/errors.rs
@@ -9,6 +9,7 @@ pub enum PrefixStoreError {
     PrefixNotFound,
     BestPathNotFound,
     RecordNotInMemory,
+    PersistFailed,
 }
 
 impl std::error::Error for PrefixStoreError {}
@@ -45,6 +46,13 @@ impl fmt::Display for PrefixStoreError {
                     memory."
                 )
             }
+            PrefixStoreError::PersistFailed => {
+                write!(
+                    f,
+                    "Error: The record for this (prefix, mui) cannot be \
+                    persisted."
+                )
+            }
         }
     }
 }
diff --git a/src/local_array/tree.rs b/src/local_array/tree.rs
index a9b22e20..c3dee862 100644
--- a/src/local_array/tree.rs
+++ b/src/local_array/tree.rs
@@ -395,7 +395,9 @@ impl<
         const KEY_SIZE: usize,
     > TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
-    pub fn new() -> Result<
+    pub fn new(
+        config: StoreConfig,
+    ) -> Result<
         TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
         Box<dyn std::error::Error>,
     > {
@@ -440,9 +442,7 @@ impl<
                     PB,
                     PREFIX_SIZE,
                     KEY_SIZE,
-                >::init(
-                    root_node, StoreConfig::default()
-                )?,
+                >::init(root_node, config)?,
             },
         )
     }
@@ -711,19 +711,19 @@ impl<
     }
 }
 
-impl<
-        AF: AddressFamily,
-        M: Meta,
-        NB: NodeBuckets<AF>,
-        PB: PrefixBuckets<AF, M>,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > Default for TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
-{
-    fn default() -> Self {
-        Self::new().unwrap()
-    }
-}
+// impl<
+//         AF: AddressFamily,
+//         M: Meta,
+//         NB: NodeBuckets<AF>,
+//         PB: PrefixBuckets<AF, M>,
+//         const PREFIX_SIZE: usize,
+//         const KEY_SIZE: usize,
+//     > Default for TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+// {
+//     fn default() -> Self {
+//         Self::new().unwrap()
+//     }
+// }
 
 // This implements the funky stats for a tree
 #[cfg(feature = "cli")]
diff --git a/src/local_vec/tests/full_table_single.rs b/src/local_vec/tests/full_table_single.rs
index c62003f3..59664192 100644
--- a/src/local_vec/tests/full_table_single.rs
+++ b/src/local_vec/tests/full_table_single.rs
@@ -2,71 +2,14 @@
 
 mod full_table {
     use inetnum::addr::Prefix;
-    use inetnum::asn::Asn;
 
+    use crate::test_types::BeBytesAsn;
     use crate::{prelude::*, PublicPrefixSingleRecord, SingleThreadedStore};
 
     use std::error::Error;
     use std::fs::File;
     use std::process;
 
-    #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
-    pub struct ComplexPrefixAs(pub Vec<u32>);
-
-    // impl MergeUpdate for ComplexPrefixAs {
-    //     type UserDataIn = ();
-    //     type UserDataOut = ();
-
-    //     fn merge_update(
-    //         &mut self,
-    //         update_record: ComplexPrefixAs,
-    //         _: Option<&Self::UserDataIn>,
-    //     ) -> Result<(), Box<dyn std::error::Error>> {
-    //         self.0 = update_record.0;
-    //         Ok(())
-    //     }
-
-    //     fn clone_merge_update(
-    //         &self,
-    //         update_meta: &Self,
-    //         _: Option<&Self::UserDataIn>,
-    //     ) -> Result<(Self, Self::UserDataOut), Box<dyn std::error::Error>>
-    //     where
-    //         Self: std::marker::Sized,
-    //     {
-    //         let mut new_meta = update_meta.0.clone();
-    //         new_meta.push(self.0[0]);
-    //         Ok((ComplexPrefixAs(new_meta), ()))
-    //     }
-    // }
-
-    impl AsRef<[u8]> for ComplexPrefixAs {
-        fn as_ref(&self) -> &[u8] {
-            todo!()
-        }
-    }
-
-    impl Meta for ComplexPrefixAs {
-        type Orderable<'a> = Asn;
-        type TBI = ();
-
-        fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
-            self.0[0].into()
-        }
-    }
-
-    // impl Orderable<u32, Rfc4271> for ComplexPrefixAs {
-    //     fn get_id(&self) -> &Self {
-    //         &self.0
-    //     }
-    // }
-
-    impl std::fmt::Display for ComplexPrefixAs {
-        fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-            write!(f, "AS{:?}", self.0)
-        }
-    }
-
     #[test]
     fn test_full_table_from_csv() -> Result<(), Box<dyn Error>> {
         // These constants are all contingent on the exact csv file,
@@ -78,7 +21,7 @@ mod full_table {
         const FOUND_PREFIXES: u32 = 1322993;
 
         fn load_prefixes(
-            pfxs: &mut Vec<PublicPrefixSingleRecord<ComplexPrefixAs>>,
+            pfxs: &mut Vec<PublicPrefixSingleRecord<BeBytesAsn>>,
         ) -> Result<(), Box<dyn Error>> {
             let file = File::open(CSV_FILE_PATH)?;
 
@@ -94,7 +37,7 @@ mod full_table {
                 let asn: u32 = record[2].parse().unwrap();
                 let pfx = PublicPrefixSingleRecord::new(
                     Prefix::new(net.into(), len)?,
-                    ComplexPrefixAs(vec![asn]),
+                    BeBytesAsn(asn.to_be_bytes()),
                 );
                 pfxs.push(pfx);
             }
@@ -108,11 +51,10 @@ mod full_table {
             // vec![3, 4, 4, 6, 7, 8],
         ];
         for _strides in strides_vec.iter().enumerate() {
-            let mut pfxs: Vec<PublicPrefixSingleRecord<ComplexPrefixAs>> =
-                vec![];
+            let mut pfxs: Vec<PublicPrefixSingleRecord<BeBytesAsn>> = vec![];
             let v4_strides = vec![8];
             let v6_strides = vec![8];
-            let mut tree_bitmap = SingleThreadedStore::<ComplexPrefixAs>::new(
+            let mut tree_bitmap = SingleThreadedStore::<BeBytesAsn>::new(
                 v4_strides, v6_strides,
             );
 
diff --git a/src/meta_examples.rs b/src/meta_examples.rs
index 321b4bcb..4cc482ba 100644
--- a/src/meta_examples.rs
+++ b/src/meta_examples.rs
@@ -60,6 +60,12 @@ impl AsRef<[u8]> for PrefixAs {
     }
 }
 
+impl From<Vec<u8>> for PrefixAs {
+    fn from(value: Vec<u8>) -> Self {
+        Self(*value.first_chunk::<4>().unwrap())
+    }
+}
+
 impl std::fmt::Display for PrefixAs {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "AS{}", u32::from_le_bytes(self.0))
@@ -108,6 +114,12 @@ impl AsRef<[u8]> for NoMeta {
     }
 }
 
+impl From<Vec<u8>> for NoMeta {
+    fn from(_value: Vec<u8>) -> Self {
+        Self::Empty
+    }
+}
+
 // impl MergeUpdate for NoMeta {
 //     type UserDataIn = ();
 //     type UserDataOut = ();
diff --git a/src/mod.rs b/src/mod.rs
new file mode 100644
index 00000000..0b47c732
--- /dev/null
+++ b/src/mod.rs
@@ -0,0 +1 @@
+pub mod test_types;
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index 8adea019..3731d7fd 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -719,8 +719,14 @@ impl<'a, M: Meta> Iterator for RecordSetIter<'a, M> {
 /// Trait for types that can be used as metadata of a record
 pub trait Meta
 where
-    Self:
-        fmt::Debug + fmt::Display + Clone + Sized + Send + Sync + AsRef<[u8]>,
+    Self: fmt::Debug
+        + fmt::Display
+        + Clone
+        + Sized
+        + Send
+        + Sync
+        + AsRef<[u8]>
+        + From<Vec<u8>>,
 {
     type Orderable<'a>: Ord
     where
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index f51300a1..5d5be38e 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -1,4 +1,5 @@
 pub use crate::{AddressFamily, IPv4, IPv6};
+pub use uuid::Uuid;
 
 pub use crate::prefix_record::{Meta, PublicPrefixRecord as PrefixRecord};
 pub use crate::stride::{Stride3, Stride4, Stride5};
@@ -24,7 +25,7 @@ pub mod multi {
 
     pub use crate::custom_alloc::CustomAllocStorage;
     pub use crate::custom_alloc::{
-        Counters, StoreStats, Upsert, UpsertReport,
+        Counters, PersistStrategy, StoreConfig, StoreStats, UpsertReport,
     };
 
     pub use routecore::bgp::path_selection::TiebreakerInfo;
diff --git a/src/test_types.rs b/src/test_types.rs
new file mode 100644
index 00000000..85d47c25
--- /dev/null
+++ b/src/test_types.rs
@@ -0,0 +1,45 @@
+use inetnum::asn::Asn;
+
+use crate::Meta;
+
+#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
+pub struct BeBytesAsn(pub [u8; 4]);
+
+impl AsRef<[u8]> for BeBytesAsn {
+    fn as_ref(&self) -> &[u8] {
+        self.0.as_ref()
+    }
+}
+
+impl From<Vec<u8>> for BeBytesAsn {
+    fn from(value: Vec<u8>) -> Self {
+        Self(*value.first_chunk::<4>().unwrap())
+    }
+}
+
+impl Meta for BeBytesAsn {
+    type Orderable<'a> = Asn;
+    type TBI = ();
+
+    fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
+        u32::from_be_bytes(self.0).into()
+    }
+}
+
+impl std::fmt::Display for BeBytesAsn {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "AS{:?}", self.0)
+    }
+}
+
+impl From<Asn> for BeBytesAsn {
+    fn from(value: Asn) -> Self {
+        Self(u32::from_be_bytes(value.to_raw()).to_le_bytes())
+    }
+}
+
+impl From<u32> for BeBytesAsn {
+    fn from(value: u32) -> Self {
+        Self(value.to_le_bytes())
+    }
+}
diff --git a/tests/best-path.rs b/tests/best-path.rs
index 9c47652a..02353cce 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -43,6 +43,12 @@ impl AsRef<[u8]> for Ipv4Route {
     }
 }
 
+impl From<Vec<u8>> for Ipv4Route {
+    fn from(value: Vec<u8>) -> Self {
+        todo!()
+    }
+}
+
 mod common {
     use std::io::Write;
 
@@ -61,7 +67,7 @@ fn test_best_path_1() -> Result<(), Box<dyn std::error::Error>> {
     let tree_bitmap =
         std::sync::Arc::new(std::sync::Arc::new(MultiThreadedStore::<
             Ipv4Route,
-        >::new()?));
+        >::try_default()?));
 
     let pfx = Prefix::from_str("185.34.0.0/16")?;
     let mut asns = [
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index 74a32b08..da7d821f 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -1,66 +1,25 @@
 use std::{str::FromStr, sync::atomic::Ordering};
 
-use common::SimpleAsn;
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
-    prelude::multi::{Record, RouteStatus},
-    MatchOptions, Meta, MultiThreadedStore,
+    prelude::multi::RouteStatus, test_types::BeBytesAsn, MatchOptions,
+    MultiThreadedStore, PublicRecord as Record,
 };
 
 mod common {
     use std::io::Write;
 
-    use inetnum::asn::Asn;
-    use rotonda_store::Meta;
-
     pub fn init() {
         let _ = env_logger::builder()
             .format(|buf, record| writeln!(buf, "{}", record.args()))
             .is_test(true)
             .try_init();
     }
-
-    #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone)]
-    pub struct SimpleAsn([u8; 4]);
-
-    impl std::fmt::Display for SimpleAsn {
-        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-            write!(f, "{}", u32::from_le_bytes(self.0))
-        }
-    }
-
-    impl Meta for SimpleAsn {
-        type Orderable<'a> = SimpleAsn;
-        type TBI = ();
-
-        fn as_orderable(&self, tbi: Self::TBI) -> Self::Orderable<'_> {
-            todo!()
-        }
-    }
-
-    impl From<Asn> for SimpleAsn {
-        fn from(value: Asn) -> Self {
-            Self(u32::from_be_bytes(value.to_raw()).to_le_bytes())
-        }
-    }
-
-    impl From<u32> for SimpleAsn {
-        fn from(value: u32) -> Self {
-            Self(value.to_le_bytes())
-        }
-    }
-
-    impl AsRef<[u8]> for SimpleAsn {
-        fn as_ref(&self) -> &[u8] {
-            &self.0
-        }
-    }
 }
 
 #[test]
 fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
-    use crate::common::SimpleAsn;
 
     let pfx_vec_1 = vec![
         Prefix::from_str("185.34.0.0/16")?,
@@ -90,7 +49,7 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
     }
 
     let tree_bitmap =
-        std::sync::Arc::new(MultiThreadedStore::<SimpleAsn>::new()?);
+        std::sync::Arc::new(MultiThreadedStore::<BeBytesAsn>::try_default()?);
 
     let mui_data_1 = MuiData {
         mui: 1,
@@ -416,7 +375,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
     ];
 
     let tree_bitmap =
-        std::sync::Arc::new(MultiThreadedStore::<common::SimpleAsn>::new()?);
+        std::sync::Arc::new(MultiThreadedStore::<BeBytesAsn>::try_default()?);
 
     const MUI_DATA: [u32; 4] = [65501, 65502, 65503, 65504];
 
diff --git a/tests/full-table.rs b/tests/full-table.rs
index 4b84f73f..50136d2c 100644
--- a/tests/full-table.rs
+++ b/tests/full-table.rs
@@ -10,43 +10,48 @@ mod tests {
     use std::process;
 
     #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
-    pub struct ComplexPrefixAs(Vec<u8>);
+    pub struct AsnList(Vec<u8>);
 
     // pub struct ComplexPrefixAs(pub Vec<u32>);
 
-    impl std::fmt::Display for ComplexPrefixAs {
+    impl std::fmt::Display for AsnList {
         fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
             write!(f, "AS{:?}", self.0)
         }
     }
 
-    impl Meta for ComplexPrefixAs {
+    impl Meta for AsnList {
         type Orderable<'a> = Asn;
         type TBI = ();
 
         fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
-            Asn::from(u32::from_le_bytes(*self.0.first_chunk::<4>().unwrap()))
+            Asn::from(u32::from_be_bytes(*self.0.first_chunk::<4>().unwrap()))
         }
     }
 
-    impl AsRef<[u8]> for ComplexPrefixAs {
+    impl AsRef<[u8]> for AsnList {
         fn as_ref(&self) -> &[u8] {
             &self.0
         }
     }
 
-    impl From<Vec<u32>> for ComplexPrefixAs {
+    impl From<Vec<u32>> for AsnList {
         fn from(value: Vec<u32>) -> Self {
-            ComplexPrefixAs(
+            AsnList(
                 value
                     .into_iter()
-                    .map(|v| v.to_le_bytes())
-                    .flatten()
+                    .flat_map(|v| v.to_le_bytes())
                     .collect::<Vec<u8>>(),
             )
         }
     }
 
+    impl From<Vec<u8>> for AsnList {
+        fn from(value: Vec<u8>) -> Self {
+            Self(value)
+        }
+    }
+
     #[test]
     fn test_full_table_from_csv() -> Result<(), Box<dyn Error>> {
         // These constants are all contingent on the exact csv file,
@@ -61,7 +66,7 @@ mod tests {
         let guard = &epoch::pin();
 
         fn load_prefixes(
-            pfxs: &mut Vec<PrefixRecord<ComplexPrefixAs>>,
+            pfxs: &mut Vec<PrefixRecord<AsnList>>,
         ) -> Result<(), Box<dyn Error>> {
             let file = File::open(CSV_FILE_PATH)?;
 
@@ -96,8 +101,8 @@ mod tests {
             // vec![3, 4, 4, 6, 7, 8],
         ];
         for _strides in strides_vec.iter().enumerate() {
-            let mut pfxs: Vec<PrefixRecord<ComplexPrefixAs>> = vec![];
-            let tree_bitmap = MultiThreadedStore::<ComplexPrefixAs>::new()?;
+            let mut pfxs: Vec<PrefixRecord<AsnList>> = vec![];
+            let tree_bitmap = MultiThreadedStore::<AsnList>::try_default()?;
             // .with_user_data("Testing".to_string());
 
             if let Err(err) = load_prefixes(&mut pfxs) {
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index 8c52128b..df9477db 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -10,7 +10,7 @@ mod tests {
     #[test]
     fn test_more_specifics_without_less_specifics(
     ) -> Result<(), Box<dyn Error>> {
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18)?, // 0
             Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24)?, // 1
@@ -90,7 +90,7 @@ mod tests {
     #[test]
     fn test_more_specifics_with_less_specifics() -> Result<(), Box<dyn Error>>
     {
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18), // 0
             Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24), // 1
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 59807bf6..b4c966fa 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -8,7 +8,7 @@ mod tests {
 
     #[test]
     fn test_more_specifics() -> Result<(), Box<dyn Error>> {
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 24), // 0
             //
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 3b51028a..1d300e37 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -22,7 +22,7 @@ mod tests {
 
     #[test]
     fn test_insert_extremes_ipv4() -> Result<(), Box<dyn std::error::Error>> {
-        let trie = &mut MultiThreadedStore::<NoMeta>::new()?;
+        let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
         let min_pfx = Prefix::new_relaxed(
             std::net::Ipv4Addr::new(0, 0, 0, 0).into(),
             1,
@@ -93,7 +93,7 @@ mod tests {
     fn test_tree_ipv4() -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             // Prefix::new_relaxed(0b0000_0000_0000_0000_0000_0000_0000_000 0_u32.into_ipaddr(), 0),
             Prefix::new_relaxed(
@@ -401,7 +401,7 @@ mod tests {
     #[test]
     fn test_ranges_ipv4() -> Result<(), Box<dyn std::error::Error>> {
         for i_net in 0..255 {
-            let tree_bitmap = MultiThreadedStore::<NoMeta>::new()?;
+            let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
 
             let pfx_vec: Vec<Prefix> = (1..32)
                 .collect::<Vec<u8>>()
@@ -459,7 +459,7 @@ mod tests {
     fn test_multi_ranges_ipv4() -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        let tree_bitmap = MultiThreadedStore::<NoMeta>::new()?;
+        let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
         for mui in [1_u32, 2, 3, 4, 5] {
             println!("Multi Uniq ID {mui}");
 
diff --git a/tests/treebitmap_v6.rs b/tests/treebitmap_v6.rs
index 6adcecb4..1cca898d 100644
--- a/tests/treebitmap_v6.rs
+++ b/tests/treebitmap_v6.rs
@@ -20,7 +20,7 @@ mod tests {
     #[test]
     fn test_arbitrary_insert_ipv6() -> Result<(), Box<dyn std::error::Error>>
     {
-        let trie = &mut MultiThreadedStore::<NoMeta>::new()?;
+        let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
         let guard = &epoch::pin();
         let a_pfx = Prefix::new_relaxed(
             ("2001:67c:1bfc::").parse::<std::net::Ipv6Addr>()?.into(),
@@ -51,6 +51,7 @@ mod tests {
         println!("prefix: {:?}", &expect_pfx);
         println!("result: {:#?}", &res);
         assert!(res.prefix.is_some());
+
         assert_eq!(res.prefix, Some(expect_pfx?));
 
         Ok(())
@@ -58,7 +59,7 @@ mod tests {
 
     #[test]
     fn test_insert_extremes_ipv6() -> Result<(), Box<dyn std::error::Error>> {
-        let trie = &mut MultiThreadedStore::<NoMeta>::new()?;
+        let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
         let min_pfx = Prefix::new_relaxed(
             std::net::Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0).into(),
             1,
@@ -135,7 +136,7 @@ mod tests {
     fn test_max_levels() -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             // 0-7
             Prefix::new_relaxed(
@@ -307,7 +308,7 @@ mod tests {
 
     #[test]
     fn test_tree_ipv6() -> Result<(), Box<dyn std::error::Error>> {
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::new()?;
+        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             // Prefix::new_relaxed(0b0000_0000_0000_0000_0000_0000_0000_000 0_u128.into_ipaddr(), 0),
             Prefix::new_relaxed(
@@ -625,7 +626,7 @@ mod tests {
     #[test]
     fn test_ranges_ipv4() -> Result<(), Box<dyn std::error::Error>> {
         for i_net in 0..255 {
-            let tree_bitmap = MultiThreadedStore::<NoMeta>::new()?;
+            let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
 
             let pfx_vec: Vec<Prefix> = (1..32)
                 .collect::<Vec<u8>>()

From 0b281a3eecf8f205f27d4a8ff2230ef3653609ba Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 11 Dec 2024 20:48:14 +0100
Subject: [PATCH 013/147] fix test_store counters

---
 src/bin/load_mrt.rs | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index 381f4b62..cfbf117a 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -11,7 +11,6 @@ use rayon::iter::ParallelBridge;
 use rayon::iter::ParallelIterator;
 use rayon::prelude::*;
 use rotonda_store::custom_alloc::PersistStrategy;
-use rotonda_store::custom_alloc::PersistTree;
 use rotonda_store::custom_alloc::UpsertReport;
 use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::{MultiThreadedStore, RouteStatus};
@@ -66,6 +65,7 @@ impl std::ops::AddAssign for UpsertCounters {
     fn add_assign(&mut self, rhs: Self) {
         self.unique_prefixes += rhs.unique_prefixes;
         self.unique_routes += rhs.unique_routes;
+        self.persisted_routes += rhs.persisted_routes;
         self.total_routes += rhs.total_routes;
     }
 }
@@ -87,10 +87,11 @@ impl std::fmt::Display for UpsertCounters {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         writeln!(f, "inserted unique prefixes:\t{}", self.unique_prefixes)?;
         writeln!(f, "inserted unique routes:\t\t{}", self.unique_routes)?;
+        writeln!(f, "persisted routes:\t\t{}", self.persisted_routes)?;
         writeln!(f, "total routes:\t\t\t{}", self.total_routes)?;
         writeln!(
             f,
-            "persisted routes:\t\t{}",
+            "calculated persisted routes:\t{}",
             self.total_routes - self.unique_routes
         )
     }
@@ -226,6 +227,7 @@ fn mt_parse_and_insert_table(
                 // let (prefix, peer_idx, pa_bytes) = e;
                 let mui = peer_idx.into();
                 let val = PaBytes(pa_bytes);
+                let mut persisted_routes = 0;
 
                 if let Some(store) = store {
                     let counters = insert(
@@ -243,12 +245,14 @@ fn mt_parse_and_insert_table(
                                 PersistStrategy::WriteAhead
                                 | PersistStrategy::PersistOnly => {
                                     persisted_prefixes.push(prefix);
+                                    persisted_routes = 1;
                                 }
                                 _ => {}
                             };
                             UpsertCounters {
                                 unique_prefixes: 1,
                                 unique_routes: 1,
+                                persisted_routes,
                                 total_routes: 1,
                             }
                         }
@@ -258,6 +262,7 @@ fn mt_parse_and_insert_table(
                                 PersistStrategy::WriteAhead
                                 | PersistStrategy::PersistOnly => {
                                     persisted_prefixes.push(prefix);
+                                    persisted_routes = 1;
                                 }
                                 _ => {}
                             };
@@ -265,18 +270,21 @@ fn mt_parse_and_insert_table(
                             UpsertCounters {
                                 unique_prefixes: 0,
                                 unique_routes: 1,
+                                persisted_routes,
                                 total_routes: 1,
                             }
                         }
                         // old prefix, old mui
                         (false, false) => {
-                            if persist_strategy == PersistStrategy::MemoryOnly
+                            if persist_strategy != PersistStrategy::MemoryOnly
                             {
                                 persisted_prefixes.push(prefix);
+                                persisted_routes = 1;
                             }
                             UpsertCounters {
                                 unique_prefixes: 0,
                                 unique_routes: 0,
+                                persisted_routes,
                                 total_routes: 1,
                             }
                         }

From 4e2d050c00e0bd1d134303ae12c4f25a6853416e Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 12 Dec 2024 12:31:13 +0100
Subject: [PATCH 014/147] cleanup

---
 proc_macros/src/lib.rs                | 46 +++++++++++++--------------
 src/local_array/store/atomic_types.rs |  1 -
 src/local_array/store/custom_alloc.rs |  2 ++
 3 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 2e3daccf..ee33f20e 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -588,14 +588,13 @@ pub fn create_store(
         impl<'a, M: Meta,
             > #store_name<M>
         {
-            /// Search for and return one or more prefixes that match the given
-            /// `search_pfx` argument.
-            ///
-            /// The search will return a [QueryResult] with the matching prefix,
-            /// if any, the type of match for the found prefix and the more and
-            /// less specifics for the requested prefix. The inclusion of more-
-            /// or less-specifics and the requested `match_type` is configurable
-            /// through the [MatchOptions] argument.
+            /// Search for and return one or more prefixes that match the
+            ///given  `search_pfx` argument.   The search will return a
+            ///[QueryResult] with the matching prefix,  if any, the type of
+            ///match for the found prefix and the more and  less specifics for
+            ///the requested prefix. The inclusion of more-  or less-specifics
+            ///and the requested `match_type` is configurable  through the
+            ///[MatchOptions] argument.
             ///
             /// The `match_type` in the `MatchOptions` indicates what match
             /// types can appear in the [QueryResult] result.
@@ -605,23 +604,24 @@ pub fn create_store(
             /// prefix, it will return an `EmptyMatch`.
             ///
             /// `LongestMatch` is less strict, and either an exactly matching
-            /// prefix or - in case there is no exact match - a longest matching
-            /// prefix will be allowed in the result. Failing both an EmptyMatch
-            /// will be returned.
+            /// prefix or - in case there is no exact match - a longest
+            /// matching prefix will be allowed in the result. Failing both an
+            /// EmptyMatch will be returned.
             ///
             /// For both `ExactMatch` and `LongestMatch` the
-            /// `include_less_specifics` and `include_more_specifics` options
-            /// will be respected and the result will contain the more and less
-            /// specifics according to the options for the requested prefix,
-            /// even if the result returns a `match_type` of `EmptyMatch`.
-            ///
-            /// `EmptyMatch` is the least strict, and will *always* return the
-            /// requested prefix, be it exactly matching, longest matching or not
-            /// matching at all (empty match), again, together with its less|more
-            /// specifics (if requested). Note that the last option, the empty
-            /// match in the result will never return less-specifics, but can
-            /// return more-specifics for a prefix that itself is not present
-            /// in the store.
+            /// `include_less_specifics` and `include_more_specifics`
+            /// options will be respected and the result will contain the
+            /// more and less specifics according to the options for the
+            /// requested prefix, even if the result returns a `match_type`
+            /// of `EmptyMatch`.
+            ///
+            /// `EmptyMatch` is the least strict, and will *always* return
+            /// the requested prefix, be it exactly matching, longest matching
+            /// or not matching at all (empty match), again, together with
+            /// its less|more specifics (if requested). Note that the last
+            /// option, the empty match in the result will never return
+            /// less-specifics, but can return more-specifics for a prefix
+            /// that itself is not present in the store.
             ///
             ///
             /// This table sums it up:
diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index eeb07131..46b7edc9 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -97,7 +97,6 @@ impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub struct PathSelections {
-    // serial: usize,
     pub(crate) path_selection_muis: (Option<u32>, Option<u32>),
 }
 
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 91eb3028..630c33ef 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -297,6 +297,8 @@ pub struct UpsertReport {
     pub mui_count: usize,
 }
 
+//------------ PersistTree ---------------------------------------------------
+
 pub struct PersistTree<
     AF: AddressFamily,
     // The size in bytes of the prefix in the peristed storage (disk), this

From 7f2e26da0029fef5fbda6b022d39095ec16870df Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 12 Dec 2024 12:49:28 +0100
Subject: [PATCH 015/147] config in outer store

---
 proc_macros/src/lib.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index ee33f20e..baa3198a 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -496,6 +496,7 @@ pub fn create_store(
         > {
             v4: #strides4_name<M>,
             v6: #strides6_name<M>,
+            config: StoreConfig
         }
 
         // impl<
@@ -571,16 +572,18 @@ pub fn create_store(
 
                 let uuid = Uuid::new_v4();
                 let mut config_v4 = config.clone();
+                let mut config_v6 = config.clone();
 
                 config_v4.persist_path = format!(
                     "{}/{}/ipv4/", config_v4.persist_path, uuid);
 
-                config.persist_path = format!(
+                config_v6.persist_path = format!(
                     "{}/{}/ipv6/", config.persist_path, uuid);
 
                 Ok(Self {
                     v4: #strides4_name::new(config_v4)?,
-                    v6: #strides6_name::new(config)?,
+                    v6: #strides6_name::new(config_v6)?,
+                    config
                 })
             }
         }
@@ -696,7 +699,7 @@ pub fn create_store(
 
                 match search_pfx.addr() {
                     std::net::IpAddr::V4(addr) => {
-                        match self.v4.store.config.persist_strategy() {
+                        match self.config.persist_strategy() {
                             PersistStrategy::PersistOnly =>
                                 self.v4.match_prefix_in_persisted_store(
                                     PrefixId::<IPv4>::new(

From d5afb1e88d90fa5fddecdf1c2349e7f0805c71d6 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 12 Dec 2024 12:49:37 +0100
Subject: [PATCH 016/147] clippy

---
 proc_macros/src/lib.rs             |  2 --
 src/local_array/store/iterators.rs |  4 ++--
 src/local_vec/store.rs             | 22 ++++++++++------------
 src/prefix_record.rs               |  2 +-
 4 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index baa3198a..f7903d98 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -714,7 +714,6 @@ pub fn create_store(
                                         search_pfx.len(),
                                     ),
                                     options,
-                                    // options.mui,
                                     guard
                                 )
                         }
@@ -727,7 +726,6 @@ pub fn create_store(
                                 search_pfx.len(),
                             ),
                             options,
-                            // options.mui,
                             guard
                         ),
                 }
diff --git a/src/local_array/store/iterators.rs b/src/local_array/store/iterators.rs
index 1fb5f6ec..a976418c 100644
--- a/src/local_array/store/iterators.rs
+++ b/src/local_array/store/iterators.rs
@@ -728,7 +728,7 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + '_ {
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
         trace!("more specifics for {:?}", start_prefix_id);
 
         // A v4 /32 or a v4 /128 doesn't have more specific prefixes 🤓.
@@ -848,7 +848,7 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + '_ {
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
         trace!("less specifics for {:?}", start_prefix_id);
         trace!("level {}, len {}", 0, start_prefix_id.get_len());
 
diff --git a/src/local_vec/store.rs b/src/local_vec/store.rs
index d4d9a35e..d9481d92 100644
--- a/src/local_vec/store.rs
+++ b/src/local_vec/store.rs
@@ -1,8 +1,8 @@
+use super::query::QuerySingleResult;
 use crate::local_vec::storage_backend::{InMemStorage, StorageBackend};
 use crate::local_vec::TreeBitMap;
 use crate::node_id::InMemNodeId;
 use crate::prefix_record::InternalPrefixRecord;
-use super::query::QuerySingleResult;
 use crate::{MatchOptions, Stats, Strides};
 
 use crate::af::{IPv4, IPv6};
@@ -29,9 +29,9 @@ impl<M: crate::prefix_record::Meta> Store<M> {
     }
 }
 
-impl<'a, M: crate::prefix_record::Meta> Store<M> {
+impl<M: crate::prefix_record::Meta> Store<M> {
     pub fn match_prefix(
-        &'a self,
+        &self,
         search_pfx: &Prefix,
         options: &MatchOptions,
     ) -> QuerySingleResult<M> {
@@ -71,28 +71,26 @@ impl<'a, M: crate::prefix_record::Meta> Store<M> {
         }
     }
 
-    pub fn prefixes_iter(&'a self) -> crate::PrefixSingleRecordIter<'a, M> {
+    pub fn prefixes_iter(&self) -> crate::PrefixSingleRecordIter<M> {
         let rs4: std::slice::Iter<InternalPrefixRecord<IPv4, M>> =
             self.v4.store.prefixes[..].iter();
         let rs6 = self.v6.store.prefixes[..].iter();
 
-        crate::PrefixSingleRecordIter::<'a, M> {
+        crate::PrefixSingleRecordIter::<M> {
             v4: Some(rs4),
             v6: rs6,
         }
     }
 
     pub fn nodes_v4_iter(
-        &'a self,
-    ) -> impl Iterator<Item = &'a SizedStrideNode<IPv4, InMemNodeId>> + 'a
-    {
+        &self,
+    ) -> impl Iterator<Item = &SizedStrideNode<IPv4, InMemNodeId>> + '_ {
         self.v4.store.nodes.iter()
     }
 
     pub fn nodes_v6_iter(
-        &'a self,
-    ) -> impl Iterator<Item = &'a SizedStrideNode<IPv6, InMemNodeId>> + 'a
-    {
+        &self,
+    ) -> impl Iterator<Item = &SizedStrideNode<IPv6, InMemNodeId>> {
         self.v6.store.nodes.iter()
     }
 
@@ -135,7 +133,7 @@ impl<'a, M: crate::prefix_record::Meta> Store<M> {
         }
     }
 
-    pub fn strides(&'a self) -> Strides {
+    pub fn strides(&self) -> Strides {
         Strides {
             v4: &self.v4.strides,
             v6: &self.v6.strides,
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index 3731d7fd..00f11947 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -2,7 +2,7 @@ use std::fmt;
 use std::fmt::Debug;
 use std::{cmp::Ordering, sync::Arc};
 
-use crate::local_array::store::atomic_types::{MultiMapValue, RouteStatus};
+use crate::local_array::store::atomic_types::RouteStatus;
 use crate::{af::AddressFamily, local_array::node::PrefixId};
 use inetnum::addr::Prefix;
 

From 22f04740048f98d7941894ea48d4671c71676e13 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 12 Dec 2024 17:01:09 +0100
Subject: [PATCH 017/147] use MemoryOnly in test

---
 tests/best-path.rs | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tests/best-path.rs b/tests/best-path.rs
index 02353cce..9b94341d 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -1,5 +1,7 @@
 use inetnum::addr::Prefix;
 use inetnum::asn::Asn;
+use rotonda_store::custom_alloc::PersistStrategy;
+use rotonda_store::custom_alloc::StoreConfig;
 use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::Record;
 use rotonda_store::prelude::multi::RouteStatus;
@@ -64,10 +66,17 @@ mod common {
 fn test_best_path_1() -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
+    let store_config = StoreConfig {
+        persist_strategy: PersistStrategy::MemoryOnly,
+        persist_path: "/tmp/rotonda/".into(),
+    };
+
     let tree_bitmap =
         std::sync::Arc::new(std::sync::Arc::new(MultiThreadedStore::<
             Ipv4Route,
-        >::try_default()?));
+        >::new_with_config(
+            store_config
+        )?));
 
     let pfx = Prefix::from_str("185.34.0.0/16")?;
     let mut asns = [

From d79f04f062bbb0965aefa6ff0aec4b6095c713c9 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 12 Dec 2024 17:02:09 +0100
Subject: [PATCH 018/147] cleanup

---
 src/local_array/store/custom_alloc.rs  |   4 +-
 src/local_array/store/default_store.rs |   2 +-
 src/local_array/tree.rs                | 128 +------------------------
 3 files changed, 5 insertions(+), 129 deletions(-)

diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 630c33ef..cccd8b0b 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -489,8 +489,8 @@ pub enum PersistStrategy {
 
 #[derive(Debug, Clone)]
 pub struct StoreConfig {
-    pub(crate) persist_strategy: PersistStrategy,
-    pub(crate) persist_path: String,
+    pub persist_strategy: PersistStrategy,
+    pub persist_path: String,
 }
 
 impl StoreConfig {
diff --git a/src/local_array/store/default_store.rs b/src/local_array/store/default_store.rs
index 05eac48f..03b4444a 100644
--- a/src/local_array/store/default_store.rs
+++ b/src/local_array/store/default_store.rs
@@ -14,7 +14,7 @@ struct DefaultStore;
 impl<M: Meta> DefaultStore<M> {
     pub fn try_default() -> Result<Self, PrefixStoreError> {
         let config = StoreConfig {
-            persist_strategy: PersistStrategy::PersistOnly,
+            persist_strategy: PersistStrategy::MemoryOnly,
             persist_path: "/tmp/rotonda/".to_string(),
         };
         Self::new_with_config(config)
diff --git a/src/local_array/tree.rs b/src/local_array/tree.rs
index c3dee862..12d2bda8 100644
--- a/src/local_array/tree.rs
+++ b/src/local_array/tree.rs
@@ -3,9 +3,7 @@ use crossbeam_epoch::{self as epoch};
 use log::{error, log_enabled, trace};
 
 use std::hash::Hash;
-use std::sync::atomic::{
-    AtomicU16, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering,
-};
+use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
 use std::{fmt::Debug, marker::PhantomData};
 
 use crate::af::AddressFamily;
@@ -234,116 +232,9 @@ impl<AF: AddressFamily> std::convert::From<StrideNodeId<AF>>
         PrefixId::new(addr_bits, len)
     }
 }
-#[derive(Debug)]
-pub struct AtomicStrideNodeId<AF: AddressFamily> {
-    stride_type: StrideType,
-    index: AtomicU32,
-    serial: AtomicUsize,
-    _af: PhantomData<AF>,
-}
-
-impl<AF: AddressFamily> AtomicStrideNodeId<AF> {
-    pub fn new(stride_type: StrideType, index: u32) -> Self {
-        Self {
-            stride_type,
-            index: AtomicU32::new(index),
-            serial: AtomicUsize::new(1),
-            _af: PhantomData,
-        }
-    }
-
-    pub fn empty() -> Self {
-        Self {
-            stride_type: StrideType::Stride4,
-            index: AtomicU32::new(0),
-            serial: AtomicUsize::new(0),
-            _af: PhantomData,
-        }
-    }
-
-    // get_serial() and update_serial() are intimately linked in the
-    // critical section of updating a node.
-    //
-    // The layout of the critical section is as follows:
-    // 1. get_serial() to retrieve the serial number of the node
-    // 2. do work in the critical section
-    // 3. store work result in the node
-    // 4. update_serial() to update the serial number of the node if
-    //    and only if the serial is the same as the one retrieved in step 1.
-    // 5. check the result of update_serial(). When successful, we're done,
-    //    otherwise, rollback the work result & repeat from step 1.
-    pub fn get_serial(&self) -> usize {
-        let serial = self.serial.load(Ordering::SeqCst);
-        std::sync::atomic::fence(Ordering::SeqCst);
-        serial
-    }
-
-    pub fn update_serial(
-        &self,
-        current_serial: usize,
-    ) -> Result<usize, usize> {
-        std::sync::atomic::fence(Ordering::Release);
-        self.serial.compare_exchange(
-            current_serial,
-            current_serial + 1,
-            Ordering::SeqCst,
-            Ordering::SeqCst,
-        )
-    }
-
-    // The idea is that we can only set the index once. An uninitialized
-    // index has a value of 0, so if we encounter a non-zero value that
-    // means somebody else already set it. We'll return an Err(index) with
-    // the index that was set.
-    pub fn set_id(&self, index: u32) -> Result<u32, u32> {
-        self.index.compare_exchange(
-            0,
-            index,
-            Ordering::SeqCst,
-            Ordering::SeqCst,
-        )
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.serial.load(Ordering::SeqCst) == 0
-    }
-
-    pub fn into_inner(self) -> (StrideType, Option<u32>) {
-        match self.serial.load(Ordering::SeqCst) {
-            0 => (self.stride_type, None),
-            _ => (self.stride_type, Some(self.index.load(Ordering::SeqCst))),
-        }
-    }
-
-    pub fn from_stridenodeid(
-        stride_type: StrideType,
-        id: StrideNodeId<AF>,
-    ) -> Self {
-        let index: AF = id.0.map_or(AF::zero(), |i| i.0);
-        Self {
-            stride_type,
-            index: AtomicU32::new(index.dangerously_truncate_to_u32()),
-            serial: AtomicUsize::new(usize::from(index != AF::zero())),
-            _af: PhantomData,
-        }
-    }
-}
-
-impl<AF: AddressFamily> std::convert::From<AtomicStrideNodeId<AF>> for usize {
-    fn from(id: AtomicStrideNodeId<AF>) -> Self {
-        id.index.load(Ordering::SeqCst) as usize
-    }
-}
 
 //------------------------- Node Collections --------------------------------
 
-// pub trait NodeCollection<AF: AddressFamily> {
-//     fn insert(&mut self, index: u16, insert_node: StrideNodeId<AF>);
-//     fn to_vec(&self) -> Vec<StrideNodeId<AF>>;
-//     fn as_slice(&self) -> &[AtomicStrideNodeId<AF>];
-//     fn empty() -> Self;
-// }
-
 #[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy, Clone)]
 pub enum StrideType {
     Stride3,
@@ -386,7 +277,6 @@ pub struct TreeBitMap<
 }
 
 impl<
-        'a,
         AF: AddressFamily,
         M: Meta,
         NB: NodeBuckets<AF>,
@@ -692,7 +582,7 @@ impl<
     // then adds all prefixes of these children recursively into a vec and
     // returns that.
     pub(crate) fn get_all_more_specifics_from_nibble<S: Stride>(
-        &'a self,
+        &self,
         current_node: &TreeBitMapNode<AF, S>,
         nibble: u32,
         nibble_len: u8,
@@ -711,20 +601,6 @@ impl<
     }
 }
 
-// impl<
-//         AF: AddressFamily,
-//         M: Meta,
-//         NB: NodeBuckets<AF>,
-//         PB: PrefixBuckets<AF, M>,
-//         const PREFIX_SIZE: usize,
-//         const KEY_SIZE: usize,
-//     > Default for TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
-// {
-//     fn default() -> Self {
-//         Self::new().unwrap()
-//     }
-// }
-
 // This implements the funky stats for a tree
 #[cfg(feature = "cli")]
 impl<

From 44839f3bcdb5f1c5bc7350a5cd2f20dd211ce7dc Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 13 Dec 2024 09:17:10 +0100
Subject: [PATCH 019/147] PublicRecord vec from persist store

---
 proc_macros/src/lib.rs                |  8 ++++----
 src/bin/load_mrt.rs                   | 24 +++++++++++++++---------
 src/local_array/store/custom_alloc.rs | 11 ++++++++---
 3 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index f7903d98..8f65981b 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -1660,15 +1660,15 @@ pub fn create_store(
                 self.v4.store.config.persist_strategy()
             }
 
-            pub fn get_values_for_prefix(&self, prefix: &Prefix) ->
-                Vec<(u32, u64, u8, Vec<u8>)> {
+            pub fn get_records_for_prefix(&self, prefix: &Prefix) ->
+                Vec<Record<M>> {
                 match prefix.is_v4() {
                     true => self.v4.store.persistence.as_ref().map_or(vec![],
-                        |p| p.get_values_for_prefix(
+                        |p| p.get_records_for_prefix(
                             PrefixId::<IPv4>::from(*prefix)
                         )),
                     false => self.v6.store.persistence.as_ref().map_or(vec![],
-                        |p| p.get_values_for_prefix(
+                        |p| p.get_records_for_prefix(
                             PrefixId::<IPv6>::from(*prefix)
                         ))
                 }
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index cfbf117a..b635bb86 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -11,6 +11,7 @@ use rayon::iter::ParallelBridge;
 use rayon::iter::ParallelIterator;
 use rayon::prelude::*;
 use rotonda_store::custom_alloc::PersistStrategy;
+use rotonda_store::custom_alloc::StoreConfig;
 use rotonda_store::custom_alloc::UpsertReport;
 use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::{MultiThreadedStore, RouteStatus};
@@ -22,7 +23,6 @@ use routecore::mrt::RibEntryIterator;
 use routecore::mrt::TableDumpIterator;
 
 #[derive(Clone, Debug)]
-//struct MyPaMap(PaMap);
 struct PaBytes(Vec<u8>);
 
 impl std::fmt::Display for PaBytes {
@@ -407,6 +407,11 @@ fn st_prime_store(
 }
 
 fn main() {
+    let store_config = StoreConfig {
+        persist_strategy: PersistStrategy::PersistOnly,
+        persist_path: "/tmp/rotonda/".into(),
+    };
+
     let args = Cli::parse();
 
     let t_total = Instant::now();
@@ -427,8 +432,10 @@ fn main() {
             return;
         }
         a if a.single_store => {
-            inner_stores
-                .push(MultiThreadedStore::<PaBytes>::try_default().unwrap());
+            inner_stores.push(
+                MultiThreadedStore::<PaBytes>::new_with_config(store_config)
+                    .unwrap(),
+            );
             println!("created a single-store\n");
             Some(&inner_stores[0])
         }
@@ -551,7 +558,7 @@ fn main() {
         println!("\nverifying disk persistence...");
         let mut max_len = 0;
         for pfx in persisted_prefixes {
-            let values = store.unwrap().get_values_for_prefix(&pfx);
+            let values = store.unwrap().get_records_for_prefix(&pfx);
             if values.is_empty() {
                 eprintln!("Found empty prefix on disk");
                 eprintln!("prefix: {}", pfx);
@@ -563,13 +570,12 @@ fn main() {
                     "len {}: {} -> {:?}",
                     max_len,
                     pfx,
-                    store.unwrap().get_values_for_prefix(&pfx)
+                    store.unwrap().get_records_for_prefix(&pfx)
                 );
             }
-            values
-                .iter()
-                .filter(|v| v.3.is_empty())
-                .for_each(|v| println!("withdraw for {}, mui {}", pfx, v.0))
+            values.iter().filter(|v| v.meta.0.is_empty()).for_each(|v| {
+                println!("withdraw for {}, mui {}", pfx, v.multi_uniq_id)
+            })
         }
     }
 }
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index cccd8b0b..7a455d02 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -317,17 +317,22 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         self.0.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
     }
 
-    pub fn get_values_for_prefix(
+    pub fn get_records_for_prefix<M: Meta>(
         &self,
         prefix: PrefixId<AF>,
-    ) -> Vec<(u32, u64, u8, Vec<u8>)> {
+    ) -> Vec<PublicRecord<M>> {
         let prefix_b = &prefix.as_bytes::<PREFIX_SIZE>();
         (*self.0.prefix(prefix_b))
             .into_iter()
             .map(|kv| {
                 let kv = kv.unwrap();
                 let (_, mui, ltime, status) = Self::parse_key(kv.0.as_ref());
-                (mui, ltime, status, kv.1.as_ref().to_vec())
+                PublicRecord::new(
+                    mui,
+                    ltime,
+                    status.try_into().unwrap(),
+                    kv.1.as_ref().to_vec().into(),
+                )
             })
             .collect::<Vec<_>>()
     }

From 986f3a4162df203191d305db4a74245eac370e64 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 17 Dec 2024 09:07:15 +0100
Subject: [PATCH 020/147] small refactor

---
 Cargo.toml                            |   3 +
 proc_macros/src/lib.rs                | 170 +++++++++-----------------
 src/bin/load_mrt.rs                   |  74 +++++++++--
 src/local_array/query.rs              |  71 ++++++++++-
 src/local_array/store/atomic_types.rs |   2 +-
 src/local_array/store/custom_alloc.rs |  23 ++--
 src/local_array/tree.rs               |   2 -
 src/local_vec/macros.rs               |   9 +-
 src/local_vec/query.rs                |  16 ++-
 src/local_vec/tree.rs                 |  25 ++--
 src/prefix_record.rs                  |  13 +-
 src/rotonda_store.rs                  |  57 +++++----
 12 files changed, 272 insertions(+), 193 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 2f2a22f1..1ad52638 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -42,6 +42,9 @@ memmap2 = { version = "0.9", optional = true }
 rand = { version = "^0.8", optional = true }
 lsm-tree = { version = "2.4.0", optional = true }
 uuid = { version = "1.11.0", features = ["v4", "fast-rng"] }
+serde = "1.0.216"
+serde_derive = "1.0.216"
+serde_json = "1.0.133"
 
 [dev-dependencies]
 csv         = { version = "1" }
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 8f65981b..1dae186b 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -343,7 +343,9 @@ pub fn stride_sizes(
 // ---------- Create Store struct -------------------------------------------
 
 // This macro creates the struct that will be the public API for the
-// PrefixStore. Therefore all methods defined in here should be public.
+// PrefixStore. It aspires to be a thin wrapper around the v4 and v6 stores,
+// so that users can use it AF agnostically. All methods defined in here
+// should be public.
 
 /// Creates a new, user-named struct with user-defined specified stride sizes
 /// that can used as a store type. The size of the prefix and the total key in
@@ -377,8 +379,7 @@ pub fn stride_sizes(
 /// bytes of the key of a record in the persisted storage. An IPv4 prefix is
 /// therefore 5 bytes (address part + prefix length), and 17 bytes for IPv6.
 /// The total number of bytes of the key is calculated thus: prefix (5 or 17)
-/// + multi_unique_id (4 bytes) + logical time of reception of the PDU into
-/// Rotonda (8 bytes).
+/// + multi_unique_id (4 bytes) + logical time of reception of the PDU into Rotonda (8 bytes).
 ///
 /// # Example
 /// ```ignore
@@ -499,19 +500,6 @@ pub fn create_store(
             config: StoreConfig
         }
 
-        // impl<
-        //         M: Meta
-        //     > Default for #store_name<M>
-        // {
-        //     fn default() -> Self {
-        //         let config = StoreConfig {
-        //             persist_strategy: PersistStrategy::PersistHistory,
-        //             persist_path: #persist_path.to_string()
-        //         };
-        //         Self::new_with_config(config).expect("failed to create store")
-        //     }
-        // }
-
         impl<
                 M: Meta
             > #store_name<M>
@@ -699,35 +687,25 @@ pub fn create_store(
 
                 match search_pfx.addr() {
                     std::net::IpAddr::V4(addr) => {
-                        match self.config.persist_strategy() {
-                            PersistStrategy::PersistOnly =>
-                                self.v4.match_prefix_in_persisted_store(
-                                    PrefixId::<IPv4>::new(
-                                        addr.into(),
-                                        search_pfx.len(),
-                                    ),
-                                    options.mui,
-                                ),
-                            _ => self.v4.match_prefix_by_store_direct(
-                                    PrefixId::<IPv4>::new(
-                                        addr.into(),
-                                        search_pfx.len(),
-                                    ),
-                                    options,
-                                    guard
-                                )
-                        }
+                        self.v4.match_prefix(
+                            PrefixId::<IPv4>::new(
+                                addr.into(),
+                                search_pfx.len(),
+                            ),
+                            options,
+                            guard
+                        )
                     },
-
-                    std::net::IpAddr::V6(addr) =>
-                        self.v6.match_prefix_by_store_direct(
+                    std::net::IpAddr::V6(addr) => {
+                        self.v6.match_prefix(
                             PrefixId::<IPv6>::new(
                                 addr.into(),
                                 search_pfx.len(),
                             ),
                             options,
                             guard
-                        ),
+                        )
+                    }
                 }
             }
 
@@ -744,42 +722,21 @@ pub fn create_store(
                 search_pfx: &Prefix,
                 guard: &Guard
             ) -> Option<Result<Record<M>, PrefixStoreError>> {
-
                 match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => self.v4.store
-                        .non_recursive_retrieve_prefix(
-                            PrefixId::<IPv4>::new(
-                                addr.into(),
-                                search_pfx.len(),
-                            ),
-                        )
-                        .0
-                        .map(|p_rec| unsafe { p_rec
-                            .get_path_selections(guard).best()
-                            .map_or_else(
-                                || Err(PrefixStoreError::BestPathNotFound),
-                                |mui| p_rec.record_map
-                                    .get_record_for_active_mui(mui)
-                                    .ok_or(PrefixStoreError::StoreNotReadyError)
-                            )
-                        }),
-                    std::net::IpAddr::V6(addr) => self.v6.store
-                        .non_recursive_retrieve_prefix(
-                            PrefixId::<IPv6>::new(
-                                addr.into(),
-                                search_pfx.len(),
-                            ),
-                        )
-                        .0
-                        .map(|p_rec| unsafe { p_rec
-                            .get_path_selections(guard).best()
-                            .map_or_else(
-                                || Err(PrefixStoreError::BestPathNotFound),
-                                |mui| p_rec.record_map
-                                    .get_record_for_active_mui(mui)
-                                    .ok_or(PrefixStoreError::StoreNotReadyError)
-                            )
-                        })
+                    std::net::IpAddr::V4(addr) => self.v4.best_path(
+                        PrefixId::<IPv4>::new(
+                            addr.into(),
+                            search_pfx.len()
+                        ),
+                        guard
+                    ),
+                    std::net::IpAddr::V6(addr) => self.v6.best_path(
+                        PrefixId::<IPv6>::new(
+                            addr.into(),
+                            search_pfx.len(),
+                        ),
+                        guard
+                    )
                 }
             }
 
@@ -804,60 +761,57 @@ pub fn create_store(
                 guard: &Guard
             ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
                 match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => self.v4.store
-                        .non_recursive_retrieve_prefix(
+                    std::net::IpAddr::V4(addr) => self.v4
+                        .calculate_and_store_best_and_backup_path(
                             PrefixId::<IPv4>::new(
                                 addr.into(),
                                 search_pfx.len(),
                             ),
-                            // guard
-                        ).0.map_or(
-                            Err(PrefixStoreError::StoreNotReadyError),
-                            |p_rec| p_rec.calculate_and_store_best_backup(
-                                tbi, guard),
+                            tbi,
+                            guard
                         ),
-                    std::net::IpAddr::V6(addr) => self.v6.store
-                        .non_recursive_retrieve_prefix(
+                    std::net::IpAddr::V6(addr) => self.v6
+                        .calculate_and_store_best_and_backup_path(
                             PrefixId::<IPv6>::new(
                                 addr.into(),
                                 search_pfx.len(),
                             ),
-                            // guard
-                        ).0.map_or(
-                            Err(PrefixStoreError::StoreNotReadyError),
-                            |p_rec| p_rec.calculate_and_store_best_backup(
-                                tbi, guard),
-                        ),
+                            tbi,
+                            guard
+                        )
                 }
             }
 
+            /// Return whether the best path selection, stored for this prefix
+            /// in the store is up to date with all the routes stored for
+            /// this prefix.
+            ///
+            /// Will return an error if the prefix does not exist in the store
+            ///
+            /// The `guard` should be a `&epoch::pin()`. It allows the
+            /// QuerySet to contain references to the meta-data objects,
+            /// instead of cloning them into it.
             pub fn is_ps_outdated(
                 &self,
                 search_pfx: &Prefix,
                 guard: &Guard
             ) -> Result<bool, PrefixStoreError> {
                 match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => self.v4.store
-                        .non_recursive_retrieve_prefix(
+                    std::net::IpAddr::V4(addr) => self.v4
+                        .is_ps_outdated(
                             PrefixId::<IPv4>::new(
                                 addr.into(),
                                 search_pfx.len(),
                             ),
-                            // guard
-                        ).0.map_or(
-                            Err(PrefixStoreError::StoreNotReadyError),
-                            |p| Ok(p.is_ps_outdated(guard))
+                            guard
                         ),
-                    std::net::IpAddr::V6(addr) => self.v6.store
-                        .non_recursive_retrieve_prefix(
+                    std::net::IpAddr::V6(addr) => self.v6
+                        .is_ps_outdated(
                             PrefixId::<IPv6>::new(
                                 addr.into(),
                                 search_pfx.len(),
                             ),
-                            // guard
-                        ).0.map_or(
-                            Err(PrefixStoreError::StoreNotReadyError),
-                            |p| Ok(p.is_ps_outdated(guard))
+                            guard
                         )
                 }
             }
@@ -1430,14 +1384,12 @@ pub fn create_store(
                         self.v4.store.mark_mui_as_active_for_prefix(
                             PrefixId::<IPv4>::from(*prefix),
                             mui,
-                            // &guard
                         )
                     }
                     std::net::IpAddr::V6(addr) => {
                         self.v6.store.mark_mui_as_active_for_prefix(
                             PrefixId::<IPv6>::from(*prefix),
                             mui,
-                            // &guard
                         )
                     }
                 }
@@ -1677,16 +1629,14 @@ pub fn create_store(
             /// Persist all the non-unique (prefix, mui, ltime) tuples
             /// with their values to disk
             pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
-                if let Some(persistence) = self.v4.store.persistence.as_ref() {
-                    if let Some(segment) = persistence.flush_to_disk()? {
-                        persistence.register_segments(&[segment])?;
-                    }
+                if let Some(persistence) =
+                    self.v4.store.persistence.as_ref() {
+                        persistence.flush_to_disk()?;
                 }
 
-                if let Some(persistence) = self.v6.store.persistence.as_ref() {
-                    if let Some(segment) = persistence.flush_to_disk()? {
-                        persistence.register_segments(&[segment])?;
-                    }
+                if let Some(persistence) =
+                    self.v6.store.persistence.as_ref() {
+                        persistence.flush_to_disk()?;
                 }
 
                 Ok(())
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index b635bb86..8e3e98b6 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -1,3 +1,5 @@
+use routecore::bgp::aspath::HopPath;
+use routecore::bgp::message::update_builder::StandardCommunitiesList;
 use std::collections::BTreeSet;
 use std::fmt;
 use std::fs::File;
@@ -16,6 +18,8 @@ use rotonda_store::custom_alloc::UpsertReport;
 use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::{MultiThreadedStore, RouteStatus};
 use rotonda_store::PublicRecord;
+use routecore::bgp::message::PduParseInfo;
+use routecore::bgp::path_attributes::OwnedPathAttributes;
 use routecore::mrt::MrtFile;
 
 use rand::seq::SliceRandom;
@@ -160,6 +164,10 @@ struct Cli {
     /// Verify the persisted entries
     #[arg(long, default_value_t = false)]
     verify: bool,
+
+    /// Persistence Strategy
+    #[arg(long)]
+    persist_strategy: Option<String>,
 }
 
 fn insert<T: rotonda_store::Meta>(
@@ -407,8 +415,8 @@ fn st_prime_store(
 }
 
 fn main() {
-    let store_config = StoreConfig {
-        persist_strategy: PersistStrategy::PersistOnly,
+    let mut store_config = StoreConfig {
+        persist_strategy: PersistStrategy::PersistHistory,
         persist_path: "/tmp/rotonda/".into(),
     };
 
@@ -421,6 +429,26 @@ fn main() {
     let mut inner_stores = vec![];
     let mut persisted_prefixes = BTreeSet::new();
 
+    store_config.persist_strategy = match &args.persist_strategy {
+        Some(a) if a == &"memory_only".to_string() => {
+            PersistStrategy::MemoryOnly
+        }
+        Some(a) if a == &"write_ahead".to_string() => {
+            PersistStrategy::WriteAhead
+        }
+        Some(a) if a == &"persist_only".to_string() => {
+            PersistStrategy::PersistOnly
+        }
+        Some(a) if a == &"persist_history".to_string() => {
+            PersistStrategy::PersistHistory
+        }
+        None => PersistStrategy::PersistHistory,
+        Some(a) => {
+            eprintln!("Unknown persist strategy: {}", a);
+            return;
+        }
+    };
+
     // Create all the stores necessary, and if at least one is created, create
     // a reference to the first one.
     let mut store = match &args {
@@ -433,10 +461,15 @@ fn main() {
         }
         a if a.single_store => {
             inner_stores.push(
-                MultiThreadedStore::<PaBytes>::new_with_config(store_config)
-                    .unwrap(),
+                MultiThreadedStore::<PaBytes>::new_with_config(
+                    store_config.clone(),
+                )
+                .unwrap(),
+            );
+            println!(
+                "created a single-store with strategy: {:?}\n",
+                store_config
             );
-            println!("created a single-store\n");
             Some(&inner_stores[0])
         }
         a if a.parse_only => {
@@ -450,6 +483,7 @@ fn main() {
                 );
             }
             println!("Number of created stores: {}", inner_stores.len());
+            println!("store config: {:?}", store_config);
             Some(&inner_stores[0])
         }
     };
@@ -566,12 +600,30 @@ fn main() {
             }
             if values.len() > max_len {
                 max_len = values.len();
-                println!(
-                    "len {}: {} -> {:?}",
-                    max_len,
-                    pfx,
-                    store.unwrap().get_records_for_prefix(&pfx)
-                );
+                let recs = store.unwrap().get_records_for_prefix(&pfx);
+                println!("LEN {} prefix: {}", max_len, pfx);
+                for rec in recs {
+                    let pa = OwnedPathAttributes::from((
+                        PduParseInfo::modern(),
+                        rec.meta.0.to_vec(),
+                    ));
+                    print!(
+                        "({})\tp[{}]",
+                        rec.multi_uniq_id,
+                        &pa.get::<HopPath>().unwrap()
+                    );
+                    if let Some(comms) = &pa.get::<StandardCommunitiesList>()
+                    {
+                        print!(" c[");
+                        comms
+                            .communities()
+                            .iter()
+                            .for_each(|c| print!("{c} "));
+                        println!("]");
+                    } else {
+                        println!(" no_c");
+                    };
+                }
             }
             values.iter().filter(|v| v.meta.0.is_empty()).for_each(|v| {
                 println!("withdraw for {}, mui {}", pfx, v.multi_uniq_id)
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 3a6bcd33..c25885fd 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -4,7 +4,7 @@ use crossbeam_epoch::{self as epoch};
 use epoch::Guard;
 
 use crate::af::AddressFamily;
-use crate::custom_alloc::PersistTree;
+use crate::custom_alloc::{PersistStrategy, PersistTree};
 use crate::local_array::store::atomic_types::{NodeBuckets, PrefixBuckets};
 use crate::prefix_record::{Meta, PublicRecord};
 use inetnum::addr::Prefix;
@@ -17,6 +17,7 @@ use crate::{MatchOptions, MatchType};
 
 use super::node::{PrefixId, SizedStrideRef, StrideNodeId};
 use super::store::atomic_types::{RouteStatus, StoredPrefix};
+use super::store::errors::PrefixStoreError;
 
 //------------ Prefix Matching ----------------------------------------------
 
@@ -121,11 +122,24 @@ where
         ))
     }
 
-    pub fn match_prefix_by_store_direct(
+    pub fn match_prefix(
+        &'a self,
+        search_pfx: PrefixId<AF>,
+        options: &MatchOptions,
+        guard: &'a Guard,
+    ) -> QueryResult<M> {
+        match self.store.config.persist_strategy() {
+            PersistStrategy::PersistOnly => {
+                self.match_prefix_in_persisted_store(search_pfx, options.mui)
+            }
+            _ => self.match_prefix_in_memory(search_pfx, options, guard),
+        }
+    }
+
+    fn match_prefix_in_memory(
         &'a self,
         search_pfx: PrefixId<AF>,
         options: &MatchOptions,
-        // mui: Option<u32>,
         guard: &'a Guard,
     ) -> QueryResult<M> {
         // `non_recursive_retrieve_prefix` returns an exact match only, so no
@@ -242,7 +256,7 @@ where
         }
     }
 
-    pub fn match_prefix_in_persisted_store(
+    fn match_prefix_in_persisted_store(
         &'a self,
         search_pfx: PrefixId<AF>,
         mui: Option<u32>,
@@ -277,6 +291,54 @@ where
         }
     }
 
+    pub fn best_path(
+        &'a self,
+        search_pfx: PrefixId<AF>,
+        guard: &Guard,
+    ) -> Option<Result<PublicRecord<M>, PrefixStoreError>> {
+        self.store
+            .non_recursive_retrieve_prefix(search_pfx)
+            .0
+            .map(|p_rec| {
+                p_rec.get_path_selections(guard).best().map_or_else(
+                    || Err(PrefixStoreError::BestPathNotFound),
+                    |mui| {
+                        p_rec
+                            .record_map
+                            .get_record_for_active_mui(mui)
+                            .ok_or(PrefixStoreError::StoreNotReadyError)
+                    },
+                )
+            })
+    }
+
+    pub fn calculate_and_store_best_and_backup_path(
+        &self,
+        search_pfx: PrefixId<AF>,
+        tbi: &<M as Meta>::TBI,
+        guard: &Guard,
+    ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
+        self.store
+            .non_recursive_retrieve_prefix(search_pfx)
+            .0
+            .map_or(Err(PrefixStoreError::StoreNotReadyError), |p_rec| {
+                p_rec.calculate_and_store_best_backup(tbi, guard)
+            })
+    }
+
+    pub fn is_ps_outdated(
+        &self,
+        search_pfx: PrefixId<AF>,
+        guard: &Guard,
+    ) -> Result<bool, PrefixStoreError> {
+        self.store
+            .non_recursive_retrieve_prefix(search_pfx)
+            .0
+            .map_or(Err(PrefixStoreError::StoreNotReadyError), |p| {
+                Ok(p.is_ps_outdated(guard))
+            })
+    }
+
     // In a LMP search we have to go over all the nibble lengths in the
     // stride up until the value of the actual nibble length were looking for
     // (until we reach stride length for all strides that aren't the last)
@@ -298,6 +360,7 @@ where
     // nibble              1010 1011 1100 1101 1110 1111    x
     // nibble len offset      4(contd.)
 
+    #[deprecated]
     pub fn match_prefix_by_tree_traversal(
         &'a self,
         search_pfx: PrefixId<AF>,
diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index 46b7edc9..bcc8a1ee 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -334,7 +334,7 @@ impl<M: Meta> MultiMapValue<M> {
         self.route_status = status;
     }
 
-    pub(crate) fn has_persisted_history(&self) -> bool {
+    pub(crate) fn has_persisted_data(&self) -> bool {
         self.persist_status.0
     }
 }
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 7a455d02..5e09615d 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -191,7 +191,7 @@ use log::{debug, info, log_enabled, trace};
 use crossbeam_epoch::{self as epoch, Atomic};
 use crossbeam_utils::Backoff;
 use epoch::{Guard, Owned};
-use lsm_tree::{AbstractTree, Segment};
+use lsm_tree::AbstractTree;
 use roaring::RoaringBitmap;
 
 use std::marker::PhantomData;
@@ -361,8 +361,18 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             .collect::<Vec<_>>()
     }
 
-    pub fn flush_to_disk(&self) -> Result<Option<Segment>, lsm_tree::Error> {
-        self.0.flush_active_memtable(0)
+    pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
+        let segment = self.0.flush_active_memtable(0);
+
+        if let Ok(Some(segment)) = segment {
+            self.0.register_segments(&[segment])?;
+            self.0.compact(
+                std::sync::Arc::new(lsm_tree::compaction::Leveled::default()),
+                0,
+            )?;
+        };
+
+        Ok(())
     }
 
     pub fn approximate_len(&self) -> usize {
@@ -373,13 +383,6 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         self.0.disk_space()
     }
 
-    pub fn register_segments(
-        &self,
-        segments: &[lsm_tree::Segment],
-    ) -> Result<(), lsm_tree::Error> {
-        self.0.register_segments(segments)
-    }
-
     #[cfg(feature = "persist")]
     pub fn persistence_key(
         // PREFIX_SIZE bytes
diff --git a/src/local_array/tree.rs b/src/local_array/tree.rs
index 12d2bda8..b42105d1 100644
--- a/src/local_array/tree.rs
+++ b/src/local_array/tree.rs
@@ -373,10 +373,8 @@ impl<
         pfx: PrefixId<AF>,
         record: PublicRecord<M>,
         update_path_selections: Option<M::TBI>,
-        // user_data: Option<&<M as MergeUpdate>::UserDataIn>,
     ) -> Result<UpsertReport, PrefixStoreError> {
         let guard = &epoch::pin();
-        // let record = MultiMapValue::new(meta, ltime, status);
 
         if pfx.get_len() == 0 {
             let res = self.update_default_route_prefix_meta(record, guard)?;
diff --git a/src/local_vec/macros.rs b/src/local_vec/macros.rs
index 2351f4fa..3b25ed64 100644
--- a/src/local_vec/macros.rs
+++ b/src/local_vec/macros.rs
@@ -1,11 +1,10 @@
 #[macro_export]
-
-// This macro expands into a match node {}
-// with match arms for all SizedStrideNode::Stride[3-8]
-// for use in insert()
-
 #[doc(hidden)]
 macro_rules! match_node_for_strides_with_local_vec {
+    // This macro expands into a match node {}
+    // with match arms for all SizedStrideNode::Stride[3-8]
+    // for use in insert()
+
     (
         $self: ident;
         // $user_data: ident;
diff --git a/src/local_vec/query.rs b/src/local_vec/query.rs
index 1b53a563..76a25ed5 100644
--- a/src/local_vec/query.rs
+++ b/src/local_vec/query.rs
@@ -17,11 +17,11 @@ impl<AF: AddressFamily> PrefixId<AF> {
     }
 
     pub fn get_net(&self) -> AF {
-        self.0.0
+        self.0 .0
     }
 
     pub fn get_len(&self) -> u8 {
-        self.0.1
+        self.0 .1
     }
 }
 
@@ -79,9 +79,7 @@ pub struct QueryResult<M: crate::prefix_record::Meta> {
     pub more_specifics: Option<RecordSet<M>>,
 }
 
-impl<M: crate::prefix_record::Meta> std::fmt::Display
-    for QueryResult<M>
-{
+impl<M: crate::prefix_record::Meta> std::fmt::Display for QueryResult<M> {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         let pfx_str = match self.prefix {
             Some(pfx) => format!("{}", pfx),
@@ -109,7 +107,7 @@ impl<M: crate::prefix_record::Meta> std::fmt::Display
 
 //------------ Longest Matching Prefix  -------------------------------------
 
-impl<'a, Store> TreeBitMap<Store>
+impl<Store> TreeBitMap<Store>
 where
     Store: StorageBackend,
 {
@@ -127,7 +125,7 @@ where
     // nibble len offset   0 1    2            3                                4
 
     pub(crate) fn match_prefix(
-        &'a self,
+        &self,
         search_pfx: PrefixId<Store::AF>,
         options: &MatchOptions,
     ) -> QuerySingleResult<Store::Meta> {
@@ -141,8 +139,8 @@ where
 
         // These result values are kept in mutable variables, and assembled
         // at the end into a QueryResult struct. This proved to result in the
-        // most efficient code, where we don't have to match on 
-        // SizedStrideNode over and over. The `match_type` field in the 
+        // most efficient code, where we don't have to match on
+        // SizedStrideNode over and over. The `match_type` field in the
         // QueryResult is computed at the end.
 
         // The final prefix
diff --git a/src/local_vec/tree.rs b/src/local_vec/tree.rs
index 91b30078..bc004f6e 100644
--- a/src/local_vec/tree.rs
+++ b/src/local_vec/tree.rs
@@ -3,7 +3,6 @@ use std::{
     marker::PhantomData,
 };
 
-use crate::{af::{AddressFamily, Zero}, custom_alloc::UpsertReport};
 use crate::local_vec::node::TreeBitMapNode;
 use crate::local_vec::storage_backend::StorageBackend;
 use crate::match_node_for_strides_with_local_vec;
@@ -12,6 +11,10 @@ use crate::prefix_record::InternalPrefixRecord;
 use crate::stats::{SizedStride, StrideStats};
 use crate::stride::*;
 use crate::synth_int::{U256, U512};
+use crate::{
+    af::{AddressFamily, Zero},
+    custom_alloc::UpsertReport,
+};
 
 #[cfg(feature = "cli")]
 use crate::node_id::InMemNodeId;
@@ -56,8 +59,8 @@ pub struct CacheGuard<
     pub guard: std::cell::Ref<'a, SizedStrideNode<AF, NodeId>>,
 }
 
-impl<'a, AF: 'static + AddressFamily, NodeId: SortableNodeId + Copy>
-    std::ops::Deref for CacheGuard<'a, AF, NodeId>
+impl<AF: 'static + AddressFamily, NodeId: SortableNodeId + Copy>
+    std::ops::Deref for CacheGuard<'_, AF, NodeId>
 {
     type Target = SizedStrideNode<AF, NodeId>;
 
@@ -74,8 +77,8 @@ pub struct PrefixCacheGuard<
     pub guard: std::cell::Ref<'a, InternalPrefixRecord<AF, Meta>>,
 }
 
-impl<'a, AF: 'static + AddressFamily, Meta: crate::prefix_record::Meta>
-    std::ops::Deref for PrefixCacheGuard<'a, AF, Meta>
+impl<AF: 'static + AddressFamily, Meta: crate::prefix_record::Meta>
+    std::ops::Deref for PrefixCacheGuard<'_, AF, Meta>
 {
     type Target = InternalPrefixRecord<AF, Meta>;
 
@@ -340,7 +343,12 @@ where
         match self.store.retrieve_prefix_mut(update_node_idx) {
             Some(update_pfx) => {
                 update_pfx.meta = meta;
-                Ok(UpsertReport { cas_count: 0, prefix_new: false, mui_new: false, mui_count: 0 })
+                Ok(UpsertReport {
+                    cas_count: 0,
+                    prefix_new: false,
+                    mui_new: false,
+                    mui_count: 0,
+                })
                 // <Store::Meta>::merge_update(&mut update_pfx.meta, meta)
             }
             // TODO
@@ -471,7 +479,8 @@ impl<Store: StorageBackend> std::fmt::Display for TreeBitMap<Store> {
         writeln!(_f, "prefix vec size {}", self.store.get_prefixes_len())?;
         writeln!(_f, "finished building tree...")?;
         writeln!(_f, "{:?} nodes created", total_nodes)?;
-        writeln!(_f,
+        writeln!(
+            _f,
             "size of node: {} bytes",
             std::mem::size_of::<SizedStrideNode<u32, InMemNodeId>>()
         )?;
@@ -555,7 +564,7 @@ impl<Store: StorageBackend> std::fmt::Display for TreeBitMap<Store> {
                 ) //  = scale / 7
             )?;
 
-            writeln!(_f," {}", prefixes_num)?;
+            writeln!(_f, " {}", prefixes_num)?;
         }
         Ok(())
     }
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index 00f11947..fb3081ec 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -399,8 +399,7 @@ impl<M: Meta>
     }
 }
 
-impl<'a, M: Meta + 'a>
-    std::iter::FromIterator<Arc<PublicPrefixSingleRecord<M>>>
+impl<M: Meta> std::iter::FromIterator<Arc<PublicPrefixSingleRecord<M>>>
     for RecordSingleSet<M>
 {
     fn from_iter<I: IntoIterator<Item = Arc<PublicPrefixSingleRecord<M>>>>(
@@ -429,7 +428,7 @@ impl<'a, M: Meta + 'a>
     }
 }
 
-impl<'a, AF: AddressFamily, M: Meta + 'a>
+impl<AF: AddressFamily, M: Meta>
     std::iter::FromIterator<(PrefixId<AF>, Arc<M>)> for RecordSingleSet<M>
 {
     fn from_iter<I: IntoIterator<Item = (PrefixId<AF>, Arc<M>)>>(
@@ -582,7 +581,7 @@ impl<M: Meta> From<(Vec<PublicPrefixRecord<M>>, Vec<PublicPrefixRecord<M>>)>
     }
 }
 
-impl<'a, M: Meta + 'a> std::iter::FromIterator<PublicPrefixRecord<M>>
+impl<M: Meta> std::iter::FromIterator<PublicPrefixRecord<M>>
     for RecordSet<M>
 {
     fn from_iter<I: IntoIterator<Item = PublicPrefixRecord<M>>>(
@@ -605,7 +604,7 @@ impl<'a, M: Meta + 'a> std::iter::FromIterator<PublicPrefixRecord<M>>
     }
 }
 
-impl<'a, AF: AddressFamily, M: Meta + 'a>
+impl<AF: AddressFamily, M: Meta>
     std::iter::FromIterator<(PrefixId<AF>, Vec<PublicRecord<M>>)>
     for RecordSet<M>
 {
@@ -674,7 +673,7 @@ pub struct RecordSetSingleIter<'a, M: Meta> {
     v6: std::slice::Iter<'a, PublicPrefixSingleRecord<M>>,
 }
 
-impl<'a, M: Meta> Iterator for RecordSetSingleIter<'a, M> {
+impl<M: Meta> Iterator for RecordSetSingleIter<'_, M> {
     type Item = PublicPrefixSingleRecord<M>;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -698,7 +697,7 @@ pub struct RecordSetIter<'a, M: Meta> {
     v6: std::slice::Iter<'a, PublicPrefixRecord<M>>,
 }
 
-impl<'a, M: Meta> Iterator for RecordSetIter<'a, M> {
+impl<M: Meta> Iterator for RecordSetIter<'_, M> {
     type Item = PublicPrefixRecord<M>;
 
     fn next(&mut self) -> Option<Self::Item> {
diff --git a/src/rotonda_store.rs b/src/rotonda_store.rs
index 36ed0d13..8187332c 100644
--- a/src/rotonda_store.rs
+++ b/src/rotonda_store.rs
@@ -1,7 +1,9 @@
 use std::{fmt, slice};
 
+pub use crate::prefix_record::{
+    Meta, PublicPrefixSingleRecord, RecordSingleSet,
+};
 use crate::prefix_record::{PublicRecord, RecordSet};
-pub use crate::prefix_record::{PublicPrefixSingleRecord, Meta, RecordSingleSet};
 use crate::{prefix_record::InternalPrefixRecord, stats::StrideStats};
 
 use inetnum::addr::Prefix;
@@ -26,7 +28,7 @@ pub struct Stats<'a> {
     pub v6: &'a AfStrideStats,
 }
 
-impl<'a> std::fmt::Display for Stats<'a> {
+impl std::fmt::Display for Stats<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(f, "v4 ")?;
         for s in self.v4.iter() {
@@ -45,7 +47,7 @@ pub struct Strides<'a> {
     pub v6: &'a Vec<u8>,
 }
 
-impl<'a> std::fmt::Debug for Strides<'a> {
+impl std::fmt::Debug for Strides<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "v4 ")?;
         for s in self.v4.iter() {
@@ -62,13 +64,13 @@ impl<'a> std::fmt::Debug for Strides<'a> {
 //------------ MatchOptions / MatchType -------------------------------------
 
 /// Options for the `match_prefix` method
-/// 
+///
 /// The `MatchOptions` struct is used to specify the options for the
 /// `match_prefix` method on the store.
-/// 
+///
 /// Note that the `match_type` field may be different from the actual
-/// `MatchType` returned from the result. 
-/// 
+/// `MatchType` returned from the result.
+///
 /// See [MultiThreadedStore::match_prefix] for more details.
 #[derive(Debug, Clone)]
 pub struct MatchOptions {
@@ -82,7 +84,7 @@ pub struct MatchOptions {
     pub include_more_specifics: bool,
     /// Whether to return records for a specific multi_uniq_id, None indicates
     /// all records.
-    pub mui: Option<u32>
+    pub mui: Option<u32>,
 }
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
@@ -108,7 +110,6 @@ impl std::fmt::Display for MatchType {
     }
 }
 
-
 //------------ PrefixRecordIter ---------------------------------------------
 
 // Converts from the InternalPrefixRecord to the (public) PrefixRecord
@@ -119,9 +120,7 @@ pub struct PrefixSingleRecordIter<'a, M: Meta> {
     pub(crate) v6: slice::Iter<'a, InternalPrefixRecord<IPv6, M>>,
 }
 
-impl<'a, M: Meta> Iterator
-    for PrefixSingleRecordIter<'a, M>
-{
+impl<M: Meta> Iterator for PrefixSingleRecordIter<'_, M> {
     type Item = PublicPrefixSingleRecord<M>;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -146,17 +145,15 @@ impl<'a, M: Meta> Iterator
     }
 }
 
-
 //------------- QueryResult -------------------------------------------------
 
 /// The type that is returned by a query.
-/// 
+///
 /// This is the result type of a query. It contains the prefix record that was
 /// found in the store, as well as less- or more-specifics as requested.
-/// 
+///
 /// See [MultiThreadedStore::match_prefix] for more details.
 
-
 #[derive(Clone, Debug)]
 pub struct QueryResult<M: crate::prefix_record::Meta> {
     /// The match type of the resulting prefix
@@ -188,15 +185,23 @@ impl<M: Meta> fmt::Display for QueryResult<M> {
             write!(f, "{},", rec)?;
         }
         writeln!(f, " ]")?;
-        writeln!(f, "less_specifics: {{ {} }}", if let Some(ls) = self.less_specifics.as_ref() {
-            format!("{}", ls)
-        } else {
-            "".to_string()
-        })?;
-        writeln!(f, "more_specifics: {{ {} }}", if let Some(ms) = self.more_specifics.as_ref() {
-            format!("{}", ms)
-        } else {
-            "".to_string()
-        })
+        writeln!(
+            f,
+            "less_specifics: {{ {} }}",
+            if let Some(ls) = self.less_specifics.as_ref() {
+                format!("{}", ls)
+            } else {
+                "".to_string()
+            }
+        )?;
+        writeln!(
+            f,
+            "more_specifics: {{ {} }}",
+            if let Some(ms) = self.more_specifics.as_ref() {
+                format!("{}", ms)
+            } else {
+                "".to_string()
+            }
+        )
     }
 }

From d79f5b13912d2d64b7b7ebad23837c8bf6003953 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 17 Dec 2024 16:23:03 +0100
Subject: [PATCH 021/147] CustomAllocStorage -> Rib refactor (big)

---
 src/bin/load_mrt.rs                           |    6 +-
 src/local_array/{store => }/errors.rs         |    0
 .../{ => in_memory}/atomic_stride.rs          |    4 +-
 src/local_array/in_memory/atomic_types.rs     |  670 ++++++++
 src/local_array/{ => in_memory}/macros.rs     |   16 +-
 src/local_array/in_memory/mod.rs              |    8 +
 src/local_array/in_memory/node.rs             | 1087 +++++++++++++
 src/local_array/in_memory/oncebox.rs          |  158 ++
 src/local_array/{ => in_memory}/tree.rs       |  284 ++--
 src/local_array/{store => }/iterators.rs      |   49 +-
 src/local_array/mod.rs                        |   16 +-
 src/local_array/node.rs                       |   11 +
 src/local_array/query.rs                      |  253 ++-
 src/local_array/rib/default_store.rs          |   28 +
 src/local_array/rib/macros.rs                 |  379 +++++
 src/local_array/rib/mod.rs                    |    7 +
 src/local_array/rib/rib.rs                    | 1426 +++++++++++++++++
 src/local_array/store/atomic_types.rs         |   53 +
 src/local_array/store/custom_alloc.rs         |  143 ++
 src/local_array/store/default_store.rs        |   59 -
 src/local_array/store/macros.rs               |   77 +
 src/local_array/store/mod.rs                  |   11 -
 src/local_array/store/oncebox.rs              |   52 +
 src/local_array/tests.rs                      |    4 +-
 src/local_array/types.rs                      |  119 ++
 src/local_vec/tree.rs                         |    2 +-
 src/prefix_record.rs                          |    5 +-
 src/prelude/mod.rs                            |   12 +-
 src/rotonda_store.rs                          |    4 +-
 29 files changed, 4556 insertions(+), 387 deletions(-)
 rename src/local_array/{store => }/errors.rs (100%)
 rename src/local_array/{ => in_memory}/atomic_stride.rs (99%)
 create mode 100644 src/local_array/in_memory/atomic_types.rs
 rename src/local_array/{ => in_memory}/macros.rs (91%)
 create mode 100644 src/local_array/in_memory/mod.rs
 create mode 100644 src/local_array/in_memory/node.rs
 create mode 100644 src/local_array/in_memory/oncebox.rs
 rename src/local_array/{ => in_memory}/tree.rs (78%)
 rename src/local_array/{store => }/iterators.rs (97%)
 create mode 100644 src/local_array/rib/default_store.rs
 create mode 100644 src/local_array/rib/macros.rs
 create mode 100644 src/local_array/rib/mod.rs
 create mode 100644 src/local_array/rib/rib.rs
 delete mode 100644 src/local_array/store/default_store.rs
 delete mode 100644 src/local_array/store/mod.rs
 create mode 100644 src/local_array/types.rs

diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index 8e3e98b6..14c17107 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -12,11 +12,11 @@ use memmap2::Mmap;
 use rayon::iter::ParallelBridge;
 use rayon::iter::ParallelIterator;
 use rayon::prelude::*;
-use rotonda_store::custom_alloc::PersistStrategy;
-use rotonda_store::custom_alloc::StoreConfig;
-use rotonda_store::custom_alloc::UpsertReport;
 use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::{MultiThreadedStore, RouteStatus};
+use rotonda_store::rib::PersistStrategy;
+use rotonda_store::rib::StoreConfig;
+use rotonda_store::rib::UpsertReport;
 use rotonda_store::PublicRecord;
 use routecore::bgp::message::PduParseInfo;
 use routecore::bgp::path_attributes::OwnedPathAttributes;
diff --git a/src/local_array/store/errors.rs b/src/local_array/errors.rs
similarity index 100%
rename from src/local_array/store/errors.rs
rename to src/local_array/errors.rs
diff --git a/src/local_array/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
similarity index 99%
rename from src/local_array/atomic_stride.rs
rename to src/local_array/in_memory/atomic_stride.rs
index a3788d76..75d9e294 100644
--- a/src/local_array/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -9,6 +9,8 @@ use crate::af::Zero;
 use crate::synth_int::AtomicU128;
 use crate::{impl_primitive_atomic_stride, AddressFamily};
 
+use super::tree::StrideNodeId;
+
 pub type Stride3 = u16;
 pub type Stride4 = u32;
 pub type Stride5 = u64;
@@ -563,7 +565,7 @@ where
     fn into_node_id<AF: AddressFamily>(
         addr_bits: AF,
         len: u8,
-    ) -> super::node::StrideNodeId<AF>;
+    ) -> StrideNodeId<AF>;
 
     // Convert a ptrbitarr into a pfxbitarr sized bitmap,
     // so we can do bitwise operations with a pfxbitarr sized
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
new file mode 100644
index 00000000..f5b9870d
--- /dev/null
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -0,0 +1,670 @@
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex, MutexGuard, RwLock};
+use std::{
+    fmt::{Debug, Display},
+    sync::atomic::Ordering,
+};
+
+use crossbeam_epoch::{self as epoch, Atomic};
+
+use crossbeam_utils::Backoff;
+use log::{debug, log_enabled, trace};
+
+use epoch::{Guard, Owned};
+use roaring::RoaringBitmap;
+
+use crate::local_array::types::{PrefixId, RouteStatus};
+// use crate::local_array::in_memory_tree::*;
+use crate::prefix_record::PublicRecord;
+use crate::prelude::Meta;
+use crate::rib::{PersistStrategy, PersistTree};
+use crate::AddressFamily;
+
+use super::super::errors::PrefixStoreError;
+use super::atomic_stride;
+use super::node::TreeBitMapNode;
+use super::oncebox::OnceBoxSlice;
+use super::tree::{Stride, Stride3, Stride4, Stride5, StrideNodeId};
+
+// ----------- Node related structs -----------------------------------------
+
+#[derive(Debug)]
+pub struct StoredNode<AF, S>
+where
+    Self: Sized,
+    S: Stride,
+    AF: AddressFamily,
+{
+    pub(crate) node_id: StrideNodeId<AF>,
+    // The ptrbitarr and pfxbitarr for this node
+    pub(crate) node: TreeBitMapNode<AF, S>,
+    // Child nodes linked from this node
+    pub(crate) node_set: NodeSet<AF, S>,
+}
+
+#[allow(clippy::type_complexity)]
+#[derive(Debug)]
+pub struct NodeSet<AF: AddressFamily, S: Stride>(
+    pub OnceBoxSlice<StoredNode<AF, S>>,
+    // A Bitmap index that keeps track of the `multi_uniq_id`s (mui) that are
+    // present in value collections in the meta-data tree in the child nodes
+    pub RwLock<RoaringBitmap>,
+);
+
+impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
+    pub fn init(p2_size: u8) -> Self {
+        if log_enabled!(log::Level::Debug) {
+            debug!(
+                "{} store: creating space for {} nodes",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                1 << p2_size
+            );
+        }
+
+        NodeSet(OnceBoxSlice::new(p2_size), RoaringBitmap::new().into())
+    }
+
+    pub fn update_rbm_index(
+        &self,
+        multi_uniq_id: u32,
+    ) -> Result<u32, crate::prelude::multi::PrefixStoreError>
+    where
+        S: atomic_stride::Stride,
+        AF: crate::AddressFamily,
+    {
+        let try_count = 0;
+        let mut rbm = self.1.write().unwrap();
+        rbm.insert(multi_uniq_id);
+
+        Ok(try_count)
+    }
+
+    pub fn remove_from_rbm_index(
+        &self,
+        multi_uniq_id: u32,
+        _guard: &crate::epoch::Guard,
+    ) -> Result<u32, crate::prelude::multi::PrefixStoreError>
+    where
+        S: atomic_stride::Stride,
+        AF: crate::AddressFamily,
+    {
+        let try_count = 0;
+
+        let mut rbm = self.1.write().unwrap();
+        rbm.remove(multi_uniq_id);
+
+        Ok(try_count)
+    }
+}
+
+// ----------- Prefix related structs ---------------------------------------
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub struct PathSelections {
+    pub(crate) path_selection_muis: (Option<u32>, Option<u32>),
+}
+
+impl PathSelections {
+    pub fn best(&self) -> Option<u32> {
+        self.path_selection_muis.0
+    }
+
+    pub fn backup(&self) -> Option<u32> {
+        self.path_selection_muis.1
+    }
+}
+
+// ----------- StoredPrefix -------------------------------------------------
+// This is the top-level struct that's linked from the slots in the buckets.
+// It contains a super_agg_record that is supposed to hold counters for the
+// records that are stored inside it, so that iterators over its linked lists
+// don't have to go into them if there's nothing there and could stop early.
+#[derive(Debug)]
+pub struct StoredPrefix<AF: AddressFamily, M: crate::prefix_record::Meta> {
+    // the serial number
+    // pub serial: usize,
+    // the prefix itself,
+    pub prefix: PrefixId<AF>,
+    // the aggregated data for this prefix
+    pub record_map: MultiMap<M>,
+    // (mui of best path entry, mui of backup path entry) from the record_map
+    path_selections: Atomic<PathSelections>,
+    // the reference to the next set of records for this prefix, if any.
+    pub next_bucket: PrefixSet<AF, M>,
+}
+
+impl<AF: AddressFamily, M: crate::prefix_record::Meta> StoredPrefix<AF, M> {
+    pub(crate) fn new<PB: PrefixBuckets<AF, M>>(
+        pfx_id: PrefixId<AF>,
+        level: u8,
+    ) -> Self {
+        // start calculation size of next set, it's dependent on the level
+        // we're in.
+        // let pfx_id = PrefixId::new(record.net, record.len);
+        let this_level = PB::get_bits_for_len(pfx_id.get_len(), level);
+        let next_level = PB::get_bits_for_len(pfx_id.get_len(), level + 1);
+
+        trace!("this level {} next level {}", this_level, next_level);
+        let next_bucket: PrefixSet<AF, M> = if next_level > 0 {
+            debug!(
+                "{} store: INSERT with new bucket of size {} at prefix len {}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                1 << (next_level - this_level),
+                pfx_id.get_len()
+            );
+            PrefixSet::init(next_level - this_level)
+        } else {
+            debug!(
+                "{} store: INSERT at LAST LEVEL with empty bucket at prefix len {}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                pfx_id.get_len()
+            );
+            PrefixSet::init(next_level - this_level)
+        };
+        // End of calculation
+
+        let rec_map = HashMap::new();
+
+        StoredPrefix {
+            // serial: 1,
+            prefix: pfx_id,
+            path_selections: Atomic::init(PathSelections {
+                path_selection_muis: (None, None),
+            }),
+            record_map: MultiMap::new(rec_map),
+            next_bucket,
+        }
+    }
+
+    pub(crate) fn get_prefix_id(&self) -> PrefixId<AF> {
+        self.prefix
+    }
+
+    pub fn get_path_selections(&self, guard: &Guard) -> PathSelections {
+        let path_selections =
+            self.path_selections.load(Ordering::Acquire, guard);
+
+        unsafe { path_selections.as_ref() }.map_or(
+            PathSelections {
+                path_selection_muis: (None, None),
+            },
+            |ps| *ps,
+        )
+    }
+
+    pub(crate) fn set_path_selections(
+        &self,
+        path_selections: PathSelections,
+        guard: &Guard,
+    ) -> Result<(), PrefixStoreError> {
+        let current = self.path_selections.load(Ordering::SeqCst, guard);
+
+        if unsafe { current.as_ref() } == Some(&path_selections) {
+            debug!("unchanged path_selections");
+            return Ok(());
+        }
+
+        self.path_selections
+            .compare_exchange(
+                current,
+                // Set the tag to indicate we're updated
+                Owned::new(path_selections).with_tag(0),
+                Ordering::AcqRel,
+                Ordering::Acquire,
+                guard,
+            )
+            .map_err(|_| PrefixStoreError::PathSelectionOutdated)?;
+        Ok(())
+    }
+
+    pub fn set_ps_outdated(
+        &self,
+        guard: &Guard,
+    ) -> Result<(), PrefixStoreError> {
+        self.path_selections
+            .fetch_update(Ordering::Acquire, Ordering::Acquire, guard, |p| {
+                Some(p.with_tag(1))
+            })
+            .map(|_| ())
+            .map_err(|_| PrefixStoreError::StoreNotReadyError)
+    }
+
+    pub fn is_ps_outdated(&self, guard: &Guard) -> bool {
+        self.path_selections.load(Ordering::Acquire, guard).tag() == 1
+    }
+
+    pub fn calculate_and_store_best_backup<'a>(
+        &'a self,
+        tbi: &M::TBI,
+        guard: &'a Guard,
+    ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
+        let path_selection_muis = self.record_map.best_backup(*tbi);
+
+        self.set_path_selections(
+            PathSelections {
+                path_selection_muis,
+            },
+            guard,
+        )?;
+
+        Ok(path_selection_muis)
+    }
+
+    pub(crate) fn get_next_bucket(&self) -> Option<&PrefixSet<AF, M>> {
+        if self.next_bucket.is_empty() {
+            None
+        } else {
+            Some(&self.next_bucket)
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct PersistStatus(bool);
+
+impl PersistStatus {
+    pub(crate) fn persisted() -> Self {
+        Self(true)
+    }
+
+    pub(crate) fn not_persisted() -> Self {
+        Self(false)
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct MultiMapValue<M> {
+    meta: M,
+    ltime: u64,
+    route_status: RouteStatus,
+    persist_status: PersistStatus,
+}
+
+impl<M: Meta> MultiMapValue<M> {
+    pub(crate) fn logical_time(&self) -> u64 {
+        self.ltime
+    }
+
+    pub(crate) fn meta(&self) -> &M {
+        &self.meta
+    }
+
+    pub(crate) fn route_status(&self) -> RouteStatus {
+        self.route_status
+    }
+
+    pub(crate) fn set_route_status(&mut self, status: RouteStatus) {
+        self.route_status = status;
+    }
+
+    pub(crate) fn has_persisted_data(&self) -> bool {
+        self.persist_status.0
+    }
+}
+
+impl<M: Meta> std::fmt::Display for MultiMapValue<M> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{} {}",
+            // self.meta(),
+            self.logical_time(),
+            self.route_status()
+        )
+    }
+}
+
+impl<M: Meta> From<(PublicRecord<M>, PersistStatus)> for MultiMapValue<M> {
+    fn from(value: (PublicRecord<M>, PersistStatus)) -> Self {
+        Self {
+            ltime: value.0.ltime,
+            route_status: value.0.status,
+            meta: value.0.meta,
+            persist_status: value.1,
+        }
+    }
+}
+
+impl<M: Meta> From<(u32, &MultiMapValue<M>)> for PublicRecord<M> {
+    fn from(value: (u32, &MultiMapValue<M>)) -> Self {
+        Self {
+            multi_uniq_id: value.0,
+            meta: value.1.meta().clone(),
+            ltime: value.1.ltime,
+            status: value.1.route_status,
+        }
+    }
+}
+
+// ----------- MultiMap ------------------------------------------------------
+// This is the record that holds the aggregates at the top-level for a given
+// prefix.
+
+#[derive(Debug)]
+pub struct MultiMap<M: Meta>(
+    Arc<Mutex<std::collections::HashMap<u32, MultiMapValue<M>>>>,
+);
+
+impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
+    pub(crate) fn new(record_map: HashMap<u32, MultiMapValue<M>>) -> Self {
+        Self(Arc::new(Mutex::new(record_map)))
+    }
+
+    fn guard_with_retry(
+        &self,
+        mut retry_count: usize,
+    ) -> (MutexGuard<HashMap<u32, MultiMapValue<M>>>, usize) {
+        let backoff = Backoff::new();
+
+        loop {
+            if let Ok(guard) = self.0.try_lock() {
+                return (guard, retry_count);
+            }
+
+            backoff.spin();
+            retry_count += 1;
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        record_map.len()
+    }
+
+    pub fn get_record_for_active_mui(
+        &self,
+        mui: u32,
+    ) -> Option<PublicRecord<M>> {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+
+        record_map.get(&mui).and_then(|r| {
+            if r.route_status() == RouteStatus::Active {
+                Some(PublicRecord::from((mui, r)))
+            } else {
+                None
+            }
+        })
+    }
+
+    pub fn best_backup(&self, tbi: M::TBI) -> (Option<u32>, Option<u32>) {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        let ord_routes = record_map
+            .iter()
+            .map(|r| (r.1.meta().as_orderable(tbi), *r.0));
+        let (best, bckup) =
+            routecore::bgp::path_selection::best_backup_generic(ord_routes);
+        (best.map(|b| b.1), bckup.map(|b| b.1))
+    }
+
+    pub(crate) fn get_record_for_mui_with_rewritten_status(
+        &self,
+        mui: u32,
+        bmin: &RoaringBitmap,
+        rewrite_status: RouteStatus,
+    ) -> Option<PublicRecord<M>> {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        record_map.get(&mui).map(|r| {
+            // We'll return a cloned record: the record in the store remains
+            // untouched.
+            let mut r = r.clone();
+            if bmin.contains(mui) {
+                r.set_route_status(rewrite_status);
+            }
+            PublicRecord::from((mui, &r))
+        })
+    }
+
+    // Helper to filter out records that are not-active (Inactive or
+    // Withdrawn), or whose mui appears in the global withdrawn index.
+    pub fn get_filtered_records(
+        &self,
+        mui: Option<u32>,
+        bmin: &RoaringBitmap,
+    ) -> Vec<PublicRecord<M>> {
+        if let Some(mui) = mui {
+            self.get_record_for_active_mui(mui).into_iter().collect()
+        } else {
+            self.as_active_records_not_in_bmin(bmin)
+        }
+    }
+
+    // return all records regardless of their local status, or any globally
+    // set status for the mui of the record. However, the local status for a
+    // record whose mui appears in the specified bitmap index, will be
+    // rewritten with the specified RouteStatus.
+    pub fn as_records_with_rewritten_status(
+        &self,
+        bmin: &RoaringBitmap,
+        rewrite_status: RouteStatus,
+    ) -> Vec<PublicRecord<M>> {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        record_map
+            .iter()
+            .map(move |r| {
+                let mut rec = r.1.clone();
+                if bmin.contains(*r.0) {
+                    rec.set_route_status(rewrite_status);
+                }
+                PublicRecord::from((*r.0, &rec))
+            })
+            .collect::<Vec<_>>()
+    }
+
+    pub fn as_records(&self) -> Vec<PublicRecord<M>> {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        record_map
+            .iter()
+            .map(|r| PublicRecord::from((*r.0, r.1)))
+            .collect::<Vec<_>>()
+    }
+
+    // Returns a vec of records whose keys are not in the supplied bitmap
+    // index, and whose local Status is set to Active. Used to filter out
+    // withdrawn routes.
+    pub fn as_active_records_not_in_bmin(
+        &self,
+        bmin: &RoaringBitmap,
+    ) -> Vec<PublicRecord<M>> {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        record_map
+            .iter()
+            .filter_map(|r| {
+                if r.1.route_status() == RouteStatus::Active
+                    && !bmin.contains(*r.0)
+                {
+                    Some(PublicRecord::from((*r.0, r.1)))
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>()
+    }
+
+    // Change the local status of the record for this mui to Withdrawn.
+    pub fn mark_as_withdrawn_for_mui(&self, mui: u32) {
+        let c_map = Arc::clone(&self.0);
+        let mut record_map = c_map.lock().unwrap();
+        if let Some(rec) = record_map.get_mut(&mui) {
+            rec.set_route_status(RouteStatus::Withdrawn);
+        }
+    }
+
+    // Change the local status of the record for this mui to Active.
+    pub fn mark_as_active_for_mui(&self, mui: u32) {
+        let record_map = Arc::clone(&self.0);
+        let mut r_map = record_map.lock().unwrap();
+        if let Some(rec) = r_map.get_mut(&mui) {
+            rec.set_route_status(RouteStatus::Active);
+        }
+    }
+
+    // Insert or replace the PublicRecord in the HashMap for the key of
+    // record.multi_uniq_id. Returns the number of entries in the HashMap
+    // after updating it, if it's more than 1. Returns None if this is the
+    // first entry.
+    pub(crate) fn upsert_record<
+        AF: AddressFamily,
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    >(
+        &self,
+        prefix: PrefixId<AF>,
+        new_rec: PublicRecord<M>,
+        persistence: &Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
+        strategy: PersistStrategy,
+    ) -> Result<(Option<usize>, usize), PrefixStoreError> {
+        let (mut record_map, retry_count) = self.guard_with_retry(0);
+        let key = new_rec.multi_uniq_id;
+
+        match (strategy, record_map.get_mut(&key)) {
+            // New record for (prefix, mui) in memory.
+
+            // We store the record in memory only.
+            (
+                PersistStrategy::PersistHistory | PersistStrategy::MemoryOnly,
+                None,
+            ) => {
+                record_map.insert(
+                    key,
+                    MultiMapValue::from((
+                        new_rec,
+                        PersistStatus::not_persisted(),
+                    )),
+                );
+
+                Ok((None, retry_count))
+            }
+            // We only persist the record.
+            (PersistStrategy::PersistOnly, None) => {
+                if let Some(persistence) = persistence {
+                    persistence.persist_record(prefix, key, &new_rec);
+                    Ok((None, retry_count))
+                } else {
+                    Err(PrefixStoreError::PersistFailed)
+                }
+            }
+            // We store both in memory and persist it.
+            (PersistStrategy::WriteAhead, None) => {
+                if let Some(persistence) = persistence {
+                    persistence.persist_record(prefix, key, &new_rec);
+                    let mmv = MultiMapValue::from((
+                        new_rec,
+                        PersistStatus::persisted(),
+                    ));
+                    record_map.insert(key, mmv);
+
+                    Ok((None, retry_count))
+                } else {
+                    Err(PrefixStoreError::PersistFailed)
+                }
+            }
+
+            // Existing record for (prefix, mui) in memory.
+
+            // We store the record in memory only, and discard the old record.
+            (PersistStrategy::MemoryOnly, Some(exist_rec)) => {
+                *exist_rec = MultiMapValue::from((
+                    new_rec,
+                    PersistStatus::not_persisted(),
+                ));
+
+                Ok((Some(record_map.len()), retry_count))
+            }
+            // We only persist record, so how come there's one in memory?
+            // Should not happen.
+            (PersistStrategy::PersistOnly, Some(_)) => {
+                panic!("Encountered illegally stored record");
+            }
+            // We store the new record in memory and persist the old record.
+            (
+                PersistStrategy::PersistHistory | PersistStrategy::WriteAhead,
+                Some(exist_rec),
+            ) => {
+                if let Some(persistence) = persistence {
+                    persistence.persist_record(prefix, key, &new_rec);
+                    *exist_rec = MultiMapValue::from((
+                        new_rec,
+                        PersistStatus::persisted(),
+                    ));
+
+                    Ok((Some(record_map.len()), retry_count))
+                } else {
+                    Err(PrefixStoreError::PersistFailed)
+                }
+            }
+        }
+    }
+}
+
+impl<M: Meta> Clone for MultiMap<M> {
+    fn clone(&self) -> Self {
+        Self(Arc::clone(&self.0))
+    }
+}
+
+// ----------- FamilyBuckets Trait ------------------------------------------
+//
+// Implementations of this trait are done by a proc-macro called
+// `stride_sizes`from the `rotonda-macros` crate.
+
+pub trait NodeBuckets<AF: AddressFamily> {
+    fn init() -> Self;
+    fn len_to_store_bits(len: u8, level: u8) -> u8;
+    fn get_stride_sizes(&self) -> &[u8];
+    fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8;
+    fn get_store3(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride3>;
+    fn get_store4(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride4>;
+    fn get_store5(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride5>;
+    fn get_strides_len() -> u8;
+    fn get_first_stride_size() -> u8;
+}
+
+pub trait PrefixBuckets<AF: AddressFamily, M: Meta>
+where
+    Self: Sized,
+{
+    fn init() -> Self;
+    fn remove(&mut self, id: PrefixId<AF>) -> Option<M>;
+    fn get_root_prefix_set(&self, len: u8) -> &'_ PrefixSet<AF, M>;
+    fn get_bits_for_len(len: u8, level: u8) -> u8;
+}
+
+//------------ PrefixSet ----------------------------------------------------
+
+// The PrefixSet is the ARRAY that holds all the child prefixes in a node.
+// Since we are storing these prefixes in the global store in a HashMap that
+// is keyed on the tuple (addr_bits, len, serial number) we can get away with
+// storing ONLY THE SERIAL NUMBER in the pfx_vec: The addr_bits and len are
+// implied in the position in the array a serial number has. A PrefixSet
+// doesn't know anything about the node it is contained in, so it needs a base
+// address to be able to calculate the complete prefix of a child prefix.
+
+#[derive(Debug)]
+#[repr(align(8))]
+pub struct PrefixSet<AF: AddressFamily, M: Meta>(
+    pub OnceBoxSlice<StoredPrefix<AF, M>>,
+);
+
+impl<AF: AddressFamily, M: Meta> PrefixSet<AF, M> {
+    pub fn init(p2_size: u8) -> Self {
+        PrefixSet(OnceBoxSlice::new(p2_size))
+    }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.0.is_null()
+    }
+
+    pub(crate) fn get_by_index(
+        &self,
+        index: usize,
+    ) -> Option<&StoredPrefix<AF, M>> {
+        self.0.get(index)
+    }
+}
diff --git a/src/local_array/macros.rs b/src/local_array/in_memory/macros.rs
similarity index 91%
rename from src/local_array/macros.rs
rename to src/local_array/in_memory/macros.rs
index 8ef26c66..6c0f7ed2 100644
--- a/src/local_array/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -47,7 +47,7 @@ macro_rules! insert_match {
             // retry_count from this macro.
             let local_retry_count = 0;
             // retrieve_node_mut updates the bitmap index if necessary.
-            if let Some(current_node) = $self.store.retrieve_node_mut($cur_i, $record.multi_uniq_id) {
+            if let Some(current_node) = $self.retrieve_node_mut($cur_i, $record.multi_uniq_id) {
                 match current_node {
                     $(
                         SizedStrideRef::$variant(current_node) => {
@@ -62,7 +62,7 @@ macro_rules! insert_match {
                                 // the length of THIS stride
                                 $stride_len,
                                 // the length of the next stride
-                                $self.store.get_stride_sizes().get(($level + 1) as usize),
+                                $self.get_stride_sizes().get(($level + 1) as usize),
                                 $is_last_stride,
                             ) {
                                 (NewNodeOrIndex::NewNode(n), retry_count) => {
@@ -70,13 +70,13 @@ macro_rules! insert_match {
                                     // $self.stats[$stats_level].inc($level);
 
                                     // get a new identifier for the node we're going to create.
-                                    let new_id = $self.store.acquire_new_node_id(($pfx.get_net(), $truncate_len + $nibble_len));
+                                    let new_id = $self.acquire_new_node_id(($pfx.get_net(), $truncate_len + $nibble_len));
 
                                     // store the new node in the global
                                     // store. It returns the created id
                                     // and the number of retries before
                                     // success.
-                                    match $self.store.store_node(new_id, $record.multi_uniq_id, n)  {
+                                    match $self.store_node(new_id, $record.multi_uniq_id, n)  {
                                         Ok((node_id, s_retry_count)) => {
                                             Ok((node_id, $acc_retry_count + s_retry_count + retry_count))
                                         },
@@ -97,7 +97,7 @@ macro_rules! insert_match {
                                     Ok((node_id, $acc_retry_count + local_retry_count + retry_count))
                                 },
                                 (NewNodeOrIndex::NewPrefix, retry_count) => {
-                                    return $self.store.upsert_prefix($pfx, $record, $update_path_selections, $guard)
+                                    return $self.upsert_prefix($pfx, $record, $update_path_selections, $guard)
                                         .and_then(|mut r| {
                                             r.cas_count += $acc_retry_count as usize + local_retry_count as usize + retry_count as usize;
                                             Ok(r)
@@ -106,7 +106,7 @@ macro_rules! insert_match {
                                     // $self.stats[$stats_level].inc_prefix_count($level);
                                 }
                                 (NewNodeOrIndex::ExistingPrefix, retry_count) => {
-                                    return $self.store.upsert_prefix($pfx, $record, $update_path_selections, $guard)
+                                    return $self.upsert_prefix($pfx, $record, $update_path_selections, $guard)
                                         .and_then(|mut r| {
                                             r.cas_count += $acc_retry_count as usize + local_retry_count as usize + retry_count as usize;
                                             Ok(r)
@@ -175,8 +175,8 @@ macro_rules! impl_primitive_atomic_stride {
                 fn into_node_id<AF: AddressFamily>(
                     addr_bits: AF,
                     len: u8
-                ) -> $crate::local_array::node::StrideNodeId<AF> {
-                    let id = $crate::local_array::node::StrideNodeId::new_with_cleaned_id(addr_bits, len);
+                ) -> StrideNodeId<AF> {
+                    let id = StrideNodeId::new_with_cleaned_id(addr_bits, len);
                     id
                 }
 
diff --git a/src/local_array/in_memory/mod.rs b/src/local_array/in_memory/mod.rs
new file mode 100644
index 00000000..0cb20754
--- /dev/null
+++ b/src/local_array/in_memory/mod.rs
@@ -0,0 +1,8 @@
+pub(crate) mod atomic_stride;
+pub(crate) mod atomic_types;
+pub(crate) mod node;
+mod oncebox;
+pub(crate) mod tree;
+
+#[macro_use]
+mod macros;
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
new file mode 100644
index 00000000..6f2c938a
--- /dev/null
+++ b/src/local_array/in_memory/node.rs
@@ -0,0 +1,1087 @@
+use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
+use std::{
+    fmt::Debug,
+    marker::PhantomData,
+};
+
+use log::trace;
+use parking_lot_core::SpinWait;
+
+use super::super::bit_span::BitSpan;
+use super::super::iterators::{SizedNodeMoreSpecificIter, SizedPrefixIter};
+use super::tree::{AtomicBitmap, AtomicStride2, AtomicStride3, AtomicStride4, AtomicStride5, CasResult, NewNodeOrIndex, SizedStrideNode, Stride, Stride3, Stride4, Stride5, StrideNodeId};
+
+// pub use crate::in_memory_tree::*;
+use crate::af::Zero;
+use crate::af::AddressFamily;
+use crate::local_array::types::PrefixId;
+
+//------------ TreeBitMap Node ----------------------------------------------
+
+// The treebitmap turned into a "trie-bitmap", really. A Node in the
+// treebitmap now only holds a ptrbitarr bitmap and a pfxbitarr bitmap, that
+// indicate whether a node or a prefix exists in that spot. The corresponding
+// node Ids and prefix ids are calculated from their position in the array.
+// Nodes do *NOT* have a clue where they are in the tree, so they don't know
+// the node id they represent. Instead, the node id is calculated from the
+// position in the tree. That's why several methods take a `base_prefix` as a
+// an argument: it represents the ID of the node itself.
+//
+// The elision of both the collection of children nodes and the prefix nodes
+// in a treebitmap node is enabled by the storage backend for the
+// multi-threaded store, since holds its entries keyed on the [node|prefix]
+// id. (in contrast with arrays or `vec`s, that have 
+pub struct TreeBitMapNode<
+    AF,
+    S,
+> where
+    Self: Sized,
+    S: Stride,
+    AF: AddressFamily,
+{
+    pub ptrbitarr: <S as Stride>::AtomicPtrSize,
+    pub pfxbitarr: <S as Stride>::AtomicPfxSize,
+    pub _af: PhantomData<AF>,
+}
+
+impl<AF, S> Debug
+    for TreeBitMapNode<AF, S>
+where
+    AF: AddressFamily,
+    S: Stride,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TreeBitMapNode")
+            .field("ptrbitarr", &self.ptrbitarr.load())
+            .field("pfxbitarr", &self.pfxbitarr.load())
+            .finish()
+    }
+}
+
+impl<AF, S>
+    std::fmt::Display for TreeBitMapNode<AF, S>
+where
+    AF: AddressFamily,
+    S: Stride
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "TreeBitMapNode {{ ptrbitarr: {:?}, pfxbitarr: {:?} }}",
+            self.ptrbitarr.load(),
+            self.pfxbitarr.load(),
+        )
+    }
+}
+
+impl<AF, S>
+    TreeBitMapNode<AF, S>
+where
+    AF: AddressFamily,
+    S: Stride
+{
+    pub(crate) fn new() -> Self {
+        TreeBitMapNode {
+            ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
+            pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
+            _af: PhantomData
+        }
+    }
+
+    // ------- Iterators ----------------------------------------------------
+
+    // Iterate over all the child node of this node
+    pub(crate) fn ptr_iter(&self, base_prefix: StrideNodeId<AF>) -> 
+        NodeChildIter<AF,S> {
+            NodeChildIter::<AF,S> {
+                base_prefix,
+                ptrbitarr: self.ptrbitarr.load(),
+                bit_span: BitSpan::new(0, 1),
+                _af: PhantomData,
+            }
+    }
+
+    // Iterate over all the prefix ids contained in this node.
+    // Note that this function is *not* used by the iterator that iterates
+    // over all prefixes. That one doesn't have to use the tree at all, but
+    // uses the store directly.
+    pub(crate) fn pfx_iter(&self, base_prefix: StrideNodeId<AF>) -> 
+        NodePrefixIter<AF, S> {
+        NodePrefixIter::<AF, S> {
+            pfxbitarr: self.pfxbitarr.load(),
+            base_prefix,
+            bit_span: BitSpan::new(0,1 ),
+            _af: PhantomData,
+            _s: PhantomData,
+        }
+    }
+
+    // Iterate over the more specific prefixes ids contained in this node
+    pub(crate) fn more_specific_pfx_iter(&self, base_prefix: StrideNodeId<AF>, 
+    start_bit_span: BitSpan, skip_self: bool) -> 
+        NodeMoreSpecificsPrefixIter<AF, S> {
+        NodeMoreSpecificsPrefixIter::<AF, S> {
+            pfxbitarr: self.pfxbitarr.load(),
+            base_prefix,
+            start_bit_span,
+            cursor: start_bit_span,
+            skip_self,
+            _s: PhantomData,
+        }
+    }
+
+    // Iterate over the nodes that contain more specifics for the requested
+    // base_prefix and corresponding bit_span.
+    pub(crate) fn more_specific_ptr_iter(&self, base_prefix: StrideNodeId<AF>, 
+    start_bit_span: BitSpan) -> 
+        NodeMoreSpecificChildIter<AF, S> {
+        NodeMoreSpecificChildIter::<AF, S> {
+            ptrbitarr: self.ptrbitarr.load(),
+            base_prefix,
+            start_bit_span,
+            cursor: None,
+        }
+    }
+
+
+    // ------- Search by Traversal methods -----------------------------------
+
+    // Inspects the stride (nibble, nibble_len) to see it there's already a 
+    // child node (if not at the last stride) or a prefix (if it's the last
+    // stride).
+    //
+    // Returns a tuple of which the first element is one of:
+    // - A newly created child node.
+    // - The index of the existing child node in the global `nodes` vec
+    // - A newly created Prefix
+    // - The index of the existing prefix in the global `prefixes` vec
+    // and the second element is the number of accumulated retries for the
+    // compare_exchange of both ptrbitarr and pfxbitarr.
+    pub(crate) fn eval_node_or_prefix_at(
+        &self,
+        nibble: u32,
+        nibble_len: u8,
+        // all the bits of the search prefix, but with the length set to
+        // the length of this stride. So bits are set beyond its length.
+        base_prefix: StrideNodeId<AF>,
+        stride_len: u8,
+        next_stride: Option<&u8>,
+        is_last_stride: bool,
+    ) -> (NewNodeOrIndex<AF>, u32) {
+
+        // THE CRITICAL SECTION
+        //
+        // UPDATING ptrbitarr & pfxbitarr
+        //
+        // This section is not as critical as creating/updating a
+        // a prefix. We need to set one bit only, and if somebody
+        // beat us to it that's fine, we'll figure that out when
+        // we try to write the prefix's serial number later on.
+        // The one thing that can go wrong here is that we are
+        // using an old ptrbitarr and overwrite bits set in the
+        // meantime elsewhere in the bitarray.
+        let mut retry_count = 0;
+        let ptrbitarr = self.ptrbitarr.load();
+        let pfxbitarr = self.pfxbitarr.load();
+        let bit_pos = S::get_bit_pos(nibble, nibble_len);
+        let new_node: SizedStrideNode<AF>;
+
+        // Check that we're not at the last stride (pfx.len <= stride_end),
+        // Note that next_stride may have a value, but we still don't want to
+        // continue, because we've exceeded the length of the prefix to
+        // be inserted.
+        // Also note that a nibble_len < S::BITS (a smaller than full nibble)
+        // does indeed indicate the last stride has been reached, but the
+        // reverse is *not* true, i.e. a full nibble can also be the last
+        // stride. Hence the `is_last_stride` argument
+        if !is_last_stride {
+
+            // We are not at the last stride
+            // Check it the ptr bit is already set in this position
+            if (S::into_stride_size(ptrbitarr) & bit_pos) == 
+                <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType>::zero() {
+                // Nope, set it and create a child node
+
+                match next_stride.unwrap() {
+                    3_u8 => {
+                        new_node = SizedStrideNode::Stride3(TreeBitMapNode {
+                            ptrbitarr: AtomicStride2(AtomicU8::new(0)),
+                            pfxbitarr: AtomicStride3(AtomicU16::new(0)),
+                            // pfx_vec: PrefixSet::empty(14),
+                            _af: PhantomData,
+                        });
+                    }
+                    4_u8 => {
+                        new_node = SizedStrideNode::Stride4(TreeBitMapNode {
+                            ptrbitarr: AtomicStride3(AtomicU16::new(0)),
+                            pfxbitarr: AtomicStride4(AtomicU32::new(0)),
+                            // pfx_vec: PrefixSet::empty(30),
+                            _af: PhantomData,
+                        });
+                    }
+                    5_u8 => {
+                        new_node = SizedStrideNode::Stride5(TreeBitMapNode {
+                            ptrbitarr: AtomicStride4(AtomicU32::new(0)),
+                            pfxbitarr: AtomicStride5(AtomicU64::new(0)),
+                            // pfx_vec: PrefixSet::empty(62),
+                            _af: PhantomData,
+                        });
+                    }
+                    _ => {
+                        panic!("can't happen");
+                    }
+                };
+
+                // THE CRITICAL SECTION
+                //
+                // UPDATING pfxbitarr
+                //
+                // preventing using an old ptrbitarr and overwrite bits set
+                // in the meantime elsewhere in the bitarray.
+                let mut a_ptrbitarr = self.ptrbitarr.compare_exchange(ptrbitarr,
+                    S::into_ptrbitarr_size(
+                    bit_pos | S::into_stride_size(ptrbitarr),
+                ));
+                let mut spinwait = SpinWait::new();
+                loop {
+                    match a_ptrbitarr {
+                        CasResult(Ok(_)) => {
+                            break;
+                        }
+                        CasResult(Err(newer_array)) => {
+                            // Someone beat us to it, so we need to use the
+                            // newer array.
+                            retry_count += 1;
+                            a_ptrbitarr = self.ptrbitarr.compare_exchange(newer_array,
+                                S::into_ptrbitarr_size(
+                                bit_pos | S::into_stride_size(newer_array),
+                            ));
+                        }
+                    };
+                    spinwait.spin_no_yield();
+                }
+
+                return (NewNodeOrIndex::NewNode(
+                    new_node
+                ), retry_count);
+            }
+        } else {
+            // only at the last stride do we create the bit in the prefix
+            // bitmap, and only if it doesn't exist already
+            if pfxbitarr & bit_pos
+                == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType as std::ops::BitAnd>::Output::zero()
+            {
+
+                // THE CRITICAL SECTION
+                //
+                // UPDATING pfxbitarr
+                //
+                // preventing using an old pfxbitarr and overwrite bits set
+                // in the meantime elsewhere in the bitarray.
+                let mut a_pfxbitarr = 
+                self.pfxbitarr.compare_exchange(
+                    pfxbitarr, bit_pos | pfxbitarr
+                );
+                let mut spinwait = SpinWait::new();
+
+                loop {
+                    match a_pfxbitarr {
+                        CasResult(Ok(_)) => {
+                            break;
+                        }
+                        CasResult(Err(newer_array)) => {
+                            // Someone beat us to it, so we need to use the
+                            // newer array.
+                            retry_count += 1;
+                            a_pfxbitarr = self.pfxbitarr.compare_exchange(
+                                newer_array, bit_pos | newer_array
+                            );
+                        }
+                    };
+                    spinwait.spin_no_yield();
+                }
+
+                return (NewNodeOrIndex::NewPrefix, retry_count);
+            }
+            return (NewNodeOrIndex::ExistingPrefix, retry_count);
+        }
+
+        // Nodes always live at the last length of a stride (i.e. the last 
+        // nibble), so we add the stride length to the length of the
+        // base_prefix (which is always the start length of the stride).
+        (NewNodeOrIndex::ExistingNode(
+            base_prefix.add_to_len(stride_len).truncate_to_len()
+        ), retry_count)
+    }
+
+    //-------- Search nibble functions --------------------------------------
+
+    // This function looks for the longest marching prefix in the provided
+    // nibble, by iterating over all the bits in it and comparing that with
+    // the appropriate bytes from the requested prefix. It mutates the 
+    // `less_specifics_vec` that was passed in to hold all the prefixes found
+    // along the way.
+    pub(crate) fn search_stride_for_longest_match_at(
+        &self,
+        search_pfx: PrefixId<AF>,
+        mut nibble: u32,
+        nibble_len: u8,
+        start_bit: u8,
+        less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
+    ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
+        let pfxbitarr = self.pfxbitarr.load();
+        let ptrbitarr = self.ptrbitarr.load();
+        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
+        let mut found_pfx = None;
+
+        trace!("start longest_match search");
+        for n_l in 1..(nibble_len + 1) {
+            // Move the bit in the right position.
+            nibble =
+                AddressFamily::get_nibble(search_pfx.get_net(), start_bit, n_l);
+            bit_pos = S::get_bit_pos(nibble, n_l);
+
+            // Check if the prefix has been set, if so select the prefix.
+            // This is not necessarily the final prefix that will be
+            // returned.
+
+            // Check it there's a prefix matching in this bitmap for this 
+            // nibble.
+            trace!("pfxbitarr {:032b}", pfxbitarr);
+
+            if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
+                let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + n_l), start_bit + n_l);
+                // f_pfx.set_serial(self.get_pfx_serial(f_pfx, nibble, n_l, guard).load(Ordering::Relaxed));
+
+                // Receiving a less_specifics_vec means that the user wants
+                // to have all the last-specific prefixes returned, so add
+                // the found prefix.
+                trace!("gather pfx in less_specifics {:?}", f_pfx);
+                trace!("ls_vec {:?}", less_specifics_vec);
+                if let Some(ls_vec) = less_specifics_vec {
+                    trace!("len {}", search_pfx.get_len());
+                    trace!("start_bit {}", start_bit);
+                    trace!("n_l {}", n_l);
+                    trace!("smaller length? {}", search_pfx.get_len() > start_bit + n_l);
+                    trace!("{}", (S::into_stride_size(ptrbitarr)
+                            & bit_pos)
+                            == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero());
+                    if search_pfx.get_len() > start_bit + n_l
+                        && (S::into_stride_size(ptrbitarr)
+                            & bit_pos)
+                            == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+                    {
+                        ls_vec.push(f_pfx);
+                    }
+                }
+
+                found_pfx = Some(f_pfx);
+            }
+        }
+
+        let base_prefix =
+            StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit);
+
+        // Check if this the last stride, or if they're no more children to
+        // go to, if so return what we found up until now.
+        if search_pfx.get_len() <= start_bit + nibble_len
+            || (S::into_stride_size(ptrbitarr) & bit_pos) == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+        // No children or at the end, return the definitive LMP we found.
+        {
+            return (
+                None,      /* no more children */
+                found_pfx, /* The definitive LMP if any */
+            );
+        }
+
+        // There's another child, return it together with the preliminary LMP
+        // we found.
+        (
+            // The identifier of the node that has children of the next
+            // stride.
+            Some(base_prefix.add_nibble(nibble, nibble_len)),
+            found_pfx,
+        )
+    }
+
+    // This function looks for the exactly matching prefix in the provided
+    // nibble. It doesn't need to iterate over anything it just compares
+    // the complete nibble, with the appropriate bits in the requested
+    // prefix. Although this is rather efficient, there's no way to collect
+    // less-specific prefixes from the search prefix.
+    pub(crate) fn search_stride_for_exact_match_at(
+        &'_ self,
+        search_pfx: PrefixId<AF>,
+        nibble: u32,
+        nibble_len: u8,
+        start_bit: u8,
+        _: &mut Option<Vec<PrefixId<AF>>>,
+    ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
+        let pfxbitarr = self.pfxbitarr.load();
+        let ptrbitarr = self.ptrbitarr.load();
+        // This is an exact match, so we're only considering the position of
+        // the full nibble.
+        let bit_pos = S::get_bit_pos(nibble, nibble_len);
+        let mut found_pfx = None;
+        let mut found_child = None;
+
+        // Is this the last nibble?
+        // Otherwise we're not looking for a prefix (exact matching only
+        // lives at last nibble)
+        match search_pfx.get_len() <= start_bit + nibble_len {
+            // We're at the last nibble.
+            true => {
+                // Check for an actual prefix at the right position, i.e. 
+                // consider the complete nibble.
+                if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
+                    let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + nibble_len), start_bit + nibble_len);
+                    found_pfx = Some(f_pfx);
+                }
+            }
+            // We're not at the last nibble.
+            false => {
+                // Check for a child node at the right position, i.e.
+                // consider the complete nibble.
+                if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+                {
+                    found_child = Some(
+                        StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)
+                    );
+                }
+            }
+        }
+
+        (
+            found_child, /* The node that has children in the next stride, if
+                            any */
+            found_pfx,   /* The exactly matching prefix, if any */
+        )
+    }
+
+    // This function looks for the exactly matching prefix in the provided
+    // nibble, just like the one above, but this *does* iterate over all the
+    // bytes in the nibble to collect the less-specific prefixes of the the 
+    // search prefix. This is of course slower, so it should only be used 
+    // when the user explicitly requests less-specifics.
+    pub(crate) fn search_stride_for_exact_match_with_less_specifics_at(
+        &self,
+        search_pfx: PrefixId<AF>,
+        mut nibble: u32,
+        nibble_len: u8,
+        start_bit: u8,
+        less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
+    ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
+        let pfxbitarr = self.pfxbitarr.load();
+        let ptrbitarr = self.ptrbitarr.load();
+        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
+        let mut found_pfx = None;
+
+        let ls_vec = less_specifics_vec
+            .as_mut()
+            .expect(concat!("You shouldn't call this function without",
+            "a `less_specifics_vec` buffer. Supply one when calling this function",
+            "or use `search_stride_for_exact_match_at`"));
+
+        for n_l in 1..(nibble_len + 1) {
+            // Move the bit in the right position.
+            nibble =
+                AddressFamily::get_nibble(search_pfx.get_net(), start_bit, n_l);
+            bit_pos = S::get_bit_pos(nibble, n_l);
+
+            // Check if the prefix has been set, if so select the prefix.
+            // This is not necessarily the final prefix that will be
+            // returned.
+
+            // Check it there's a prefix matching in this bitmap for this 
+            // nibble.
+            if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
+                // since we want an exact match only, we will fill the prefix
+                // field only if we're exactly at the last bit of the nibble
+                if n_l == nibble_len {
+                    let f_pfx =
+                        PrefixId::new(
+                            search_pfx.get_net().truncate_to_len(start_bit + n_l), start_bit + n_l);
+                        found_pfx = Some(f_pfx);
+                }
+
+                // Receiving a less_specifics_vec means that the user wants to
+                // have all the last-specific prefixes returned, so add the
+                // found prefix.
+                let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + n_l), start_bit + n_l);
+                ls_vec.push(f_pfx);
+            }
+        }
+
+        if found_pfx.is_none() {
+            // no prefix here, clear out all of the prefixes we found along
+            // the way, since it doesn't make sense to return less-specifics
+            // if we don't have a exact match.
+            ls_vec.clear();
+        }
+
+        // Check if this the last stride, or if they're no more children to 
+        // go to, if so return what we found up until now.
+        match search_pfx.get_len() <= start_bit + nibble_len
+            || (S::into_stride_size(ptrbitarr) & bit_pos)
+                == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType as std::ops::BitAnd>::Output::zero()
+        {
+            // No children or at the end, return the definitive LMP we found.
+            true => (
+                None,      /* no more children */
+                found_pfx, /* The definitive LMP if any */
+            ),
+            // There's another child, we won't return the found_pfx, since
+            // we're not at the last nibble and we want an exact match only.
+            false => (
+                Some(StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)),
+                None,
+            ),
+        }
+    }
+
+    // Search a stride for more-specific prefixes and child nodes containing
+    // more specifics for `search_prefix`.
+    pub(crate) fn add_more_specifics_at(
+        &self,
+        nibble: u32,
+        nibble_len: u8,
+        base_prefix: StrideNodeId<AF>,
+    ) -> (
+        Vec<StrideNodeId<AF>>, /* child nodes with more more-specifics in
+                                  this stride */
+        Vec<PrefixId<AF>>,     /* more-specific prefixes in this stride */
+    ) {
+        trace!("start adding more specifics");
+        let pfxbitarr = self.pfxbitarr.load();
+        let ptrbitarr = self.ptrbitarr.load();
+        trace!("ptrbitarr {:032b}", ptrbitarr);
+        trace!("pfxbitarr {:032b}", pfxbitarr);
+        let mut found_children_with_more_specifics = vec![];
+        let mut found_more_specifics_vec: Vec<PrefixId<AF>> = vec![];
+
+        // This is an exact match, so we're only considering the position of
+        // the full nibble.
+        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
+        let mut found_child = None;
+
+        // Is there also a child node here?
+        // Note that even without a child node, there may be more specifics 
+        // further up in this pfxbitarr or children in this ptrbitarr.
+        if (S::into_stride_size(ptrbitarr) & bit_pos)
+            > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
+            )
+        {
+            found_child = Some(base_prefix.add_nibble(nibble, nibble_len));
+        }
+
+        if let Some(child) = found_child {
+            found_children_with_more_specifics.push(child);
+        }
+
+        // We're expanding the search for more-specifics bit-by-bit.
+        // `ms_nibble_len` is the number of bits including the original
+        // nibble we're considering, e.g. if our prefix has a length of 25
+        // and we've all strides sized 4, we would end up with a last
+        // nibble_len of 1. `ms_nibble_len` will expand then from 2 up and
+        // till 4.
+        // 
+        // ex.:
+        // nibble: 1 , (nibble_len: 1)
+        // Iteration:
+        // ms_nibble_len=1,n_l=0: 10, n_l=1: 11
+        // ms_nibble_len=2,n_l=0: 100, n_l=1: 101, n_l=2: 110, n_l=3: 111
+        // ms_nibble_len=3,n_l=0: 1000, n_l=1: 1001, n_l=2: 1010, ..., 
+        // n_l=7: 1111
+
+        for ms_nibble_len in nibble_len + 1..=S::STRIDE_LEN {
+            // iterate over all the possible values for this `ms_nibble_len`,
+            // e.g. two bits can have 4 different values.
+            for n_l in 0..(1 << (ms_nibble_len - nibble_len)) {
+                // move the nibble left with the amount of bits we're going
+                // to loop over. e.g. a stride of size 4 with a nibble 0000
+                // 0000 0000 0011 becomes 0000 0000 0000 1100, then it will
+                // iterate over ...1100,...1101,...1110,...1111
+                let ms_nibble =
+                    (nibble << (ms_nibble_len - nibble_len)) + n_l as u32;
+                bit_pos = S::get_bit_pos(ms_nibble, ms_nibble_len);
+
+                if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+                {
+                    found_children_with_more_specifics.push(
+                    base_prefix.add_nibble(ms_nibble, ms_nibble_len)
+                    );
+                }
+
+                if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
+                    found_more_specifics_vec.push(
+                        base_prefix.add_nibble(ms_nibble, ms_nibble_len).into()                    )
+                }
+            }
+        }
+
+        trace!("found_children_with_more_specifics {:?}", found_children_with_more_specifics);
+        trace!("found_more_specifics_vec {:?}", found_more_specifics_vec);
+
+        (
+            // We're done here, the caller should now go over all nodes in
+            // found_children_with_more_specifics vec and add ALL prefixes
+            // found in there.
+            found_children_with_more_specifics,
+            found_more_specifics_vec,
+        )
+    }
+}
+
+
+// ------------ Iterator methods --------------------------------------------
+
+// ----------- NodeChildIter ------------------------------------------------
+
+// create an iterator over all child nodes id
+//
+// we don't have a collection of local nodes anymore, since the id of the
+// node are deterministically generated, as the prefix+len they represent
+// in the treebitmap. This has both the advantage of using less memory,
+// and being easier to use in a concurrently updated tree. The
+// disadvantage is that we have to look up the child nodes on the fly
+// when we want to iterate over all children of a node.
+//
+// ptr child nodes only exist at the last nibble of the stride size
+// (`child_len`). Since children  in the first nibbles are leaf nodes.
+// leaf nodes will only be prefixes. So if we have a first stride of
+// size 5, all ptr nodes wil have StrideNodeIds with len = 5.
+//
+// Ex.:
+//
+// Stride no.          1       2       3      4       5       6       7       
+// StrideSize          5       5       4      3       3       3       3      
+// child pfxs len      /1-5   /5-10    /10-14 /15-17  /18-20  /21-23  /24-26
+// child Nodes len     /5      /10     /14    /17     /20     /23     /26 
+//
+// Stride no.          8      9
+// StrideSize          3      3
+// child pfxs len      /27-29 /30-32
+// child Nodes len     /29    /32
+
+#[derive(Debug, Copy, Clone)]
+pub(crate) struct NodeChildIter<AF: AddressFamily, S: Stride> {
+   base_prefix: StrideNodeId<AF>,
+   ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
+   bit_span: BitSpan, // start with 0
+   _af: PhantomData<AF>,
+}
+
+impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
+    NodeChildIter<AF, S>
+{
+    type Item = StrideNodeId<AF>;
+    fn next(&mut self) -> Option<Self::Item> {
+        // iterate over all the possible values for this stride length, e.g.
+        // two bits can have 4 different values.
+        for cursor in self.bit_span.bits..(1 << S::STRIDE_LEN) {
+            // move the bit_span left with the amount of bits we're going to
+            // loop over.
+            // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
+            // becomes 0000 0000 0000 1100, then it will iterate over 
+            // ...1100,...1101,...1110,...1111
+            let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
+            if (S::into_stride_size(self.ptrbitarr) & bit_pos) >
+                <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+            {
+                self.bit_span.bits = cursor + 1;
+                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
+            }    
+            
+        }
+        None
+    }
+}
+
+// ----------- NodeMoreSpecificChildIter ------------------------------------
+
+// Create an iterator over all the child nodes that hold a more specific
+// prefixes of the specified start_bit_span. This basically the same Iterator
+// as the ChildNodeIter, except that it stops (potentially) earlier, to avoid
+// including nodes with adjacent prefixes. Starting an iterator with a 
+// `start_bit_span` of { bits: 0, len: 0 } will return all child nodes of 
+// this node. In that case you could also use the `NodeChildIter` instead.
+//
+// inputs
+//
+// `base_prefix`
+// This iterator take a `base_prefix` since the nodes themselves have no
+// knowledge of their own prefixes, those are inferred by their position in
+// the tree (therefore, it's actually a Trie). Note that `base_prefix` +
+// `bit_span` define the actual starting prefix for this iterator.
+//
+// `ptrbitarr` 
+// is the bitmap that holds the slots that have child nodes. 
+//
+// `start_bit_span` 
+// is the bit span that is going to be used as a starting point for the 
+// iterator.
+// 
+// `cursor`
+// holds the current cursor offset from the start_bit_span.bits, the sum of
+// these describe the current position in the bitmap. Used for re-entry into
+// the iterator. A new iterator should start with None.
+//
+// How this works
+//
+// The iterator starts at the start_bit_span.bits position in the bitmap and
+// advances until it reaches either a one in the bitmap, or the maximum
+// position for the particular more-specifics for this bit_span.
+//
+// e.x.
+// The stride size is 5 and the starting bit span is {bits: 2, len: 4} (0010)
+// The starting point is therefore the bit_array 0010. The iterator will go
+// over 0010 0 and 0010 1. The next bits to consider would be 0011 0 which
+// would not fit our starting bit_span of 0010. So we have to stop after 2
+// iterations. This means that the number of iterations is determined by the
+// difference between the number of bits in the stride size (5) and the the
+// number of bits in the start_bit_span (4). The number of iterations in the
+// above example is therefore 1 << (5 - 4) = 2. Remember that a ptrbitarr
+// holds only one stride size (the largest for its stride size), so we're
+// done now.
+#[derive(Debug, Copy, Clone)]
+pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily, S: Stride> {
+   base_prefix: StrideNodeId<AF>,
+   ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
+   start_bit_span: BitSpan,
+   cursor: Option<u32>,
+}
+
+impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
+    NodeMoreSpecificChildIter<AF, S>
+{
+    type Item = StrideNodeId<AF>;
+    fn next(&mut self) -> Option<Self::Item> {
+        // Early exits
+
+        // Empty bitmap
+        if self.ptrbitarr == <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType::zero() {
+            trace!("empty ptrbitrarr. this iterator is done.");
+            return None;
+        }
+
+        // Previous iteration incremented the cursor beyond the stride size.
+        if let Some(cursor) = self.cursor { 
+            if cursor >= (1 << (S::STRIDE_LEN - self.start_bit_span.len)) {
+                trace!("cursor.bits >= (1 << (S::STRIDE_LEN - self.start_bit_span.len))");
+                trace!("cursor: {}", cursor);
+                trace!("start_bit_span: {} {}", self.start_bit_span.bits, self.start_bit_span.len);
+                return None;
+            }
+        }
+
+        // No early exits, we're in business.
+        trace!("NodeMoreSpecificChildIter");
+        trace!("base_prefix {}", self.base_prefix);
+        trace!("stride_size {}", S::STRIDE_LEN);
+        trace!("start_bit_span bits {} len {} ", self.start_bit_span.bits, self.start_bit_span.len);
+        trace!("cursor bits {:?}", self.cursor);
+        trace!("          x1  4   8  12  16  20  24  28  32");
+        trace!("ptrbitarr {:032b}", self.ptrbitarr);
+
+        let start = if let Some(bits) = self.cursor { bits } else { self.start_bit_span.bits };
+        // We either stop if we have reached the maximum number of bits that
+        // we should check for this bit_span or we stop at the end of the
+        // stride (in case of a start_bit_span.bits == 0).
+        let stop = <u32>::min((1 << (S::STRIDE_LEN - self.start_bit_span.len)) + start, (1 << S::STRIDE_LEN) - 1); 
+
+        trace!("start {:?} stop {}", start, stop);
+        for cursor in start..=stop {
+            // move the bit_span left with the amount of bits we're going to loop over.
+            // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
+            // becomes 0000 0000 0000 1100, then it will iterate over 
+            // ...1100,...1101,...1110,...1111
+            let bit_pos = 
+                S::get_bit_pos(
+                    cursor, S::STRIDE_LEN);
+            trace!("cmpbitarr x{:032b} {}", bit_pos, stop - start);
+            if (S::into_stride_size(self.ptrbitarr) & bit_pos) >
+                <<S as Stride>::AtomicPfxSize as AtomicBitmap
+                >::InnerType::zero()
+            {
+                trace!("bingo!");
+                self.cursor = Some(cursor + 1);
+
+                trace!("next bit_span {} {} with cursor {:?}", self.start_bit_span.bits, self.start_bit_span.len, self.cursor);
+                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
+            }
+        }
+        trace!("No more nodes. End of the iterator.");
+        None
+    }
+}
+
+impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride3> {
+    pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
+        SizedNodeMoreSpecificIter::<AF>::Stride3(self)
+    }
+}
+
+impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride4> {
+    pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
+        SizedNodeMoreSpecificIter::<AF>::Stride4(self)
+    }
+}
+
+impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride5> {
+    pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
+        SizedNodeMoreSpecificIter::<AF>::Stride5(self)
+    }
+}
+
+
+// ----------- NodePrefixIter -----------------------------------------------
+
+// Create an iterator of all prefix ids hosted by this node.
+
+// Partition for stride 3
+//
+// pfxbitarr (AF::BITS)  0 1 2  3  4  5  6   7   8   9  10  11  12  13  14 
+// bit_span (binary)     * 0 1 00 01 10 11 000 001 010 011 100 101 110 111 * *
+// bit_span (dec.)       * 0 1  0  1  2  3   0   1   2   3   4   5   6   7 * *
+// len                   0 1   2           3                               
+//
+// pfxbitarr (example)   1 0 0 0  0  0  1  1   1   0   0   0   0   0   0   0 0
+// pos (example)         0 0 0 0  0  0  0  0   1   0   0   0   0   0   0   0 0                 
+//
+// Ex.:
+// `pos` describes the bit that is currently under consideration. 
+// 
+// `pfxbitarr` is the bitmap that contains the prefixes. Every 1 in the
+// bitmap means that the prefix is hosted by this node. Moreover, the
+// position in the bitmap describes the address part of the prefix, given
+// a `base prefix`. The described prefix is the bits of the `base_prefix`
+// bitmap appended by the `bit span` bits.
+//
+// The length of the prefix is
+// described by sum of the length of the base_prefix and the `len`
+// variable.
+//
+// The `bit_span` variable starts counting at every new prefix length.
+pub(crate) struct NodePrefixIter<AF: AddressFamily, S: Stride> {
+    base_prefix: StrideNodeId<AF>,
+    pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
+    bit_span: BitSpan, // start with 0
+    _af: PhantomData<AF>,
+    _s: PhantomData<S>,
+}
+
+impl<AF: AddressFamily, S: Stride> std::iter::Iterator for 
+    NodePrefixIter<AF, S> {
+        type Item = PrefixId<AF>;
+
+        fn next(&mut self) -> Option<Self::Item> {
+        // iterate over all the possible values for this stride length, e.g.
+        // two bits can have 4 different values.
+        for cursor in self.bit_span.bits..(1 << S::STRIDE_LEN) {
+            
+            let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
+            if self.pfxbitarr & bit_pos >
+                <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+            {
+                self.bit_span.bits = cursor + 1;
+                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN).into());
+            }    
+            
+        }
+        None
+    }
+}
+
+// Creates an Iterator that returns all prefixes that exist in a node that
+// are a more-specific prefix of the `base_prefix` + `start_bit_span`.
+//
+// Inputs
+// 
+// `base_prefix`
+// This iterator take a `base_prefix` since the nodes themselves have no
+// knowledge of their own prefixes, those are inferred by their position in
+// the tree (therefore, it's actually a Trie). Note that `base_prefix` +
+// `bit_span` define the actual starting prefix for this iterator.
+//
+// `pfxbitarr` 
+// is the bitmap that holds the slots that have prefixes.
+//
+// `start_bit_span` 
+// is the bit span that is going to be used as a starting point for the 
+// iterator.
+// 
+// `cursor`
+// holds the current cursor offset from the start_bit_span.bits, the sum of
+// these describe the current position in the bitmap. Used for re-entry into
+// the iterator. A new iterator should start with None.
+//
+// How this works
+//
+// The iterator starts at the start_bit_span.bits position in the bitmap and
+// advances until it reaches either a one in the bitmap, or the maximum
+// position for the particular more-specifics for this bit_span. When it
+// reaches the maximum position it determines whether there are more stride-
+// sizes available in this bitmap. If there are, it advances to the next
+// stride-size in the first position. If not it terminates the iterator.
+//
+// e.x.
+// The stride size is 5 and the starting bit span is {bits: 1, len: 3} (001)
+// This means that the stride size that we have to consider are 4 and 5. 3 
+// being the size of the current bit_span and 5 being the size of the total
+// stride. 
+// The starting point is therefore the bit_array 001. The iterator will go
+// over 001 00, 001 01, 001 10 and 001 11. The next bits to consider would be
+//  010 00 which would not fit our starting bit_span of 0010. So we have to
+// stop after 2 iterations. This means that the number of iterations is
+// determined by the difference between the number of bits in the stride size
+// (5) and the the number of bits in the start_bit_span (4). The number of 
+// iterations in the above example is therefore 1 << (5 - 3) = 4.
+// Unlike the MoreSpecificPrefixIter, we will have to consider more lengths
+// than just the bit_span len. We will have to jump a few pfxbitarr bits and
+// move to the next stride size in the bitmap, starting at bit_array 0010, or
+// the bit_span { bits: 2, len: 3 }, a.k.a. 0010 << 1. But now we will have
+// to go over a different amount of 1 << (5 - 4) = 2 iterations to reap the
+// next bit_spans of 0010 0 and 0010 1.
+
+pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily, S: Stride> {
+    // immutables
+    base_prefix: StrideNodeId<AF>,
+    pfxbitarr: <<S as super::atomic_stride::Stride>::AtomicPfxSize 
+        as super::atomic_stride::AtomicBitmap>::InnerType,
+    // we need to keep around only the `bits` part of the `bit_span`
+    // technically, (it needs resetting the current state to it after each
+    // prefix-length), but we'll keep the start-length as well for clarity
+    // and increment it on a different field ('cur_len').
+    start_bit_span: BitSpan,
+    cursor: BitSpan,
+    skip_self: bool,
+    _s: PhantomData<S>,
+}
+
+impl<AF: AddressFamily, S: Stride> std::iter::Iterator for 
+    NodeMoreSpecificsPrefixIter<AF, S> {
+        type Item = PrefixId<AF>;
+
+        fn next(&mut self) -> Option<Self::Item> {
+
+            // Easy early exit conditions
+
+            // Empty bitmap
+            if self.pfxbitarr == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
+                trace!("empty pfxbitarr. This iterator is done.");
+                return None;
+            }
+
+            // No early exits, We're in business.
+            trace!("len_offset {}", ((1<< self.cursor.len) - 1));
+            trace!("start_bit {}", self.start_bit_span.bits);
+            trace!("number of check bits in len {}", 
+                (1 << (self.cursor.len - self.start_bit_span.len)));
+
+            trace!("next more specifics prefix iter start bits {} len {}",
+                self.start_bit_span.bits, self.start_bit_span.len);
+
+            let mut res = None;
+
+            // Move to the next len if we're at the first prefix-length that matches,
+            // if `skip_self` is set. Typically this flag is set for the first stride.
+            // In the consecutive strides, we don't want to skip the base prefix, since
+            // that base_prefix is a more specific prefix of the one requested in the
+            // first stride.
+            if self.skip_self && (1 << (self.cursor.len - self.start_bit_span.len)) == 1 {
+                // self.cursor.len += (self.start_bit_span.bits & 1) as u8;
+                trace!("skipping self");
+                self.start_bit_span.bits <<= 1;
+                self.cursor.bits = self.start_bit_span.bits;
+                self.cursor.len += 1;
+                self.skip_self = false;
+                trace!("new start bits {} len {}", self.start_bit_span.bits, self.start_bit_span.len);
+                trace!("new cursor bits {} len {}", self.cursor.bits, self.cursor.len);
+            }
+
+            // Previous iteration or the skip_self routine may have
+            // incremented the cursor beyond the end of the stride size.
+            if self.cursor.len > S::STRIDE_LEN {
+                trace!("cursor.len > S::STRIDE_LEN. This iterator is done.");
+                return None;
+            }
+            
+            loop {
+                trace!("          x1  4   8  12  16  20  24  28  32  36  40  44  48  52  56  60  64");
+                trace!("cmpnibble {:064b} ({} + {}) len {} stride_size {}", 
+                    S::get_bit_pos(self.cursor.bits, self.cursor.len), 
+                    (1<< self.cursor.len) - 1, 
+                    self.cursor.bits, 
+                    self.cursor.len + self.base_prefix.get_len(),
+                    S::STRIDE_LEN
+                );
+
+                trace!("pfxbitarr {:064b}", self.pfxbitarr);
+
+                if (S::get_bit_pos(self.cursor.bits, self.cursor.len) | self.pfxbitarr) == self.pfxbitarr {
+                    trace!("found prefix with len {} at pos {} pfx len {}",
+                        self.cursor.len, 
+                        self.cursor.bits,
+                        self.base_prefix.get_len() + self.cursor.len,
+                    );
+                    res = Some(self.base_prefix
+                        .add_nibble(self.cursor.bits, self.cursor.len).into());
+                    trace!("found prefix {:?}", res);
+                }
+
+                // Determine when we're at the end of the bits applicable to
+                // this combo of this start_bit_span.
+                // bitspan offset: 
+                // self.start_bit_span.bits
+                // number of matches in this length:
+                // 1 << (self.cursor.len - self.start_bit_span.len)
+                let max_pos_offset =  
+                    self.start_bit_span.bits + 
+                    (1 << (self.cursor.len - self.start_bit_span.len)) - 1;
+
+                trace!("max_pos_offset {} > cursor bit_pos {}?", max_pos_offset, self.cursor.bits);
+                trace!("number of check bits in len {}", (1 << (self.cursor.len - self.start_bit_span.len)));
+
+                // case 1. At the beginning or inside a prefix-length.
+                if max_pos_offset > self.cursor.bits {
+                    self.cursor.bits += 1;
+                } 
+                // case 2. At the end of a prefix-lengths in this stride.
+                else if self.cursor.len < S::STRIDE_LEN {
+                    trace!("move len to {}", self.cursor.len + 1);
+                    self.start_bit_span.bits <<= 1;
+                    self.cursor.bits = self.start_bit_span.bits;
+                    self.cursor.len += 1;
+                }
+                // case 4. At the end of a prefix-length, but not at the end of the pfxbitarr.
+                else {
+                    self.start_bit_span.bits <<= 1;
+                    self.cursor.bits = self.start_bit_span.bits;
+                    self.cursor.len += 1;
+                    trace!("end of stride, next cursor bits {} len {}", self.cursor.bits, self.cursor.len);
+                    return res;
+                }
+
+                trace!("some res {:?}", res);
+                if res.is_some() { return res; }
+            }
+        }    
+}
+
+impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride3> {
+    pub fn wrap(self) -> SizedPrefixIter<AF> {
+        SizedPrefixIter::<AF>::Stride3(self)
+    }
+}
+
+impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride4> {
+    pub fn wrap(self) -> SizedPrefixIter<AF> {
+        SizedPrefixIter::<AF>::Stride4(self)
+    }
+}
+
+impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride5> {
+    pub fn wrap(self) -> SizedPrefixIter<AF> {
+        SizedPrefixIter::Stride5(self)
+    }
+}
diff --git a/src/local_array/in_memory/oncebox.rs b/src/local_array/in_memory/oncebox.rs
new file mode 100644
index 00000000..9752ba5f
--- /dev/null
+++ b/src/local_array/in_memory/oncebox.rs
@@ -0,0 +1,158 @@
+use std::ptr::null_mut;
+use std::slice;
+use std::sync::atomic::{AtomicPtr, Ordering};
+
+#[derive(Debug, Default)]
+pub struct OnceBox<T> {
+    ptr: AtomicPtr<T>,
+}
+
+impl<T> OnceBox<T> {
+    pub fn new() -> Self {
+        Self {
+            ptr: AtomicPtr::new(null_mut()),
+        }
+    }
+
+    pub fn get(&self) -> Option<&T> {
+        let ptr = self.ptr.load(Ordering::Relaxed);
+        if ptr.is_null() {
+            None
+        } else {
+            Some(unsafe { &*ptr })
+        }
+    }
+
+    pub fn get_or_init(&self, create: impl FnOnce() -> T) -> (&T, bool) {
+        let mut its_us = false;
+        if let Some(res) = self.get() {
+            return (res, its_us);
+        }
+        let ptr = Box::leak(Box::new(create()));
+        let res = match self.ptr.compare_exchange(
+            null_mut(),
+            ptr,
+            Ordering::Acquire,
+            Ordering::Relaxed,
+        ) {
+            Ok(current) => {
+                // We set the new value, return it.
+                assert!(current.is_null());
+                its_us = true;
+                ptr as *const _
+            }
+            Err(current) => {
+                // `current` is the real value we need to drop our value.
+                assert!(!current.is_null());
+                let _ = unsafe { Box::from_raw(ptr) };
+                current as *const _
+            }
+        };
+        (unsafe { &*res }, its_us)
+    }
+}
+
+impl<T> Drop for OnceBox<T> {
+    fn drop(&mut self) {
+        let ptr = self.ptr.swap(null_mut(), Ordering::Relaxed);
+        if !ptr.is_null() {
+            let _ = unsafe { Box::from_raw(ptr) };
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct OnceBoxSlice<T> {
+    ptr: AtomicPtr<OnceBox<T>>,
+    p2_size: u8,
+}
+
+impl<T> OnceBoxSlice<T> {
+    pub fn new(p2_size: u8) -> Self {
+        Self {
+            ptr: AtomicPtr::new(null_mut()),
+            p2_size,
+        }
+    }
+
+    pub fn is_null(&self) -> bool {
+        self.ptr.load(Ordering::Relaxed).is_null()
+    }
+
+    pub fn get(&self, idx: usize) -> Option<&T> {
+        let ptr = self.ptr.load(Ordering::Relaxed);
+        if ptr.is_null() {
+            None
+        } else {
+            let slice =
+                unsafe { slice::from_raw_parts(ptr, 1 << self.p2_size) };
+            slice.get(idx).and_then(|inner| inner.get())
+        }
+    }
+
+    pub fn get_or_init(
+        &self,
+        idx: usize,
+        create: impl FnOnce() -> T,
+    ) -> (&T, bool) {
+        // assert!(idx < (1 << self.p2_size));
+        let slice = self.get_or_make_slice();
+        slice[idx].get_or_init(create)
+    }
+
+    fn get_or_make_slice(&self) -> &[OnceBox<T>] {
+        let ptr = self.ptr.load(Ordering::Relaxed);
+        if !ptr.is_null() {
+            return unsafe { slice::from_raw_parts(ptr, 1 << self.p2_size) };
+        }
+
+        // Create a slice, set it, get again.
+        let mut vec = Vec::with_capacity(1 << self.p2_size);
+        for _ in 0..(1 << self.p2_size) {
+            vec.push(OnceBox::new())
+        }
+        // Convert Vec<[OnceBox<T>] -> Box<[OnceBox<T>] -> &mut [OnceBox<T>]
+        //  -> *mut OnceBox<T>
+        let ptr = Box::leak(vec.into_boxed_slice()).as_mut_ptr();
+        let res = match self.ptr.compare_exchange(
+            null_mut(),
+            ptr,
+            Ordering::Acquire,
+            Ordering::Relaxed,
+        ) {
+            Ok(current) => {
+                // We set the new value, return it.
+                assert!(current.is_null());
+                ptr
+            }
+            Err(current) => {
+                // There was a value already: current. Drop our new thing and
+                // return current.
+                assert!(!current.is_null());
+                let _ = unsafe {
+                    Box::from_raw(slice::from_raw_parts_mut(
+                        ptr,
+                        1 << self.p2_size,
+                    ))
+                };
+                current
+            }
+        };
+
+        unsafe { slice::from_raw_parts(res, 1 << self.p2_size) }
+    }
+}
+
+impl<T> Drop for OnceBoxSlice<T> {
+    fn drop(&mut self) {
+        let ptr = self.ptr.swap(null_mut(), Ordering::Relaxed);
+        if !ptr.is_null() {
+            let _ = unsafe {
+                Box::from_raw(slice::from_raw_parts_mut(
+                    ptr,
+                    1 << self.p2_size,
+                ))
+            };
+        }
+    }
+}
diff --git a/src/local_array/tree.rs b/src/local_array/in_memory/tree.rs
similarity index 78%
rename from src/local_array/tree.rs
rename to src/local_array/in_memory/tree.rs
index b42105d1..1acb7562 100644
--- a/src/local_array/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -1,20 +1,24 @@
 use crate::prefix_record::{Meta, PublicRecord};
-use crossbeam_epoch::{self as epoch};
+use crate::prelude::multi::PrefixId;
+use crossbeam_epoch::{self as epoch, Atomic, Guard, Owned};
 use log::{error, log_enabled, trace};
+use roaring::RoaringBitmap;
 
 use std::hash::Hash;
-use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
+use std::sync::atomic::{
+    AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering,
+};
 use std::{fmt::Debug, marker::PhantomData};
 
+use super::atomic_types::{NodeBuckets, PrefixBuckets};
 use crate::af::AddressFamily;
-use crate::custom_alloc::{CustomAllocStorage, StoreConfig, UpsertReport};
 use crate::insert_match;
-use crate::local_array::store::atomic_types::{NodeBuckets, PrefixBuckets};
+use crate::rib::{Rib, StoreConfig, UpsertReport};
 
+use super::super::errors::PrefixStoreError;
 pub(crate) use super::atomic_stride::*;
-use super::store::errors::PrefixStoreError;
 
-pub(crate) use crate::local_array::node::TreeBitMapNode;
+use super::node::TreeBitMapNode;
 
 #[cfg(feature = "cli")]
 use ansi_term::Colour;
@@ -74,80 +78,6 @@ pub(crate) enum NewNodeOrIndex<AF: AddressFamily> {
     ExistingPrefix,
 }
 
-#[derive(Hash, Eq, PartialEq, Debug, Copy, Clone)]
-pub struct PrefixId<AF: AddressFamily>(Option<(AF, u8)>);
-
-impl<AF: AddressFamily> PrefixId<AF> {
-    pub fn new(net: AF, len: u8) -> Self {
-        PrefixId(Some((net, len)))
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.0.is_none()
-    }
-
-    pub fn get_net(&self) -> AF {
-        self.0.unwrap().0
-    }
-
-    pub fn get_len(&self) -> u8 {
-        self.0.unwrap().1
-    }
-
-    // This should never fail, since there shouldn't be a invalid prefix in
-    // this prefix id in the first place.
-    pub fn into_pub(&self) -> inetnum::addr::Prefix {
-        inetnum::addr::Prefix::new(
-            self.get_net().into_ipaddr(),
-            self.get_len(),
-        )
-        .unwrap_or_else(|p| panic!("can't convert {:?} into prefix.", p))
-    }
-
-    // Increment the length of the prefix without changing the bits part.
-    // This is used to iterate over more-specific prefixes for this prefix,
-    // since the more specifics iterator includes the requested `base_prefix`
-    // itself.
-    pub fn inc_len(self) -> Self {
-        Self(self.0.map(|(net, len)| (net, len + 1)))
-    }
-
-    pub fn as_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
-        match self.0 {
-            Some(r) => r.0.as_prefix_bytes(r.1),
-            _ => [255; PREFIX_SIZE],
-        }
-    }
-}
-
-impl<AF: AddressFamily> std::default::Default for PrefixId<AF> {
-    fn default() -> Self {
-        PrefixId(None)
-    }
-}
-
-impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
-    fn from(value: inetnum::addr::Prefix) -> Self {
-        Self(Some((AF::from_ipaddr(value.addr()), value.len())))
-    }
-}
-
-impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<PrefixId<AF>>
-    for [u8; PREFIX_SIZE]
-{
-    fn from(value: PrefixId<AF>) -> Self {
-        value.as_bytes::<PREFIX_SIZE>()
-    }
-}
-
-impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<[u8; PREFIX_SIZE]>
-    for PrefixId<AF>
-{
-    fn from(value: [u8; PREFIX_SIZE]) -> Self {
-        PrefixId(Some(AF::from_prefix_bytes(value)))
-    }
-}
-
 //--------------------- Per-Stride-Node-Id Type -----------------------------
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
@@ -265,15 +195,133 @@ impl std::fmt::Display for StrideType {
 
 //--------------------- TreeBitMap ------------------------------------------
 
+#[derive(Debug)]
 pub struct TreeBitMap<
     AF: AddressFamily,
     M: Meta,
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
-    const PREFIX_SIZE: usize,
-    const KEY_SIZE: usize,
 > {
-    pub store: CustomAllocStorage<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
+    pub(crate) node_buckets: NB,
+    pub(crate) prefix_buckets: PB,
+    withdrawn_muis_bmin: Atomic<RoaringBitmap>,
+    _af: PhantomData<AF>,
+    _m: PhantomData<M>,
+}
+
+impl<
+        AF: AddressFamily,
+        M: Meta,
+        NB: NodeBuckets<AF>,
+        PB: PrefixBuckets<AF, M>,
+    > TreeBitMap<AF, M, NB, PB>
+{
+    pub(crate) fn init() -> Self {
+        Self {
+            node_buckets: NodeBuckets::init(),
+            prefix_buckets: PB::init(),
+            withdrawn_muis_bmin: RoaringBitmap::new().into(),
+            _af: PhantomData,
+            _m: PhantomData,
+        }
+    }
+
+    pub fn withdrawn_muis_bmin<'a>(
+        &'a self,
+        guard: &'a Guard,
+    ) -> &'a RoaringBitmap {
+        unsafe {
+            self.withdrawn_muis_bmin
+                .load(Ordering::Acquire, guard)
+                .deref()
+        }
+    }
+
+    // Change the status of the mui globally to Withdrawn. Iterators and match
+    // functions will by default not return any records for this mui.
+    pub fn mark_mui_as_withdrawn(
+        &self,
+        mui: u32,
+        guard: &Guard,
+    ) -> Result<(), PrefixStoreError> {
+        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
+
+        let mut new = unsafe { current.as_ref() }.unwrap().clone();
+        new.insert(mui);
+
+        #[allow(clippy::assigning_clones)]
+        loop {
+            match self.withdrawn_muis_bmin.compare_exchange(
+                current,
+                Owned::new(new),
+                Ordering::AcqRel,
+                Ordering::Acquire,
+                guard,
+            ) {
+                Ok(_) => return Ok(()),
+                Err(updated) => {
+                    new =
+                        unsafe { updated.current.as_ref() }.unwrap().clone();
+                }
+            }
+        }
+    }
+
+    // Change the status of the mui globally to Active. Iterators and match
+    // functions will default to the status on the record itself.
+    pub fn mark_mui_as_active(
+        &self,
+        mui: u32,
+        guard: &Guard,
+    ) -> Result<(), PrefixStoreError> {
+        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
+
+        let mut new = unsafe { current.as_ref() }.unwrap().clone();
+        new.remove(mui);
+
+        #[allow(clippy::assigning_clones)]
+        loop {
+            match self.withdrawn_muis_bmin.compare_exchange(
+                current,
+                Owned::new(new),
+                Ordering::AcqRel,
+                Ordering::Acquire,
+                guard,
+            ) {
+                Ok(_) => return Ok(()),
+                Err(updated) => {
+                    new =
+                        unsafe { updated.current.as_ref() }.unwrap().clone();
+                }
+            }
+        }
+    }
+
+    // Whether this mui is globally withdrawn. Note that this overrules
+    // (by default) any (prefix, mui) combination in iterators and match
+    // functions.
+    pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
+        unsafe {
+            self.withdrawn_muis_bmin
+                .load(Ordering::Acquire, guard)
+                .as_ref()
+        }
+        .unwrap()
+        .contains(mui)
+    }
+
+    // Whether this mui is globally active. Note that the local statuses of
+    // records (prefix, mui) may be set to withdrawn in iterators and match
+    // functions.
+    pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
+        !unsafe {
+            self.withdrawn_muis_bmin
+                .load(Ordering::Acquire, guard)
+                .as_ref()
+        }
+        .unwrap()
+        .contains(mui)
+    }
 }
 
 impl<
@@ -283,15 +331,15 @@ impl<
         PB: PrefixBuckets<AF, M>,
         const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
-    > TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+    > Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
     pub fn new(
         config: StoreConfig,
     ) -> Result<
-        TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
+        Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
         Box<dyn std::error::Error>,
     > {
-        let root_node = match CustomAllocStorage::<
+        let root_node = match Rib::<
             AF,
             M,
             NB,
@@ -323,18 +371,7 @@ impl<
             }
         };
 
-        Ok(
-            TreeBitMap {
-                store: CustomAllocStorage::<
-                    AF,
-                    M,
-                    NB,
-                    PB,
-                    PREFIX_SIZE,
-                    KEY_SIZE,
-                >::init(root_node, config)?,
-            },
-        )
+        Rib::<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>::init(root_node, config)
     }
 
     // Partition for stride 4
@@ -382,12 +419,12 @@ impl<
         }
 
         let mut stride_end: u8 = 0;
-        let mut cur_i = self.store.get_root_node_id();
+        let mut cur_i = self.get_root_node_id();
         let mut level: u8 = 0;
         let mut acc_retry_count = 0;
 
         loop {
-            let stride = self.store.get_stride_sizes()[level as usize];
+            let stride = self.get_stride_sizes()[level as usize];
             stride_end += stride;
             let nibble_len = if pfx.get_len() < stride_end {
                 stride + pfx.get_len() - stride_end
@@ -452,10 +489,6 @@ impl<
         }
     }
 
-    pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
-        self.store.get_root_node_id()
-    }
-
     // Yes, we're hating this. But, the root node has no room for a serial of
     // the prefix 0/0 (the default route), which doesn't even matter, unless,
     // UNLESS, somebody wants to store a default route. So we have to store a
@@ -486,40 +519,39 @@ impl<
     ) -> Result<UpsertReport, PrefixStoreError> {
         trace!("Updating the default route...");
 
-        if let Some(root_node) = self.store.retrieve_node_mut(
-            self.store.get_root_node_id(),
+        if let Some(root_node) = self.retrieve_node_mut(
+            self.get_root_node_id(),
             record.multi_uniq_id,
             // guard,
         ) {
             match root_node {
                 SizedStrideRef::Stride3(_) => {
-                    self.store
-                        .buckets
-                        .get_store3(self.store.get_root_node_id())
+                    self.in_memory_tree
+                        .node_buckets
+                        .get_store3(self.get_root_node_id())
                         .update_rbm_index(record.multi_uniq_id)?;
                 }
                 SizedStrideRef::Stride4(_) => {
-                    self.store
-                        .buckets
-                        .get_store4(self.store.get_root_node_id())
+                    self.in_memory_tree
+                        .node_buckets
+                        .get_store4(self.get_root_node_id())
                         .update_rbm_index(record.multi_uniq_id)?;
                 }
                 SizedStrideRef::Stride5(_) => {
-                    self.store
-                        .buckets
-                        .get_store5(self.store.get_root_node_id())
+                    self.in_memory_tree
+                        .node_buckets
+                        .get_store5(self.get_root_node_id())
                         .update_rbm_index(record.multi_uniq_id)?;
                 }
             };
         };
 
-        self.store.upsert_prefix(
+        self.upsert_prefix(
             PrefixId::new(AF::zero(), 0),
             record,
             // Do not update the path selection for the default route.
             None,
             guard,
-            // user_data,
         )
     }
 
@@ -531,8 +563,8 @@ impl<
         start_node_id: StrideNodeId<AF>,
         found_pfx_vec: &mut Vec<PrefixId<AF>>,
     ) {
-        trace!("{:?}", self.store.retrieve_node(start_node_id));
-        match self.store.retrieve_node(start_node_id) {
+        trace!("{:?}", self.retrieve_node(start_node_id));
+        match self.retrieve_node(start_node_id) {
             Some(SizedStrideRef::Stride3(n)) => {
                 found_pfx_vec.extend(
                     n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
@@ -608,18 +640,17 @@ impl<
         PB: PrefixBuckets<AF, M>,
         const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
-    > std::fmt::Display for TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+    > std::fmt::Display for Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
     fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(_f, "{} prefixes created", self.store.get_prefixes_count())?;
-        writeln!(_f, "{} nodes created", self.store.get_nodes_count())?;
+        writeln!(_f, "{} prefixes created", self.get_prefixes_count())?;
+        writeln!(_f, "{} nodes created", self.get_nodes_count())?;
         writeln!(_f)?;
 
         writeln!(
             _f,
             "stride division {:?}",
-            self.store
-                .get_stride_sizes()
+            self.get_stride_sizes()
                 .iter()
                 .map_while(|s| if s > &0 { Some(*s) } else { None })
                 .collect::<Vec<_>>()
@@ -636,8 +667,7 @@ impl<
 
         trace!(
             "stride_sizes {:?}",
-            self.store
-                .get_stride_sizes()
+            self.get_stride_sizes()
                 .iter()
                 .map_while(|s| if s > &0 { Some(*s) } else { None })
                 .enumerate()
@@ -647,7 +677,7 @@ impl<
         for crate::stats::CreatedNodes {
             depth_level: len,
             count: prefix_count,
-        } in self.store.counters.get_prefix_stats()
+        } in self.counters.get_prefix_stats()
         {
             let max_pfx = u128::overflowing_pow(2, len as u32);
             let n = (prefix_count as u32 / SCALE) as usize;
diff --git a/src/local_array/store/iterators.rs b/src/local_array/iterators.rs
similarity index 97%
rename from src/local_array/store/iterators.rs
rename to src/local_array/iterators.rs
index a976418c..f9176e9e 100644
--- a/src/local_array/store/iterators.rs
+++ b/src/local_array/iterators.rs
@@ -8,21 +8,24 @@
 // individual nodes. The Node Iterators live in the node.rs file.
 use std::sync::atomic::Ordering;
 
-use super::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
-use super::custom_alloc::CustomAllocStorage;
-use crate::local_array::store::atomic_types::RouteStatus;
+use super::in_memory::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
+use super::in_memory::tree::{Stride3, Stride4, Stride5, StrideNodeId};
+use super::types::PrefixId;
+use crate::local_array::in_memory::tree::SizedStrideRef;
+use crate::local_array::types::RouteStatus;
 use crate::prefix_record::PublicRecord;
+use crate::rib;
 use crate::{
     af::AddressFamily,
     local_array::{
         bit_span::BitSpan,
-        node::{
-            NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter, PrefixId,
-            SizedStrideRef, Stride3, Stride4, Stride5, StrideNodeId,
+        in_memory::node::{
+            NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter,
         },
     },
     prefix_record::Meta,
 };
+use rib::Rib;
 
 use crossbeam_epoch::Guard;
 use inetnum::addr::Prefix;
@@ -293,7 +296,7 @@ pub(crate) struct MoreSpecificPrefixIter<
     const PREFIX_SIZE: usize,
     const KEY_SIZE: usize,
 > {
-    store: &'a CustomAllocStorage<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
+    store: &'a Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
     cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
     cur_pfx_iter: SizedPrefixIter<AF>,
     start_bit_span: BitSpan,
@@ -708,7 +711,7 @@ impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
 // ----------- Iterator initialization methods for CustomAllocStorage -------
 
 // These are only the methods that are starting the iterations. All other
-// methods for CustomAllocStorage are in the main custom_alloc.rs file.
+// methods for Rib are in the main rib.rs file.
 
 impl<
         'a,
@@ -718,7 +721,7 @@ impl<
         PB: PrefixBuckets<AF, M>,
         const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
-    > CustomAllocStorage<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+    > Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
     // Iterator over all more-specific prefixes, starting from the given
     // prefix at the given level and cursor.
@@ -814,11 +817,8 @@ impl<
                     }
                 };
 
-                let global_withdrawn_bmin = unsafe {
-                    self.withdrawn_muis_bmin
-                        .load(Ordering::Acquire, guard)
-                        .deref()
-                };
+                let global_withdrawn_bmin =
+                    self.in_memory_tree.withdrawn_muis_bmin(guard);
 
                 Some(MoreSpecificPrefixIter {
                     store: self,
@@ -860,15 +860,15 @@ impl<
             None
         } else {
             let cur_len = start_prefix_id.get_len() - 1;
-            let cur_bucket = self.prefixes.get_root_prefix_set(cur_len);
-            let global_withdrawn_bmin = unsafe {
-                self.withdrawn_muis_bmin
-                    .load(Ordering::Acquire, guard)
-                    .deref()
-            };
+            let cur_bucket = self
+                .in_memory_tree
+                .prefix_buckets
+                .get_root_prefix_set(cur_len);
+            let global_withdrawn_bmin =
+                self.in_memory_tree.withdrawn_muis_bmin(guard);
 
             Some(LessSpecificPrefixIter {
-                prefixes: &self.prefixes,
+                prefixes: &self.in_memory_tree.prefix_buckets,
                 cur_len,
                 cur_bucket,
                 cur_level: 0,
@@ -887,8 +887,11 @@ impl<
         &'a self,
     ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
         PrefixIter {
-            prefixes: &self.prefixes,
-            cur_bucket: self.prefixes.get_root_prefix_set(0),
+            prefixes: &self.in_memory_tree.prefix_buckets,
+            cur_bucket: self
+                .in_memory_tree
+                .prefix_buckets
+                .get_root_prefix_set(0),
             cur_len: 0,
             cur_level: 0,
             cursor: 0,
diff --git a/src/local_array/mod.rs b/src/local_array/mod.rs
index 812c15c6..b123f8ff 100644
--- a/src/local_array/mod.rs
+++ b/src/local_array/mod.rs
@@ -1,11 +1,9 @@
-mod atomic_stride;
-pub(crate) mod bit_span;
-pub(crate) mod node;
-pub(crate) mod query;
+pub(crate) mod in_memory;
 mod tests;
-pub(crate) mod tree;
-
-pub mod store;
 
-#[macro_use]
-mod macros;
+pub(crate) mod bit_span;
+pub mod errors;
+pub mod iterators;
+pub mod query;
+pub mod rib;
+pub mod types;
diff --git a/src/local_array/node.rs b/src/local_array/node.rs
index 88ebfd06..040f7a9e 100644
--- a/src/local_array/node.rs
+++ b/src/local_array/node.rs
@@ -80,6 +80,7 @@ where
     AF: AddressFamily,
     S: Stride
 {
+<<<<<<< Updated upstream
     pub(crate) fn new() -> Self {
         TreeBitMapNode {
             ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
@@ -88,6 +89,16 @@ where
         }
     }
 
+=======
+    pub(crate) fn empty() -> Self {
+        TreeBitMapNode { 
+            ptrbitarr: <S as Stride>::AtomicPtrSize::new(), 
+            pfxbitarr: <S as Stride>::AtomicPfxSize::new(),
+            _af: PhantomData
+        }
+    }
+    
+>>>>>>> Stashed changes
     // ------- Iterators ----------------------------------------------------
 
     // Iterate over all the child node of this node
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index c25885fd..7d5426fa 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -4,25 +4,27 @@ use crossbeam_epoch::{self as epoch};
 use epoch::Guard;
 
 use crate::af::AddressFamily;
-use crate::custom_alloc::{PersistStrategy, PersistTree};
-use crate::local_array::store::atomic_types::{NodeBuckets, PrefixBuckets};
+use crate::local_array::in_memory::atomic_types::{
+    NodeBuckets, PrefixBuckets,
+};
 use crate::prefix_record::{Meta, PublicRecord};
+use crate::rib::{PersistStrategy, PersistTree, Rib};
 use inetnum::addr::Prefix;
 
 use crate::QueryResult;
 
-use crate::local_array::node::TreeBitMapNode;
-use crate::local_array::tree::TreeBitMap;
+use crate::local_array::in_memory::node::TreeBitMapNode;
 use crate::{MatchOptions, MatchType};
 
-use super::node::{PrefixId, SizedStrideRef, StrideNodeId};
-use super::store::atomic_types::{RouteStatus, StoredPrefix};
-use super::store::errors::PrefixStoreError;
+use super::errors::PrefixStoreError;
+use super::in_memory::atomic_types::StoredPrefix;
+use super::in_memory::tree::{SizedStrideRef, StrideNodeId};
+use super::types::{PrefixId, RouteStatus};
 
 //------------ Prefix Matching ----------------------------------------------
 
 impl<'a, AF, M, NB, PB, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
-    TreeBitMap<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+    Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 where
     AF: AddressFamily,
     M: Meta,
@@ -36,9 +38,9 @@ where
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> QueryResult<M> {
-        let result = self.store.non_recursive_retrieve_prefix(prefix_id);
+        let result = self.non_recursive_retrieve_prefix(prefix_id);
         let prefix = result.0;
-        let more_specifics_vec = self.store.more_specific_prefix_iter_from(
+        let more_specifics_vec = self.more_specific_prefix_iter_from(
             prefix_id,
             mui,
             include_withdrawn,
@@ -71,12 +73,12 @@ where
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> QueryResult<M> {
-        let result = self.store.non_recursive_retrieve_prefix(prefix_id);
+        let result = self.non_recursive_retrieve_prefix(prefix_id);
 
         let prefix = result.0;
         let less_specifics_vec = result.1.map(
             |(prefix_id, _level, _cur_set, _parents, _index)| {
-                self.store.less_specific_prefix_iter(
+                self.less_specific_prefix_iter(
                     prefix_id,
                     mui,
                     include_withdrawn,
@@ -112,14 +114,20 @@ where
         guard: &'a Guard,
     ) -> Result<
         impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a,
-        std::io::Error,
+        PrefixStoreError,
     > {
-        Ok(self.store.more_specific_prefix_iter_from(
-            prefix_id,
-            mui,
-            include_withdrawn,
-            guard,
-        ))
+        let bmin = self.in_memory_tree.withdrawn_muis_bmin(guard);
+
+        if mui.is_some() && bmin.contains(mui.unwrap()) {
+            Err(PrefixStoreError::PrefixNotFound)
+        } else {
+            Ok(self.more_specific_prefix_iter_from(
+                prefix_id,
+                mui,
+                include_withdrawn,
+                guard,
+            ))
+        }
     }
 
     pub fn match_prefix(
@@ -128,7 +136,7 @@ where
         options: &MatchOptions,
         guard: &'a Guard,
     ) -> QueryResult<M> {
-        match self.store.config.persist_strategy() {
+        match self.config.persist_strategy() {
             PersistStrategy::PersistOnly => {
                 self.match_prefix_in_persisted_store(search_pfx, options.mui)
             }
@@ -145,35 +153,28 @@ where
         // `non_recursive_retrieve_prefix` returns an exact match only, so no
         // longest matching prefix!
         let mut stored_prefix =
-            self.store.non_recursive_retrieve_prefix(search_pfx).0.map(
-                |pfx| {
-                    (
-                        pfx.prefix,
-                        if !options.include_withdrawn {
-                            // Filter out all the withdrawn records, both
-                            // with globally withdrawn muis, and with local
-                            // statuses
-                            // set to Withdrawn.
-                            self.get_filtered_records(pfx, options.mui, guard)
-                                .into_iter()
-                                .collect()
-                        } else {
-                            // Do no filter out any records, but do rewrite
-                            // the local statuses of the records with muis
-                            // that appear in the specified bitmap index.
-                            pfx.record_map.as_records_with_rewritten_status(
-                                unsafe {
-                                    self.store
-                                        .withdrawn_muis_bmin
-                                        .load(Ordering::Acquire, guard)
-                                        .deref()
-                                },
-                                RouteStatus::Withdrawn,
-                            )
-                        },
-                    )
-                },
-            );
+            self.non_recursive_retrieve_prefix(search_pfx).0.map(|pfx| {
+                (
+                    pfx.prefix,
+                    if !options.include_withdrawn {
+                        // Filter out all the withdrawn records, both
+                        // with globally withdrawn muis, and with local
+                        // statuses
+                        // set to Withdrawn.
+                        self.get_filtered_records(pfx, options.mui, guard)
+                            .into_iter()
+                            .collect()
+                    } else {
+                        // Do no filter out any records, but do rewrite
+                        // the local statuses of the records with muis
+                        // that appear in the specified bitmap index.
+                        pfx.record_map.as_records_with_rewritten_status(
+                            self.in_memory_tree.withdrawn_muis_bmin(guard),
+                            RouteStatus::Withdrawn,
+                        )
+                    },
+                )
+            });
 
         // Check if we have an actual exact match, if not then fetch the
         // first lesser-specific with the greatest length, that's the Longest
@@ -188,7 +189,6 @@ where
             // so we need to find the longest matching prefix.
             (MatchType::LongestMatch | MatchType::EmptyMatch, _) => {
                 stored_prefix = self
-                    .store
                     .less_specific_prefix_iter(
                         search_pfx,
                         options.mui,
@@ -216,36 +216,34 @@ where
                 .unwrap_or_default(),
             less_specifics: if options.include_less_specifics {
                 Some(
-                    self.store
-                        .less_specific_prefix_iter(
-                            if let Some(ref pfx) = stored_prefix {
-                                pfx.0
-                            } else {
-                                search_pfx
-                            },
-                            options.mui,
-                            options.include_withdrawn,
-                            guard,
-                        )
-                        .collect(),
+                    self.less_specific_prefix_iter(
+                        if let Some(ref pfx) = stored_prefix {
+                            pfx.0
+                        } else {
+                            search_pfx
+                        },
+                        options.mui,
+                        options.include_withdrawn,
+                        guard,
+                    )
+                    .collect(),
                 )
             } else {
                 None
             },
             more_specifics: if options.include_more_specifics {
                 Some(
-                    self.store
-                        .more_specific_prefix_iter_from(
-                            if let Some(pfx) = stored_prefix {
-                                pfx.0
-                            } else {
-                                search_pfx
-                            },
-                            options.mui,
-                            options.include_withdrawn,
-                            guard,
-                        )
-                        .collect(),
+                    self.more_specific_prefix_iter_from(
+                        if let Some(pfx) = stored_prefix {
+                            pfx.0
+                        } else {
+                            search_pfx
+                        },
+                        options.mui,
+                        options.include_withdrawn,
+                        guard,
+                    )
+                    .collect(),
                 )
                 // The user requested more specifics, but there aren't any, so
                 // we need to return an empty vec, not a None.
@@ -268,7 +266,7 @@ where
             search_pfx.as_bytes::<PREFIX_SIZE>().to_vec()
         };
 
-        if let Some(persist) = &self.store.persistence {
+        if let Some(persist) = &self.persist_tree {
             QueryResult {
                 prefix: Some(search_pfx.into_pub()),
                 match_type: MatchType::ExactMatch,
@@ -296,8 +294,7 @@ where
         search_pfx: PrefixId<AF>,
         guard: &Guard,
     ) -> Option<Result<PublicRecord<M>, PrefixStoreError>> {
-        self.store
-            .non_recursive_retrieve_prefix(search_pfx)
+        self.non_recursive_retrieve_prefix(search_pfx)
             .0
             .map(|p_rec| {
                 p_rec.get_path_selections(guard).best().map_or_else(
@@ -318,8 +315,7 @@ where
         tbi: &<M as Meta>::TBI,
         guard: &Guard,
     ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
-        self.store
-            .non_recursive_retrieve_prefix(search_pfx)
+        self.non_recursive_retrieve_prefix(search_pfx)
             .0
             .map_or(Err(PrefixStoreError::StoreNotReadyError), |p_rec| {
                 p_rec.calculate_and_store_best_backup(tbi, guard)
@@ -331,8 +327,7 @@ where
         search_pfx: PrefixId<AF>,
         guard: &Guard,
     ) -> Result<bool, PrefixStoreError> {
-        self.store
-            .non_recursive_retrieve_prefix(search_pfx)
+        self.non_recursive_retrieve_prefix(search_pfx)
             .0
             .map_or(Err(PrefixStoreError::StoreNotReadyError), |p| {
                 Ok(p.is_ps_outdated(guard))
@@ -376,45 +371,44 @@ where
         // which lives on the root node itself! We are *not* going to return
         // all of the prefixes in the tree as more-specifics.
         if search_pfx.get_len() == 0 {
-            match self.store.load_default_route_prefix_serial() {
-                0 => {
-                    return QueryResult {
-                        prefix: None,
-                        prefix_meta: vec![],
-                        match_type: MatchType::EmptyMatch,
-                        less_specifics: None,
-                        more_specifics: None,
-                    };
-                }
-
-                _serial => {
-                    let prefix_meta = self
-                        .store
-                        .retrieve_prefix(PrefixId::new(AF::zero(), 0))
-                        .map(|sp| sp.0.record_map.as_records())
-                        .unwrap_or_default();
-                    return QueryResult {
-                        prefix: Prefix::new(
-                            search_pfx.get_net().into_ipaddr(),
-                            search_pfx.get_len(),
-                        )
-                        .ok(),
-                        prefix_meta,
-                        match_type: MatchType::ExactMatch,
-                        less_specifics: None,
-                        more_specifics: None,
-                    };
-                }
-            }
+            // match self.load_default_route_prefix_serial() {
+            //     0 => {
+            //         return QueryResult {
+            //             prefix: None,
+            //             prefix_meta: vec![],
+            //             match_type: MatchType::EmptyMatch,
+            //             less_specifics: None,
+            //             more_specifics: None,
+            //         };
+            //     }
+
+            //     _serial => {
+            let prefix_meta = self
+                .retrieve_prefix(PrefixId::new(AF::zero(), 0))
+                .map(|sp| sp.0.record_map.as_records())
+                .unwrap_or_default();
+            return QueryResult {
+                prefix: Prefix::new(
+                    search_pfx.get_net().into_ipaddr(),
+                    search_pfx.get_len(),
+                )
+                .ok(),
+                prefix_meta,
+                match_type: MatchType::ExactMatch,
+                less_specifics: None,
+                more_specifics: None,
+            };
+            // }
+            // }
         }
 
         let mut stride_end = 0;
 
         let root_node_id = self.get_root_node_id();
-        let mut node = match self.store.get_stride_for_id(root_node_id) {
-            3 => self.store.retrieve_node(root_node_id).unwrap(),
-            4 => self.store.retrieve_node(root_node_id).unwrap(),
-            _ => self.store.retrieve_node(root_node_id).unwrap(),
+        let mut node = match self.get_stride_for_id(root_node_id) {
+            3 => self.retrieve_node(root_node_id).unwrap(),
+            4 => self.retrieve_node(root_node_id).unwrap(),
+            _ => self.retrieve_node(root_node_id).unwrap(),
         };
 
         let mut nibble;
@@ -458,7 +452,7 @@ where
         // having to match over a SizedStrideNode again in the
         // `post-processing` section.
 
-        for stride in self.store.get_stride_sizes() {
+        for stride in self.get_stride_sizes() {
             stride_end += stride;
 
             let last_stride = search_pfx.get_len() < stride_end;
@@ -519,7 +513,7 @@ where
                         // exit nodes.
                         (Some(n), Some(pfx_idx)) => {
                             match_prefix_idx = Some(pfx_idx);
-                            node = self.store.retrieve_node(n).unwrap();
+                            node = self.retrieve_node(n).unwrap();
 
                             if last_stride {
                                 if options.include_more_specifics {
@@ -538,7 +532,7 @@ where
                             }
                         }
                         (Some(n), None) => {
-                            node = self.store.retrieve_node(n).unwrap();
+                            node = self.retrieve_node(n).unwrap();
 
                             if last_stride {
                                 if options.include_more_specifics {
@@ -637,7 +631,7 @@ where
                     ) {
                         (Some(n), Some(pfx_idx)) => {
                             match_prefix_idx = Some(pfx_idx);
-                            node = self.store.retrieve_node(n).unwrap();
+                            node = self.retrieve_node(n).unwrap();
 
                             if last_stride {
                                 if options.include_more_specifics {
@@ -656,7 +650,7 @@ where
                             }
                         }
                         (Some(n), None) => {
-                            node = self.store.retrieve_node(n).unwrap();
+                            node = self.retrieve_node(n).unwrap();
 
                             if last_stride {
                                 if options.include_more_specifics {
@@ -744,7 +738,7 @@ where
                     ) {
                         (Some(n), Some(pfx_idx)) => {
                             match_prefix_idx = Some(pfx_idx);
-                            node = self.store.retrieve_node(n).unwrap();
+                            node = self.retrieve_node(n).unwrap();
 
                             if last_stride {
                                 if options.include_more_specifics {
@@ -763,7 +757,7 @@ where
                             }
                         }
                         (Some(n), None) => {
-                            node = self.store.retrieve_node(n).unwrap();
+                            node = self.retrieve_node(n).unwrap();
 
                             if last_stride {
                                 if options.include_more_specifics {
@@ -839,7 +833,7 @@ where
         let mut match_type: MatchType = MatchType::EmptyMatch;
         let prefix = None;
         if let Some(pfx_idx) = match_prefix_idx {
-            match_type = match self.store.retrieve_prefix(pfx_idx) {
+            match_type = match self.retrieve_prefix(pfx_idx) {
                 Some(prefix) => {
                     if prefix.0.prefix.get_len() == search_pfx.get_len() {
                         MatchType::ExactMatch
@@ -864,7 +858,7 @@ where
                     .unwrap()
                     .iter()
                     .filter_map(move |p| {
-                        self.store.retrieve_prefix(*p).map(|p| {
+                        self.retrieve_prefix(*p).map(|p| {
                             Some((p.0.prefix, p.0.record_map.as_records()))
                         })
                     })
@@ -876,8 +870,7 @@ where
                 more_specifics_vec.map(|vec| {
                     vec.into_iter()
                         .map(|p| {
-                            self.store
-                                .retrieve_prefix(p)
+                            self.retrieve_prefix(p)
                                 .unwrap_or_else(|| {
                                     panic!(
                                         "more specific {:?} does not exist",
@@ -903,13 +896,7 @@ where
         mui: Option<u32>,
         guard: &Guard,
     ) -> Vec<PublicRecord<M>> {
-        let bmin = unsafe {
-            self.store
-                .withdrawn_muis_bmin
-                .load(Ordering::Acquire, guard)
-                .as_ref()
-        }
-        .unwrap();
+        let bmin = self.in_memory_tree.withdrawn_muis_bmin(guard);
 
         pfx.record_map.get_filtered_records(mui, bmin)
     }
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
new file mode 100644
index 00000000..aa683a16
--- /dev/null
+++ b/src/local_array/rib/default_store.rs
@@ -0,0 +1,28 @@
+use crate::prelude::multi::*;
+use crate::prelude::*;
+
+// The default stride sizes for IPv4, IPv6, resp.
+#[create_store((
+    ([5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 5, 18),
+    ([4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
+    4, 4, 4, 4, 4, 4, 4, 4, 4], 17, 30)
+))]
+struct DefaultStore;
+
+/// Try some
+impl<M: Meta> DefaultStore<M> {
+    pub fn try_default() -> Result<Self, PrefixStoreError> {
+        let config = StoreConfig::default();
+        Self::new_with_config(config)
+            .map_err(|_| PrefixStoreError::StoreNotReadyError)
+    }
+}
+
+impl Default for StoreConfig {
+    fn default() -> Self {
+        Self {
+            persist_strategy: PersistStrategy::MemoryOnly,
+            persist_path: "/tmp/rotonda/".to_string(),
+        }
+    }
+}
diff --git a/src/local_array/rib/macros.rs b/src/local_array/rib/macros.rs
new file mode 100644
index 00000000..2e5d8fab
--- /dev/null
+++ b/src/local_array/rib/macros.rs
@@ -0,0 +1,379 @@
+#[macro_export]
+#[doc(hidden)]
+macro_rules! impl_search_level {
+    (
+        $(
+            $stride: ident;
+            $id: ident;
+        ),
+    * ) => {
+        $(
+        SearchLevel {
+            f: &|search_level: &SearchLevel<AF, $stride>,
+                nodes,
+                mut level: u8,
+            | {
+                    // HASHING FUNCTION
+                    let index = Self::hash_node_id($id, level);
+
+                    // Read the node from the block pointed to by the Atomic
+                    // pointer.
+                    // let stored_node = unsafe {
+                    //     &mut nodes.0[index]
+                    // };
+                    // let this_node = stored_node.load(Ordering::Acquire, guard);
+
+                    match nodes.0.get(index) {
+                        None => None,
+                        Some(stored_node) => {
+                            let StoredNode { node_id, node, node_set, .. } = stored_node;
+                            if $id == *node_id {
+                                // YES, It's the one we're looking for!
+                                return Some(SizedStrideRef::$stride(&node));
+                            };
+                            // Meh, it's not, but we can a go to the next
+                            // level and see if it lives there.
+                            level += 1;
+                            match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
+                                // on to the next level!
+                                next_bit_shift if next_bit_shift > 0 => {
+                                    (search_level.f)(
+                                        search_level,
+                                        &node_set,
+                                        level,
+                                        // guard,
+                                    )
+                                }
+                                // There's no next level, we found nothing.
+                                _ => None,
+                            }
+                        }
+                    }
+                }
+        }
+        )*
+    };
+}
+
+#[macro_export]
+#[doc(hidden)]
+macro_rules! impl_search_level_for_mui {
+    (
+        $(
+            $stride: ident;
+            $id: ident;
+            $mui: ident;
+        ),
+    * ) => {
+        $(
+        SearchLevel {
+            f: &|search_level: &SearchLevel<AF, $stride>,
+                nodes,
+                mut level: u8| {
+                    // HASHING FUNCTION
+                    let index = Self::hash_node_id($id, level);
+
+                    // Read the node from the block pointed to by the Atomic
+                    // pointer.
+                    // let stored_node = unsafe {
+                    //     &mut nodes.0[index].assume_init_ref()
+                    // };
+                    // let this_node = stored_node.load(Ordering::Acquire, guard);
+
+                    match nodes.0.get(index) {
+                        None => None,
+                        Some(this_node) => {
+                            let StoredNode { node_id, node, node_set, .. } = this_node;
+
+                            // early return if the mui is not in the index
+                            // stored in this node, meaning the mui does not
+                            // appear anywhere in the sub-tree formed from
+                            // this node.
+                            let bmin = node_set.1.read().unwrap(); // load(Ordering::Acquire, guard).deref()
+                            if !bmin.contains($mui) {
+                                return None;
+                            }
+
+                            if $id == *node_id {
+                                // YES, It's the one we're looking for!
+                                return Some(SizedStrideRef::$stride(&node));
+                            };
+                            // Meh, it's not, but we can a go to the next
+                            // level and see if it lives there.
+                            level += 1;
+                            match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
+                                // on to the next level!
+                                next_bit_shift if next_bit_shift > 0 => {
+                                    (search_level.f)(
+                                        search_level,
+                                        &node_set,
+                                        level,
+                                        // guard,
+                                    )
+                                }
+                                // There's no next level, we found nothing.
+                                _ => None,
+                            }
+                        }
+                    }
+                }
+        }
+        )*
+    };
+}
+
+// This macro creates a closure that is used in turn in the macro
+// 'eBox', that is used in the public `insert` method on a TreeBitMap.
+//
+// It retrieves the node specified by $id recursively, creates it if it does
+// not exist. It is responsible for setting/updating the RBMIN, but is does
+// *not* set/update the pfxbitarr or ptrbitarr of the TreeBitMapNode. The
+// `insert_match` takes care of the latter.
+//
+// This closure should not be called repeatedly to create the same node, if it
+// returns `None` that is basically a data race in the store and therefore an
+// error. Also the caller should make sure to stay within the limit of the
+// defined number of levels, although the closure will return at the end of
+// the maximum depth.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! retrieve_node_mut_closure {
+    (
+        $(
+            $stride: ident;
+            $id: ident;
+            $multi_uniq_id: ident;
+        ),
+    * ) => {$(
+        SearchLevel {
+            f: &|
+                search_level: &SearchLevel<AF, $stride>,
+                nodes,
+                mut level: u8,
+            | {
+                // HASHING FUNCTION
+                let index = Self::hash_node_id($id, level);
+                let node;
+
+                match nodes.0.get(index) {
+                    // This arm only ever gets called in multi-threaded code
+                    // where our thread (running this code *now*), andgot ahead
+                    // of another thread: After the other thread created the
+                    // TreeBitMapNode first, it was overtaken by our thread
+                    // running this method, so our thread enounters an empty node
+                    // in the store.
+                    None => {
+                        let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            $id.get_id().1, level
+                        );
+                        let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            $id.get_id().1, level + 1
+                        );
+                        let node_set = NodeSet::init(next_level - this_level);
+
+                        // See if we can create the node
+                        (node, _) = nodes.0.get_or_init(index, || StoredNode {
+                            node_id: $id,
+                            node: TreeBitMapNode::new(),
+                            node_set
+                        });
+
+                        // We may have lost, and a different node than we
+                        // intended could live here, if so go a level deeper
+                        if $id == node.node_id {
+                            // Nope, its ours or at least the node we need.
+                            let _retry_count = node.node_set.update_rbm_index(
+                                $multi_uniq_id
+                            ).ok();
+
+                            return Some(SizedStrideRef::$stride(&node.node));
+                        };
+                    },
+                    Some(this_node) => {
+                        node = this_node;
+                        if $id == this_node.node_id {
+                            // YES, It's the one we're looking for!
+
+                            // Update the rbm_index in this node with the
+                            // multi_uniq_id that the caller specified. This
+                            // is the only atomic operation we need to do
+                            // here. The NodeSet that the index is attached
+                            // to, does not need to be written to, it's part
+                            // of a trie, so it just needs to "exist" (and it
+                            // already does).
+                            let retry_count = this_node.node_set.update_rbm_index(
+                                $multi_uniq_id
+                            ).ok();
+
+                            trace!("Retry_count rbm index {:?}", retry_count);
+                            trace!("add multi uniq id to bitmap index {} for node {}",
+                                $multi_uniq_id, this_node.node
+                            );
+                            return Some(SizedStrideRef::$stride(&this_node.node));
+                        };
+                    }
+                }
+                // It isn't ours. Move one level deeper.
+                level += 1;
+                match <NB as NodeBuckets<AF>>::len_to_store_bits(
+                    $id.get_id().1, level
+                ) {
+                    // on to the next level!
+                    next_bit_shift if next_bit_shift > 0 => {
+                        (search_level.f)(
+                            search_level,
+                            &node.node_set,
+                            level,
+                        )
+                    }
+                    // There's no next level, we found nothing.
+                    _ => None,
+                }
+            }
+        }
+    )*};
+}
+
+#[macro_export]
+#[doc(hidden)]
+macro_rules! store_node_closure {
+    (
+        $(
+            $stride: ident;
+            $id: ident;
+            // $multi_uniq_id: ident;
+            $guard: ident;
+            $back_off: ident;
+        ),
+    *) => {
+        $(
+        SearchLevel {
+            f: &|
+            search_level: &SearchLevel<AF, $stride>,
+            nodes,
+            new_node: TreeBitMapNode<AF, $stride>,
+            multi_uniq_id: u32,
+            mut level: u8,
+            retry_count: u32| {
+                let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level);
+                trace!("{:032b}", $id.get_id().0);
+                trace!("id {:?}", $id.get_id());
+                trace!("multi_uniq_id {}", multi_uniq_id);
+
+                // HASHING FUNCTION
+                let index = Self::hash_node_id($id, level);
+
+                match nodes.0.get(index) {
+                    None => {
+                        // No node exists, so we create one here.
+                        let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level + 1);
+
+                        if log_enabled!(log::Level::Trace) {
+                            trace!("Empty node found, creating new node {} len{} lvl{}",
+                                $id, $id.get_id().1, level + 1
+                            );
+                            trace!("Next level {}",
+                                next_level
+                            );
+                            trace!("Creating space for {} nodes",
+                                if next_level >= this_level { 1 << (next_level - this_level) } else { 1 }
+                            );
+                        }
+
+                        trace!("multi uniq id {}", multi_uniq_id);
+
+                        let node_set = NodeSet::init(next_level - this_level);
+
+                        let ptrbitarr = new_node.ptrbitarr.load();
+                        let pfxbitarr = new_node.pfxbitarr.load();
+
+                        let (stored_node, its_us) = nodes.0.get_or_init(
+                            index,
+                            || StoredNode {
+                                node_id: $id,
+                                node: new_node,
+                                node_set
+                            }
+                        );
+
+                        if stored_node.node_id == $id {
+                            stored_node.node_set.update_rbm_index(
+                                multi_uniq_id
+                            )?;
+
+                            if !its_us && ptrbitarr != 0 {
+                                stored_node.node.ptrbitarr.merge_with(ptrbitarr);
+                            }
+
+                            if !its_us && pfxbitarr != 0 {
+                                stored_node.node.pfxbitarr.merge_with(pfxbitarr);
+                            }
+                        }
+
+                        return Ok(($id, retry_count));
+                    }
+                    Some(stored_node) => {
+                        // A node exists, might be ours, might be
+                        // another one.
+
+                        if log_enabled!(log::Level::Trace) {
+                            trace!("
+                                {} store: Node here exists {:?}",
+                                    std::thread::current().name().unwrap_or("unnamed-thread"),
+                                    stored_node.node_id
+                            );
+                            trace!("node_id {:?}", stored_node.node_id.get_id());
+                            trace!("node_id {:032b}", stored_node.node_id.get_id().0);
+                            trace!("id {}", $id);
+                            trace!("     id {:032b}", $id.get_id().0);
+                        }
+
+                        // See if somebody beat us to creating our
+                        // node already, if so, we still need to do
+                        // work: we have to update the bitmap index
+                        // with the multi_uniq_id we've got from the
+                        // caller.
+                        if $id == stored_node.node_id {
+                            stored_node.node_set.update_rbm_index(
+                                multi_uniq_id
+                            )?;
+
+                            if new_node.ptrbitarr.load() != 0 {
+                                stored_node.node.ptrbitarr.merge_with(new_node.ptrbitarr.load());
+                            }
+                            if new_node.pfxbitarr.load() != 0 {
+                                stored_node.node.pfxbitarr.merge_with(new_node.pfxbitarr.load());
+                            }
+
+                            return Ok(($id, retry_count));
+                        } else {
+                            // it's not "our" node, make a (recursive)
+                            // call to create it.
+                            level += 1;
+                            trace!("Collision with node_id {}, move to next level: {} len{} next_lvl{} index {}",
+                                stored_node.node_id, $id, $id.get_id().1, level, index
+                            );
+
+                            return match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
+                                // on to the next level!
+                                next_bit_shift if next_bit_shift > 0 => {
+                                    (search_level.f)(
+                                        search_level,
+                                        &stored_node.node_set,
+                                        new_node,
+                                        multi_uniq_id,
+                                        level,
+                                        retry_count
+                                    )
+                                }
+                                // There's no next level!
+                                _ => panic!("out of storage levels, current level is {}", level),
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    )*
+    };
+}
diff --git a/src/local_array/rib/mod.rs b/src/local_array/rib/mod.rs
new file mode 100644
index 00000000..3f8d12eb
--- /dev/null
+++ b/src/local_array/rib/mod.rs
@@ -0,0 +1,7 @@
+pub mod rib;
+
+pub(crate) mod default_store;
+
+pub use default_store::DefaultStore;
+#[macro_use]
+mod macros;
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
new file mode 100644
index 00000000..94c0d2e6
--- /dev/null
+++ b/src/local_array/rib/rib.rs
@@ -0,0 +1,1426 @@
+// ----------- THE STORE ----------------------------------------------------
+//
+// The CustomAllocStore provides in-memory storage for the BitTreeMapNodes
+// and for prefixes and their meta-data. The storage for node is on the
+// `buckets` field, and the prefixes are stored in, well, the `prefixes`
+// field. They are both organised in the same way, as chained hash tables,
+// one per (prefix|node)-length. The hashing function (that is detailed
+// lower down in this file), basically takes the address part of the
+// node|prefix and uses `(node|prefix)-address part % bucket size`
+// as its index.
+//
+// Both the prefixes and the buckets field have one bucket per (prefix|node)
+// -length that start out with a fixed-size array. The size of the arrays is
+// set in the rotonda_macros/maps.rs file.
+//
+// For lower (prefix|node)-lengths the number of elements in the array is
+// equal to the number of prefixes in that length, so there's exactly one
+// element per (prefix|node). For greater lengths there will be collisions,
+// in that case the stored (prefix|node) will have a reference to another
+// bucket (also of a fixed size), that holds a (prefix|node) that collided
+// with the one that was already stored. A (node|prefix) lookup will have to
+// go over all (node|prefix) buckets until it matches the requested (node|
+// prefix) or it reaches the end of the chain.
+//
+// The chained (node|prefixes) are occupied at a first-come, first-serve
+// basis, and are not re-ordered on new insertions of (node|prefixes). This
+// may change in the future, since it prevents iterators from being ordered.
+//
+// One of the nice things of having one table per (node|prefix)-length is that
+// a search can start directly at the prefix-length table it wishes, and go
+// go up and down into other tables if it needs to (e.g., because more- or
+// less-specifics were asked for). In contrast if you do a lookup by
+// traversing the tree of nodes, we would always have to go through the root-
+// node first and then go up the tree to the requested node. The lower nodes
+// of the tree (close to the root) would be a formidable bottle-neck then.
+//
+// Currently, the meta-data is an atomically stored value, that is required to
+// implement the `Meta` and the `Clone` trait. New meta-data
+// instances are stored atomically without further ado, but updates to a
+// piece of meta-data are done by merging the previous meta-data with the new
+// meta-data, through use of the `MergeUpdate` trait.
+//
+// The `upsert_prefix` methods retrieve only the most recent insert
+// for a prefix (for now).
+//
+// Future work could have a user-configurable retention strategy that allows
+// the meta-data to be stored as a linked-list of references, where each
+// meta-data object has a reference to its predecessor.
+//
+// Prefix example
+//
+//         (level 0 arrays)         prefixes  bucket
+//                                    /len     size
+//         ┌──┐
+// len /0  │ 0│                        1        1     ■
+//         └──┘                                       │
+//         ┌──┬──┐                                    │
+// len /1  │00│01│                     2        2     │
+//         └──┴──┘                                 perfect
+//         ┌──┬──┬──┬──┐                             hash
+// len /2  │  │  │  │  │               4        4     │
+//         └──┴──┴──┴──┘                              │
+//         ┌──┬──┬──┬──┬──┬──┬──┬──┐                  │
+// len /3  │  │  │  │  │  │  │  │  │   8        8     ■
+//         └──┴──┴──┴──┴──┴──┴──┴──┘
+//         ┌──┬──┬──┬──┬──┬──┬──┬──┐                        ┌────────────┐
+// len /4  │  │  │  │  │  │  │  │  │   8        16 ◀────────│ collision  │
+//         └──┴──┴──┴┬─┴──┴──┴──┴──┘                        └────────────┘
+//                   └───┐
+//                       │              ┌─collision─────────┐
+//                   ┌───▼───┐          │                   │
+//                   │       │ ◀────────│ 0x0100 and 0x0101 │
+//                   │ 0x010 │          └───────────────────┘
+//                   │       │
+//                   ├───────┴──────────────┬──┬──┐
+//                   │ StoredPrefix 0x0101  │  │  │
+//                   └──────────────────────┴─┬┴─┬┘
+//                                            │  │
+//                       ┌────────────────────┘  └──┐
+//            ┌──────────▼──────────┬──┐          ┌─▼┬──┐
+//         ┌─▶│ metadata (current)  │  │          │ 0│ 1│ (level 1 array)
+//         │  └─────────────────────┴──┘          └──┴──┘
+//    merge└─┐                        │             │
+//    update │           ┌────────────┘             │
+//           │┌──────────▼──────────┬──┐        ┌───▼───┐
+//         ┌─▶│ metadata (previous) │  │        │       │
+//         │  └─────────────────────┴──┘        │  0x0  │
+//    merge└─┐                        │         │       │
+//    update │           ┌────────────┘         ├───────┴──────────────┬──┐
+//           │┌──────────▼──────────┬──┐        │ StoredPrefix 0x0110  │  │
+//            │ metadata (oldest)   │  │        └──────────────────────┴──┘
+//            └─────────────────────┴──┘                                 │
+//                                                         ┌─────────────┘
+//                                              ┌──────────▼──────────────┐
+//                                              │ metadata (current)      │
+//                                              └─────────────────────────┘
+
+// Note about the memory usage of the data-structures of the Buckets
+//
+// As said, the prefixes and nodes are stored in buckets. A bucket right now
+// is of type `[MaybeUnit<Atomic<StoredPrefix>>]`, this has the advantage
+// that the length can be variable, based on the stride size for that level.
+// It saves us to have to implement a generic something.
+// Another advantage is the fixed place in which an atomic StoredPrefix
+// lives: this makes compare-and-swapping it relatively straight forward.
+// Each accessing thread would try to read/write the exact same entry in the
+// array, so shouldn't be any 'rug pulling' on the whole array.
+//
+// A disadvantage is that this is a fixed size, sparse array the moment it
+// is created. Theoretically, an `Atomic<Vec<StoredPrefix>`
+// would not have this disadvantage. Manipulating the whole vec atomically
+// though is very tricky (we would have to atomically compare-and-swap the
+// whole vec each time the prefix meta-data is changed) and inefficient,
+// since we would have to either keep the vec sorted on `PrefixId` at all
+// times, or, we would have to inspect each value in the vec on *every* read
+// or write. the StoredPrefix (this is a challenge in itself, since the
+// StoredPrefix needs to be read atomically to retrieve the PrefixId).
+// Compare-and-swapping a whole vec most probably would need a hash over the
+// vec to determine whether it was changed. I gave up on this approach,
+//
+// Another approach to try to limit the memory use is to try to use other
+// indexes in the same array on collision (the array mentioned above), before
+// heading off and following the reference to the next bucket. This would
+// limit the amount of (sparse) arrays being created for a typical prefix
+// treebitmap, at the cost of longer average search times. Two
+// implementations of this approach are Cuckoo hashing[^1], and Skip Lists.
+// Skip lists[^2] are a probabilistic data-structure, famously used by Redis,
+// (and by TiKv). I haven't tries either of these. Crossbeam has a SkipList
+// implementation, that wasn't ready at the time I wrote this. Cuckoo
+// hashing has the advantage of being easier to understand/implement. Maybe
+// Cuckoo hashing can also be combined with Fibonacci hashing[^3]. Note that
+// Robin Hood hashing maybe faster than Cuckoo hashing for reads, but it
+// requires shifting around existing entries, which is rather costly to do
+// atomically (and complex).
+
+// [^1]: [https://en.wikipedia.org/wiki/Cuckoo_hashing]
+// [^3]: [https://docs.rs/crossbeam-skiplist/0.1.1/crossbeam_skiplist/]
+// [^3]: [https://probablydance.com/2018/06/16/fibonacci-hashing-
+//  the-optimization-that-the-world-forgot-or-a-better-alternative-
+//  to-integer-modulo/]
+
+// Notes on memory leaks in Rotonda-store
+//
+// Both valgrind and miri report memory leaks on the multi-threaded prefix
+// store. Valgrind only reports it when it a binary stops using the tree,
+// while still keeping it around. An interrupted use of the mt-prefix-store
+// does not report any memory leaks. Miri is persistent in reporting memory
+// leaks in the mt-prefix-store. They both report the memory leaks in the same
+// location: the init method of the node- and prefix-buckets.
+//
+// I have reasons to believe these reported memory leaks aren't real, or that
+// crossbeam-epoch leaks a bit of memory when creating a new `Atomic`
+// instance. Since neither prefix nor node buckets can or should be dropped
+// this is not a big issue anyway, it just means that an `Atomic` occupies
+// more memory than it could in an optimal situation. Since we're not storing
+// the actual meta-data in an `Atomic` (it is stored in an `flurry Map`), this
+// means memory usage won't grow on updating the meta-data on a prefix,
+// (unless the meta-data itself grows of course, but that's up to the user).
+//
+// To get a better understanding on the nature of the reported memory leaks I
+// have created a branch (`vec_set`) that replaces the dynamically sized array
+// with a (equally sparse) Vec, that is not filled with `Atomic:::null()`, but
+// with `Option<StoredPrefix` instead, in order to see if this would eliminate
+// the memory leaks reporting. It did not. Valgrind still reports the memory
+// leaks at the same location, although they're now reported as `indirectly
+// leaked`, instead of `directly`. Miri is unchanged. The attentive reader may
+// now suspect that the `Atomic` inside the `AtomicStoredPrefix` may be the
+// culprit, but: a. Both miri and valgrind report the leaks still in the same
+// place: the creation of the buckets, not the creation of the stored
+//    prefixes. b. Tests that look at the memory usage of the prefix stores
+// under heavy modification of existing prefixes do not exhibit memory leaking
+//    behavior. c. Tests that look at memory usage under addition of new
+//    prefixes exhibit linear incrementation of memory usage (which is
+// expected). d. Tests that look at memory usage under contention do not
+//    exhibit increased memory usage either.
+//
+// My strong suspicion is that both Miri and Valgrind report the sparse slots
+// in the buckets as leaks, no matter whether they're `Atomic::null()`, or
+// `None` values, probably as a result of the way `crossbeam-epoch` indexes
+// into these, with pointer arithmetic (unsafe as hell).
+//
+// I would be super grateful if somebody would prove me wrong and can point to
+// an actual memory leak in the mt-prefix-store (and even more if they can
+// produce a fix for it).
+
+use crate::local_array::in_memory::atomic_stride::AtomicBitmap;
+use std::path::Path;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use log::{debug, info, log_enabled, trace};
+
+use crossbeam_epoch as epoch;
+use crossbeam_utils::Backoff;
+use epoch::{Guard, Owned};
+use lsm_tree::AbstractTree;
+
+use std::marker::PhantomData;
+
+use crate::local_array::in_memory::tree::{
+    SizedStrideNode, Stride, Stride3, Stride4, Stride5, StrideNodeId,
+    TreeBitMap,
+};
+use crate::local_array::types::PrefixId;
+use crate::stats::CreatedNodes;
+use crate::{
+    local_array::{bit_span::BitSpan, errors::PrefixStoreError},
+    prefix_record::PublicRecord,
+};
+
+use super::super::types::RouteStatus;
+use crate::local_array::in_memory::atomic_types::{
+    NodeBuckets, StoredPrefix,
+};
+use crate::local_array::in_memory::atomic_types::{NodeSet, StoredNode};
+use crate::local_array::in_memory::atomic_types::{PrefixBuckets, PrefixSet};
+use crate::local_array::in_memory::node::TreeBitMapNode;
+use crate::local_array::in_memory::tree::SizedStrideRef;
+
+// Make sure to also import the other methods for the Rib, so the proc macro
+// create_store can use them.
+pub use crate::local_array::iterators;
+pub use crate::local_array::query;
+
+use crate::{
+    impl_search_level, impl_search_level_for_mui, retrieve_node_mut_closure,
+    store_node_closure, Meta,
+};
+
+use crate::AddressFamily;
+
+//------------ Counters -----------------------------------------------------
+
+#[derive(Debug)]
+pub struct Counters {
+    nodes: AtomicUsize,
+    prefixes: [AtomicUsize; 129],
+}
+
+impl Counters {
+    pub fn get_nodes_count(&self) -> usize {
+        self.nodes.load(Ordering::Relaxed)
+    }
+
+    pub fn inc_nodes_count(&self) {
+        self.nodes.fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub fn get_prefixes_count(&self) -> Vec<usize> {
+        self.prefixes
+            .iter()
+            .map(|pc| pc.load(Ordering::Relaxed))
+            .collect::<Vec<_>>()
+    }
+
+    pub fn inc_prefixes_count(&self, len: u8) {
+        self.prefixes[len as usize].fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub fn get_prefix_stats(&self) -> Vec<CreatedNodes> {
+        self.prefixes
+            .iter()
+            .enumerate()
+            .filter_map(|(len, count)| {
+                let count = count.load(Ordering::Relaxed);
+                if count != 0 {
+                    Some(CreatedNodes {
+                        depth_level: len as u8,
+                        count,
+                    })
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+}
+
+impl Default for Counters {
+    fn default() -> Self {
+        let mut prefixes: Vec<AtomicUsize> = Vec::with_capacity(129);
+        for _ in 0..=128 {
+            prefixes.push(AtomicUsize::new(0));
+        }
+
+        Self {
+            nodes: AtomicUsize::new(0),
+            prefixes: prefixes.try_into().unwrap(),
+        }
+    }
+}
+
+//------------ StoreStats ----------------------------------------------
+
+#[derive(Debug)]
+pub struct StoreStats {
+    pub v4: Vec<CreatedNodes>,
+    pub v6: Vec<CreatedNodes>,
+}
+
+//------------ UpsertReport --------------------------------------------------
+
+#[derive(Debug)]
+pub struct UpsertReport {
+    // Indicates the number of Atomic Compare-and-Swap operations were
+    // necessary to create/update the Record entry. High numbers indicate
+    // contention.
+    pub cas_count: usize,
+    // Indicates whether this was the first mui record for this prefix was
+    // created. So, the prefix did not exist before hand.
+    pub prefix_new: bool,
+    // Indicates whether this mui was new for this prefix. False means an old
+    // value was overwritten.
+    pub mui_new: bool,
+    // The number of mui records for this prefix after the upsert operation.
+    pub mui_count: usize,
+}
+
+//------------ PersistTree ---------------------------------------------------
+
+pub struct PersistTree<
+    AF: AddressFamily,
+    // The size in bytes of the prefix in the peristed storage (disk), this
+    // amounnts to the bytes for the addres (4 for IPv4, 16 for IPv6) and 1
+    // bytefor the prefix length.
+    const PREFIX_SIZE: usize,
+    // The size in bytes of the complete key in the persisted storage, this
+    // is PREFIX_SIZE bytes (4; 16) + mui size (4) + ltime (8)
+    const KEY_SIZE: usize,
+>(lsm_tree::Tree, PhantomData<AF>);
+
+impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
+    PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
+{
+    pub fn insert(&self, key: [u8; KEY_SIZE], value: &[u8]) -> (u32, u32) {
+        self.0.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
+    }
+
+    pub fn get_records_for_prefix<M: Meta>(
+        &self,
+        prefix: PrefixId<AF>,
+    ) -> Vec<PublicRecord<M>> {
+        let prefix_b = &prefix.as_bytes::<PREFIX_SIZE>();
+        (*self.0.prefix(prefix_b))
+            .into_iter()
+            .map(|kv| {
+                let kv = kv.unwrap();
+                let (_, mui, ltime, status) = Self::parse_key(kv.0.as_ref());
+                PublicRecord::new(
+                    mui,
+                    ltime,
+                    status.try_into().unwrap(),
+                    kv.1.as_ref().to_vec().into(),
+                )
+            })
+            .collect::<Vec<_>>()
+    }
+
+    pub fn get_records_for_key<M: Meta + From<Vec<u8>>>(
+        &self,
+        key: &[u8],
+    ) -> Vec<(inetnum::addr::Prefix, PublicRecord<M>)> {
+        (*self.0.prefix(key))
+            .into_iter()
+            .map(|kv| {
+                let kv = kv.unwrap();
+                let (pfx, mui, ltime, status) =
+                    Self::parse_key(kv.0.as_ref());
+
+                (
+                    PrefixId::<AF>::from(pfx).into_pub(),
+                    PublicRecord::new(
+                        mui,
+                        ltime,
+                        status.try_into().unwrap(),
+                        kv.1.as_ref().to_vec().into(),
+                    ),
+                )
+            })
+            .collect::<Vec<_>>()
+    }
+
+    pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
+        let segment = self.0.flush_active_memtable(0);
+
+        if let Ok(Some(segment)) = segment {
+            self.0.register_segments(&[segment])?;
+            self.0.compact(
+                std::sync::Arc::new(lsm_tree::compaction::Leveled::default()),
+                0,
+            )?;
+        };
+
+        Ok(())
+    }
+
+    pub fn approximate_len(&self) -> usize {
+        self.0.approximate_len()
+    }
+
+    pub fn disk_space(&self) -> u64 {
+        self.0.disk_space()
+    }
+
+    #[cfg(feature = "persist")]
+    pub fn persistence_key(
+        // PREFIX_SIZE bytes
+        prefix_id: PrefixId<AF>,
+        // 4 bytes
+        mui: u32,
+        // 8 bytes
+        ltime: u64,
+        // 1 byte
+        status: RouteStatus,
+    ) -> [u8; KEY_SIZE] {
+        assert!(KEY_SIZE > PREFIX_SIZE);
+        let key = &mut [0_u8; KEY_SIZE];
+
+        // prefix 5 or 17 bytes
+        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
+
+        // mui 4 bytes
+        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
+            .first_chunk_mut::<4>()
+            .unwrap() = mui.to_le_bytes();
+
+        // ltime 8 bytes
+        *key[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
+            .first_chunk_mut::<8>()
+            .unwrap() = ltime.to_le_bytes();
+
+        // status 1 byte
+        key[PREFIX_SIZE + 12] = status.into();
+
+        *key
+    }
+
+    #[cfg(feature = "persist")]
+    pub fn prefix_mui_persistence_key(
+        prefix_id: PrefixId<AF>,
+        mui: u32,
+    ) -> Vec<u8> {
+        let mut key = vec![];
+        // prefix 5 or 17 bytes
+        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
+
+        // mui 4 bytes
+        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
+            .first_chunk_mut::<4>()
+            .unwrap() = mui.to_le_bytes();
+
+        key
+    }
+
+    #[cfg(feature = "persist")]
+    pub fn parse_key(bytes: &[u8]) -> ([u8; PREFIX_SIZE], u32, u64, u8) {
+        (
+            // prefix 5 or 17 bytes
+            *bytes.first_chunk::<PREFIX_SIZE>().unwrap(),
+            // mui 4 bytes
+            u32::from_le_bytes(
+                *bytes[PREFIX_SIZE..PREFIX_SIZE + 4]
+                    .first_chunk::<4>()
+                    .unwrap(),
+            ),
+            // ltime 8 bytes
+            u64::from_le_bytes(
+                *bytes[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
+                    .first_chunk::<8>()
+                    .unwrap(),
+            ),
+            // status 1 byte
+            bytes[PREFIX_SIZE + 12],
+        )
+    }
+
+    #[cfg(feature = "persist")]
+    pub fn parse_prefix(bytes: &[u8]) -> [u8; PREFIX_SIZE] {
+        *bytes.first_chunk::<PREFIX_SIZE>().unwrap()
+    }
+
+    #[cfg(feature = "persist")]
+    pub(crate) fn persist_record<M: Meta>(
+        &self,
+        prefix: PrefixId<AF>,
+        mui: u32,
+        record: &PublicRecord<M>,
+    ) {
+        self.insert(
+            PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
+                prefix,
+                mui,
+                record.ltime,
+                record.status,
+            ),
+            record.meta.as_ref(),
+        );
+    }
+}
+
+impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
+    std::fmt::Debug for PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
+{
+    fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        todo!()
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum PersistStrategy {
+    WriteAhead,
+    PersistHistory,
+    MemoryOnly,
+    PersistOnly,
+}
+
+#[derive(Debug, Clone)]
+pub struct StoreConfig {
+    pub persist_strategy: PersistStrategy,
+    pub persist_path: String,
+}
+
+impl StoreConfig {
+    pub fn persist_strategy(&self) -> PersistStrategy {
+        self.persist_strategy
+    }
+}
+
+// ----------- CustomAllocStorage -------------------------------------------
+//
+// CustomAllocStorage is a storage backend that uses a custom allocator, that
+// consists of arrays that point to other arrays on collision.
+#[derive(Debug)]
+pub struct Rib<
+    AF: AddressFamily,
+    M: Meta,
+    NB: NodeBuckets<AF>,
+    PB: PrefixBuckets<AF, M>,
+    const PREFIX_SIZE: usize,
+    const KEY_SIZE: usize,
+> {
+    pub config: StoreConfig,
+    pub(crate) in_memory_tree: TreeBitMap<AF, M, NB, PB>,
+    #[cfg(feature = "persist")]
+    pub persist_tree: Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
+    // Global Roaring BitMap INdex that stores MUIs.
+    pub counters: Counters,
+}
+
+impl<
+        'a,
+        AF: AddressFamily,
+        M: crate::prefix_record::Meta,
+        NB: NodeBuckets<AF>,
+        PB: PrefixBuckets<AF, M>,
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+{
+    pub(crate) fn init(
+        root_node: SizedStrideNode<AF>,
+        config: StoreConfig,
+    ) -> Result<Self, Box<dyn std::error::Error>> {
+        info!("store: initialize store {}", AF::BITS);
+
+        let persistence = match config.persist_strategy {
+            PersistStrategy::MemoryOnly => None,
+            _ => {
+                let persist_path = Path::new(&config.persist_path);
+                Some(PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>(
+                    lsm_tree::Config::new(persist_path).open()?,
+                    PhantomData,
+                ))
+            }
+        };
+
+        let store = Rib {
+            config,
+            in_memory_tree: TreeBitMap::<AF, M, NB, PB>::init(),
+            persist_tree: persistence,
+            counters: Counters::default(),
+        };
+
+        let _retry_count = store.store_node(
+            StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0),
+            0_u32,
+            root_node,
+        )?;
+
+        Ok(store)
+    }
+
+    pub(crate) fn acquire_new_node_id(
+        &self,
+        (prefix_net, sub_prefix_len): (AF, u8),
+    ) -> StrideNodeId<AF> {
+        StrideNodeId::new_with_cleaned_id(prefix_net, sub_prefix_len)
+    }
+
+    // Create a new node in the store with payload `next_node`.
+    //
+    // Next node will be ignored if a node with the same `id` already exists,
+    // but the multi_uniq_id will be added to the rbm_index of the NodeSet.
+    //
+    // Returns: a tuple with the node_id of the created node and the number of
+    // retry_count
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn store_node(
+        &self,
+        id: StrideNodeId<AF>,
+        multi_uniq_id: u32,
+        next_node: SizedStrideNode<AF>,
+    ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
+        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+            f: &'s dyn Fn(
+                &SearchLevel<AF, S>,
+                &NodeSet<AF, S>,
+                TreeBitMapNode<AF, S>,
+                u32, // multi_uniq_id
+                u8,  // the store level
+                u32, // retry_count
+            ) -> Result<
+                (StrideNodeId<AF>, u32),
+                PrefixStoreError,
+            >,
+        }
+
+        let search_level_3 =
+            store_node_closure![Stride3; id; guard; back_off;];
+        let search_level_4 =
+            store_node_closure![Stride4; id; guard; back_off;];
+        let search_level_5 =
+            store_node_closure![Stride5; id; guard; back_off;];
+
+        if log_enabled!(log::Level::Trace) {
+            debug!(
+                "{} store: Store node {}: {:?} mui {}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                id,
+                next_node,
+                multi_uniq_id
+            );
+        }
+        self.counters.inc_nodes_count();
+
+        match next_node {
+            SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
+                &search_level_3,
+                self.in_memory_tree.node_buckets.get_store3(id),
+                new_node,
+                multi_uniq_id,
+                0,
+                0,
+            ),
+            SizedStrideNode::Stride4(new_node) => (search_level_4.f)(
+                &search_level_4,
+                self.in_memory_tree.node_buckets.get_store4(id),
+                new_node,
+                multi_uniq_id,
+                0,
+                0,
+            ),
+            SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
+                &search_level_5,
+                self.in_memory_tree.node_buckets.get_store5(id),
+                new_node,
+                multi_uniq_id,
+                0,
+                0,
+            ),
+        }
+    }
+
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn retrieve_node(
+        &'a self,
+        id: StrideNodeId<AF>,
+    ) -> Option<SizedStrideRef<'a, AF>> {
+        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+            f: &'s dyn for<'a> Fn(
+                &SearchLevel<AF, S>,
+                &'a NodeSet<AF, S>,
+                u8,
+            )
+                -> Option<SizedStrideRef<'a, AF>>,
+        }
+
+        let search_level_3 = impl_search_level![Stride3; id;];
+        let search_level_4 = impl_search_level![Stride4; id;];
+        let search_level_5 = impl_search_level![Stride5; id;];
+
+        if log_enabled!(log::Level::Trace) {
+            trace!(
+                "{} store: Retrieve node {} from l{}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                id,
+                id.get_id().1
+            );
+        }
+
+        match self.get_stride_for_id(id) {
+            3 => (search_level_3.f)(
+                &search_level_3,
+                self.in_memory_tree.node_buckets.get_store3(id),
+                0,
+            ),
+            4 => (search_level_4.f)(
+                &search_level_4,
+                self.in_memory_tree.node_buckets.get_store4(id),
+                0,
+            ),
+            _ => (search_level_5.f)(
+                &search_level_5,
+                self.in_memory_tree.node_buckets.get_store5(id),
+                0,
+            ),
+        }
+    }
+
+    // retrieve a node, but only its bitmap index contains the specified mui.
+    // Used for iterators per mui.
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn retrieve_node_for_mui(
+        &'a self,
+        id: StrideNodeId<AF>,
+        // The mui that is tested to be present in the nodes bitmap index
+        mui: u32,
+    ) -> Option<SizedStrideRef<'a, AF>> {
+        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+            f: &'s dyn for<'a> Fn(
+                &SearchLevel<AF, S>,
+                &'a NodeSet<AF, S>,
+                u8,
+            )
+                -> Option<SizedStrideRef<'a, AF>>,
+        }
+
+        let search_level_3 = impl_search_level_for_mui![Stride3; id; mui;];
+        let search_level_4 = impl_search_level_for_mui![Stride4; id; mui;];
+        let search_level_5 = impl_search_level_for_mui![Stride5; id; mui;];
+
+        if log_enabled!(log::Level::Trace) {
+            trace!(
+                "{} store: Retrieve node {} from l{} for mui {}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                id,
+                id.get_id().1,
+                mui
+            );
+        }
+
+        match self.get_stride_for_id(id) {
+            3 => (search_level_3.f)(
+                &search_level_3,
+                self.in_memory_tree.node_buckets.get_store3(id),
+                0,
+            ),
+            4 => (search_level_4.f)(
+                &search_level_4,
+                self.in_memory_tree.node_buckets.get_store4(id),
+                0,
+            ),
+            _ => (search_level_5.f)(
+                &search_level_5,
+                self.in_memory_tree.node_buckets.get_store5(id),
+                0,
+            ),
+        }
+    }
+
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn retrieve_node_mut(
+        &'a self,
+        id: StrideNodeId<AF>,
+        multi_uniq_id: u32,
+    ) -> Option<SizedStrideRef<'a, AF>> {
+        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+            f: &'s dyn for<'a> Fn(
+                &SearchLevel<AF, S>,
+                &'a NodeSet<AF, S>,
+                u8,
+            )
+                -> Option<SizedStrideRef<'a, AF>>,
+        }
+
+        let search_level_3 =
+            retrieve_node_mut_closure![Stride3; id; multi_uniq_id;];
+        let search_level_4 =
+            retrieve_node_mut_closure![Stride4; id; multi_uniq_id;];
+        let search_level_5 =
+            retrieve_node_mut_closure![Stride5; id; multi_uniq_id;];
+
+        if log_enabled!(log::Level::Trace) {
+            trace!(
+                "{} store: Retrieve node mut {} from l{}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                id,
+                id.get_id().1
+            );
+        }
+
+        match self.in_memory_tree.node_buckets.get_stride_for_id(id) {
+            3 => (search_level_3.f)(
+                &search_level_3,
+                self.in_memory_tree.node_buckets.get_store3(id),
+                0,
+            ),
+
+            4 => (search_level_4.f)(
+                &search_level_4,
+                self.in_memory_tree.node_buckets.get_store4(id),
+                0,
+            ),
+            _ => (search_level_5.f)(
+                &search_level_5,
+                self.in_memory_tree.node_buckets.get_store5(id),
+                0,
+            ),
+        }
+    }
+
+    pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
+        StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0)
+    }
+
+    pub fn get_nodes_count(&self) -> usize {
+        self.counters.get_nodes_count()
+    }
+
+    // Prefixes related methods
+
+    // THE CRITICAL SECTION
+    //
+    // CREATING OR UPDATING A PREFIX IN THE STORE
+    //
+    // YES, THE MAGIC HAPPENS HERE!
+    //
+    // This uses the TAG feature of crossbeam_utils::epoch to ensure that we
+    // are not overwriting a prefix meta-data that already has been created
+    // or was updated by another thread.
+    //
+    // Our plan:
+    //
+    // 1. LOAD
+    //    Load the current prefix and meta-data from the store if any.
+    // 2. INSERT
+    //    If there is no current meta-data, create it.
+    // 3. UPDATE
+    //    If there is a prefix, meta-data combo, then load it and merge
+    //    the existing meta-dat with our meta-data using the `MergeUpdate`
+    //    trait (a so-called 'Read-Copy-Update').
+    // 4. SUCCESS
+    //    See if we can successfully store the updated meta-data in the store.
+    // 5. DONE
+    //    If Step 4 succeeded we're done!
+    // 6. FAILURE - REPEAT
+    //    If Step 4 failed we're going to do the whole thing again.
+
+    pub(crate) fn upsert_prefix(
+        &self,
+        prefix: PrefixId<AF>,
+        record: PublicRecord<M>,
+        update_path_selections: Option<M::TBI>,
+        guard: &Guard,
+    ) -> Result<UpsertReport, PrefixStoreError> {
+        let mut prefix_is_new = true;
+        let mut mui_is_new = true;
+
+        let (mui_count, cas_count) =
+            match self.non_recursive_retrieve_prefix_mut(prefix) {
+                // There's no StoredPrefix at this location yet. Create a new
+                // PrefixRecord and try to store it in the empty slot.
+                (locked_prefix, false) => {
+                    if log_enabled!(log::Level::Debug) {
+                        debug!(
+                            "{} store: Create new prefix record",
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread")
+                        );
+                    }
+
+                    let (mui_count, retry_count) =
+                        locked_prefix.record_map.upsert_record(
+                            prefix,
+                            record,
+                            &self.persist_tree,
+                            self.config.persist_strategy,
+                        )?;
+
+                    // See if someone beat us to creating the record.
+                    if mui_count.is_some() {
+                        mui_is_new = false;
+                        prefix_is_new = false;
+                    } else {
+                        // No, we were the first, we created a new prefix
+                        self.counters.inc_prefixes_count(prefix.get_len());
+                    }
+
+                    (mui_count, retry_count)
+                }
+                // There already is a StoredPrefix with a record at this
+                // location.
+                (stored_prefix, true) => {
+                    if log_enabled!(log::Level::Debug) {
+                        debug!(
+                        "{} store: Found existing prefix record for {}/{}",
+                        std::thread::current()
+                            .name()
+                            .unwrap_or("unnamed-thread"),
+                        prefix.get_net(),
+                        prefix.get_len()
+                    );
+                    }
+                    prefix_is_new = false;
+
+                    // Update the already existing record_map with our
+                    // caller's record.
+                    stored_prefix.set_ps_outdated(guard)?;
+
+                    let (mui_count, retry_count) =
+                        stored_prefix.record_map.upsert_record(
+                            prefix,
+                            record,
+                            &self.persist_tree,
+                            self.config.persist_strategy,
+                        )?;
+                    mui_is_new = mui_count.is_none();
+
+                    if let Some(tbi) = update_path_selections {
+                        stored_prefix
+                            .calculate_and_store_best_backup(&tbi, guard)?;
+                    }
+
+                    (mui_count, retry_count)
+                }
+            };
+
+        Ok(UpsertReport {
+            prefix_new: prefix_is_new,
+            cas_count,
+            mui_new: mui_is_new,
+            mui_count: mui_count.unwrap_or(1),
+        })
+    }
+
+    // Change the status of the record for the specified (prefix, mui)
+    // combination  to Withdrawn.
+    pub fn mark_mui_as_withdrawn_for_prefix(
+        &self,
+        prefix: PrefixId<AF>,
+        mui: u32,
+    ) -> Result<(), PrefixStoreError> {
+        let (stored_prefix, exists) =
+            self.non_recursive_retrieve_prefix_mut(prefix);
+
+        if !exists {
+            return Err(PrefixStoreError::StoreNotReadyError);
+        }
+
+        stored_prefix.record_map.mark_as_withdrawn_for_mui(mui);
+
+        Ok(())
+    }
+
+    // Change the status of the record for the specified (prefix, mui)
+    // combination  to Active.
+    pub fn mark_mui_as_active_for_prefix(
+        &self,
+        prefix: PrefixId<AF>,
+        mui: u32,
+    ) -> Result<(), PrefixStoreError> {
+        let (stored_prefix, exists) =
+            self.non_recursive_retrieve_prefix_mut(prefix);
+
+        if !exists {
+            return Err(PrefixStoreError::StoreNotReadyError);
+        }
+
+        stored_prefix.record_map.mark_as_active_for_mui(mui);
+
+        Ok(())
+    }
+
+    // Change the status of the mui globally to Withdrawn. Iterators and match
+    // functions will by default not return any records for this mui.
+    // pub fn mark_mui_as_withdrawn(
+    //     &self,
+    //     mui: u32,
+    //     guard: &Guard,
+    // ) -> Result<(), PrefixStoreError> {
+    //     let current = self
+    //         .in_memory_tree
+    //         .withdrawn_muis_bmin
+    //         .load(Ordering::Acquire, guard);
+
+    //     let mut new = unsafe { current.as_ref() }.unwrap().clone();
+    //     new.insert(mui);
+
+    //     #[allow(clippy::assigning_clones)]
+    //     loop {
+    //         match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
+    //             current,
+    //             Owned::new(new),
+    //             Ordering::AcqRel,
+    //             Ordering::Acquire,
+    //             guard,
+    //         ) {
+    //             Ok(_) => return Ok(()),
+    //             Err(updated) => {
+    //                 new =
+    //                     unsafe { updated.current.as_ref() }.unwrap().clone();
+    //             }
+    //         }
+    //     }
+    // }
+
+    // Change the status of the mui globally to Active. Iterators and match
+    // functions will default to the status on the record itself.
+    // pub fn mark_mui_as_active(
+    //     &self,
+    //     mui: u32,
+    //     guard: &Guard,
+    // ) -> Result<(), PrefixStoreError> {
+    //     let current = self
+    //         .in_memory_tree
+    //         .withdrawn_muis_bmin
+    //         .load(Ordering::Acquire, guard);
+
+    //     let mut new = unsafe { current.as_ref() }.unwrap().clone();
+    //     new.remove(mui);
+
+    //     #[allow(clippy::assigning_clones)]
+    //     loop {
+    //         match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
+    //             current,
+    //             Owned::new(new),
+    //             Ordering::AcqRel,
+    //             Ordering::Acquire,
+    //             guard,
+    //         ) {
+    //             Ok(_) => return Ok(()),
+    //             Err(updated) => {
+    //                 new =
+    //                     unsafe { updated.current.as_ref() }.unwrap().clone();
+    //             }
+    //         }
+    //     }
+    // }
+
+    // Whether this mui is globally withdrawn. Note that this overrules (by
+    // default) any (prefix, mui) combination in iterators and match functions.
+    // pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
+    //     unsafe {
+    //         self.in_memory_tree
+    //             .withdrawn_muis_bmin
+    //             .load(Ordering::Acquire, guard)
+    //             .as_ref()
+    //     }
+    //     .unwrap()
+    //     .contains(mui)
+    // }
+
+    // Whether this mui is globally active. Note that the local statuses of
+    // records (prefix, mui) may be set to withdrawn in iterators and match
+    // functions.
+    // pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
+    //     !unsafe {
+    //         self.in_memory_tree
+    //             .withdrawn_muis_bmin
+    //             .load(Ordering::Acquire, guard)
+    //             .as_ref()
+    //     }
+    //     .unwrap()
+    //     .contains(mui)
+    // }
+
+    // This function is used by the upsert_prefix function above.
+    //
+    // We're using a Chained Hash Table and this function returns one of:
+    // - a StoredPrefix that already exists for this search_prefix_id
+    // - the Last StoredPrefix in the chain.
+    // - an error, if no StoredPrefix whatsoever can be found in the store.
+    //
+    // The error condition really shouldn't happen, because that basically
+    // means the root node for that particular prefix length doesn't exist.
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn non_recursive_retrieve_prefix_mut(
+        &'a self,
+        search_prefix_id: PrefixId<AF>,
+    ) -> (&'a StoredPrefix<AF, M>, bool) {
+        trace!("non_recursive_retrieve_prefix_mut_with_guard");
+        let mut prefix_set = self
+            .in_memory_tree
+            .prefix_buckets
+            .get_root_prefix_set(search_prefix_id.get_len());
+        let mut level: u8 = 0;
+
+        trace!("root prefix_set {:?}", prefix_set);
+        loop {
+            // HASHING FUNCTION
+            let index = Self::hash_prefix_id(search_prefix_id, level);
+
+            // probe the slot with the index that's the result of the hashing.
+            // let locked_prefix = prefix_set.0.get(index);
+            let stored_prefix = match prefix_set.0.get(index) {
+                Some(p) => {
+                    trace!("prefix set found.");
+                    (p, true)
+                }
+                None => {
+                    // We're at the end of the chain and haven't found our
+                    // search_prefix_id anywhere. Return the end-of-the-chain
+                    // StoredPrefix, so the caller can attach a new one.
+                    trace!(
+                        "no record. returning last found record in level
+                        {}, with index {}.",
+                        level,
+                        index
+                    );
+                    let index = Self::hash_prefix_id(search_prefix_id, level);
+                    trace!("calculate next index {}", index);
+                    let var_name = (
+                        prefix_set
+                            .0
+                            .get_or_init(index, || {
+                                StoredPrefix::new::<PB>(
+                                    PrefixId::new(
+                                        search_prefix_id.get_net(),
+                                        search_prefix_id.get_len(),
+                                    ),
+                                    level,
+                                )
+                            })
+                            .0,
+                        false,
+                    );
+                    var_name
+                }
+            };
+
+            if search_prefix_id == stored_prefix.0.prefix {
+                // GOTCHA!
+                // Our search-prefix is stored here, so we're returning
+                // it, so its PrefixRecord can be updated by the caller.
+                trace!("found requested prefix {:?}", search_prefix_id);
+                return stored_prefix;
+            } else {
+                // A Collision. Follow the chain.
+                level += 1;
+                prefix_set = &stored_prefix.0.next_bucket;
+                continue;
+            }
+        }
+    }
+
+    // This function is used by the match_prefix, and [more|less]_specifics
+    // public methods on the TreeBitMap (indirectly).
+    #[allow(clippy::type_complexity)]
+    pub fn non_recursive_retrieve_prefix(
+        &'a self,
+        id: PrefixId<AF>,
+    ) -> (
+        Option<&'a StoredPrefix<AF, M>>,
+        Option<(
+            PrefixId<AF>,
+            u8,
+            &'a PrefixSet<AF, M>,
+            [Option<(&'a PrefixSet<AF, M>, usize)>; 32],
+            usize,
+        )>,
+    ) {
+        let mut prefix_set = self
+            .in_memory_tree
+            .prefix_buckets
+            .get_root_prefix_set(id.get_len());
+        let mut parents = [None; 32];
+        let mut level: u8 = 0;
+        let backoff = Backoff::new();
+
+        loop {
+            // The index of the prefix in this array (at this len and
+            // level) is calculated by performing the hash function
+            // over the prefix.
+
+            // HASHING FUNCTION
+            let index = Self::hash_prefix_id(id, level);
+
+            if let Some(stored_prefix) = prefix_set.0.get(index) {
+                if id == stored_prefix.get_prefix_id() {
+                    trace!("found requested prefix {:?}", id);
+                    parents[level as usize] = Some((prefix_set, index));
+                    return (
+                        Some(stored_prefix),
+                        Some((id, level, prefix_set, parents, index)),
+                    );
+                };
+                // Advance to the next level.
+
+                prefix_set = &stored_prefix.next_bucket;
+                level += 1;
+                backoff.spin();
+                continue;
+            }
+
+            trace!("no prefix found for {:?}", id);
+            parents[level as usize] = Some((prefix_set, index));
+            return (None, Some((id, level, prefix_set, parents, index)));
+        }
+    }
+
+    #[deprecated]
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn retrieve_prefix(
+        &'a self,
+        prefix_id: PrefixId<AF>,
+    ) -> Option<(&'a StoredPrefix<AF, M>, usize)> {
+        struct SearchLevel<
+            's,
+            AF: AddressFamily,
+            M: crate::prefix_record::Meta,
+        > {
+            f: &'s dyn for<'a> Fn(
+                &SearchLevel<AF, M>,
+                &'a PrefixSet<AF, M>,
+                u8,
+            )
+                -> Option<(&'a StoredPrefix<AF, M>, usize)>,
+        }
+
+        let search_level = SearchLevel {
+            f: &|search_level: &SearchLevel<AF, M>,
+                 prefix_set: &PrefixSet<AF, M>,
+                 mut level: u8| {
+                // HASHING FUNCTION
+                let index = Self::hash_prefix_id(prefix_id, level);
+
+                if let Some(stored_prefix) = prefix_set.0.get(index) {
+                    if prefix_id == stored_prefix.prefix {
+                        trace!("found requested prefix {:?}", prefix_id,);
+                        return Some((stored_prefix, 0));
+                    };
+                    level += 1;
+
+                    (search_level.f)(
+                        search_level,
+                        &stored_prefix.next_bucket,
+                        level,
+                    );
+                }
+                None
+            },
+        };
+
+        (search_level.f)(
+            &search_level,
+            self.in_memory_tree
+                .prefix_buckets
+                .get_root_prefix_set(prefix_id.get_len()),
+            0,
+        )
+    }
+
+    #[allow(dead_code)]
+    fn remove_prefix(&mut self, index: PrefixId<AF>) -> Option<M> {
+        match index.is_empty() {
+            false => self.in_memory_tree.prefix_buckets.remove(index),
+            true => None,
+        }
+    }
+
+    pub fn get_prefixes_count(&self) -> usize {
+        self.counters.get_prefixes_count().iter().sum()
+    }
+
+    pub fn get_prefixes_count_for_len(&self, len: u8) -> usize {
+        self.counters.get_prefixes_count()[len as usize]
+    }
+
+    // Stride related methods
+
+    pub(crate) fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8 {
+        self.in_memory_tree.node_buckets.get_stride_for_id(id)
+    }
+
+    pub fn get_stride_sizes(&self) -> &[u8] {
+        self.in_memory_tree.node_buckets.get_stride_sizes()
+    }
+
+    // pub(crate) fn get_strides_len() -> u8 {
+    //     NB::get_strides_len()
+    // }
+
+    pub(crate) fn get_first_stride_size() -> u8 {
+        NB::get_first_stride_size()
+    }
+
+    // Calculates the id of the node that COULD host a prefix in its
+    // ptrbitarr.
+    pub(crate) fn get_node_id_for_prefix(
+        &self,
+        prefix: &PrefixId<AF>,
+    ) -> (StrideNodeId<AF>, BitSpan) {
+        let mut acc = 0;
+        for i in self.get_stride_sizes() {
+            acc += *i;
+            if acc >= prefix.get_len() {
+                let node_len = acc - i;
+                return (
+                    StrideNodeId::new_with_cleaned_id(
+                        prefix.get_net(),
+                        node_len,
+                    ),
+                    // NOT THE HASHING FUNCTION!
+                    // Do the right shift in a checked manner, for the sake
+                    // of 0/0. A search for 0/0 will perform a 0 << MAX_LEN,
+                    // which will panic in debug mode (undefined behaviour
+                    // in prod).
+                    BitSpan::new(
+                        ((prefix.get_net() << node_len).checked_shr_or_zero(
+                            (AF::BITS - (prefix.get_len() - node_len)).into(),
+                        ))
+                        .dangerously_truncate_to_u32(),
+                        prefix.get_len() - node_len,
+                    ),
+                );
+            }
+        }
+        panic!("prefix length for {:?} is too long", prefix);
+    }
+
+    // ------- THE HASHING FUNCTION -----------------------------------------
+
+    // Ok, so hashing is really hard, but we're keeping it simple, and
+    // because we're keeping it simple we're having lots of collisions, but
+    // we don't care!
+    //
+    // We're using a part of bitarray representation of the address part of
+    // a prefix the as the hash. Sounds complicated, but isn't.
+    // Suppose we have an IPv4 prefix, say 130.24.55.0/24.
+    // The address part is 130.24.55.0 or as a bitarray that would be:
+    //
+    // pos  0    4    8    12   16   20   24   28
+    // bit  1000 0010 0001 1000 0011 0111 0000 0000
+    //
+    // First, we're discarding the bits after the length of the prefix, so
+    // we'll have:
+    //
+    // pos  0    4    8    12   16   20
+    // bit  1000 0010 0001 1000 0011 0111
+    //
+    // Now we're dividing this bitarray into one or more levels. A level can
+    // be an arbitrary number of bits between 1 and the length of the prefix,
+    // but the number of bits summed over all levels should be exactly the
+    // prefix length. So in our case they should add up to 24. A possible
+    // division could be: 4, 4, 4, 4, 4, 4. Another one would be: 12, 12. The
+    // actual division being used is described in the function
+    // `<NB>::get_bits_for_len` in the `rotonda-macros` crate. Each level has
+    // its own hash, so for our example prefix this would be:
+    //
+    // pos   0    4    8    12   16   20
+    // level 0              1
+    // hash  1000 0010 0001 1000 0011 0111
+    //
+    // level 1 hash: 1000 0010 0001
+    // level 2 hash: 1000 0011 0011
+    //
+    // The hash is now converted to a usize integer, by shifting it all the
+    // way to the right in a u32 and then converting to a usize. Why a usize
+    // you ask? Because the hash is used by the CustomAllocStorage as the
+    // index to the array for that specific prefix length and level.
+    // So for our example this means that the hash on level 1 is now 0x821
+    // (decimal 2081) and the hash on level 2 is 0x833 (decimal 2099).
+    // Now, if we only consider the hash on level 1 and that we're going to
+    // use that as the index to the array that stores all prefixes, you'll
+    // notice very quickly that all prefixes starting with 130.[16..31] will
+    // cause a collision: they'll all point to the same array element. These
+    // collisions are resolved by creating a linked list from each array
+    // element, where each element in the list has an array of its own that
+    // uses the hash function with the level incremented.
+
+    pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
+        // And, this is all of our hashing function.
+        let last_level = if level > 0 {
+            <NB>::len_to_store_bits(id.get_id().1, level - 1)
+        } else {
+            0
+        };
+        let this_level = <NB>::len_to_store_bits(id.get_id().1, level);
+        trace!("bits division {}", this_level);
+        trace!(
+            "calculated index ({} << {}) >> {}",
+            id.get_id().0,
+            last_level,
+            ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
+        );
+        // HASHING FUNCTION
+        ((id.get_id().0 << last_level)
+            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
+            .dangerously_truncate_to_u32() as usize
+    }
+
+    pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
+        // And, this is all of our hashing function.
+        let last_level = if level > 0 {
+            <PB>::get_bits_for_len(id.get_len(), level - 1)
+        } else {
+            0
+        };
+        let this_level = <PB>::get_bits_for_len(id.get_len(), level);
+        trace!(
+            "bits division {}; no of bits {}",
+            this_level,
+            this_level - last_level
+        );
+        trace!(
+            "calculated index ({} << {}) >> {}",
+            id.get_net(),
+            last_level,
+            ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
+        );
+        // HASHING FUNCTION
+        ((id.get_net() << last_level)
+            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
+            .dangerously_truncate_to_u32() as usize
+    }
+}
diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
index bcc8a1ee..e0e1ff40 100644
--- a/src/local_array/store/atomic_types.rs
+++ b/src/local_array/store/atomic_types.rs
@@ -48,6 +48,7 @@ pub struct NodeSet<AF: AddressFamily, S: Stride>(
 );
 
 impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
+<<<<<<< Updated upstream
     pub fn init(p2_size: u8) -> Self {
         if log_enabled!(log::Level::Debug) {
             debug!(
@@ -58,6 +59,22 @@ impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
         }
 
         NodeSet(OnceBoxSlice::new(p2_size), RoaringBitmap::new().into())
+=======
+    pub fn init(pow2_size: u8) -> Self {
+        if log_enabled!(log::Level::Debug) {
+            debug!(
+                "{} store: creating space for {} nodes",
+                std::thread::current().name().unwrap(),
+                1 << pow2_size
+            );
+        }
+
+        // let mut l = vec![];
+        // for _i in 0..size {
+        //     l.push(OnceBox::new());
+        // }
+        NodeSet(OnceBoxSlice::new(pow2_size), RoaringBitmap::new().into())
+>>>>>>> Stashed changes
     }
 
     pub fn update_rbm_index(
@@ -155,7 +172,11 @@ impl<AF: AddressFamily, M: crate::prefix_record::Meta> StoredPrefix<AF, M> {
                 std::thread::current().name().unwrap_or("unnamed-thread"),
                 pfx_id.get_len()
             );
+<<<<<<< Updated upstream
             PrefixSet::init(next_level - this_level)
+=======
+            PrefixSet::empty(next_level - this_level)
+>>>>>>> Stashed changes
         };
         // End of calculation
 
@@ -690,18 +711,50 @@ pub struct PrefixSet<AF: AddressFamily, M: Meta>(
 );
 
 impl<AF: AddressFamily, M: Meta> PrefixSet<AF, M> {
+<<<<<<< Updated upstream
     pub fn init(p2_size: u8) -> Self {
         PrefixSet(OnceBoxSlice::new(p2_size))
+=======
+    pub fn init(pow2_size: u8) -> Self {
+        // let mut l = Vec::with_capacity(size);
+
+        // let l = AtomicStoredPrefix<AF, M>>>::new(vec![
+        // AtomicStoredPrefix::empty(),
+        // ]);
+        trace!("creating space for {} prefixes in prefix_set", pow2_size);
+        // for _i in 0..size {
+        //     l.push(OnceBox::new());
+        // }
+        PrefixSet(OnceBoxSlice::new(pow2_size))
+>>>>>>> Stashed changes
     }
 
     pub(crate) fn is_empty(&self) -> bool {
+<<<<<<< Updated upstream
+=======
+        // if self.0.len() == 1 {
+        //     true
+        // } else {
+        //     false
+        // }
+>>>>>>> Stashed changes
         self.0.is_null()
     }
 
     pub(crate) fn get_by_index(
         &self,
         index: usize,
+<<<<<<< Updated upstream
     ) -> Option<&StoredPrefix<AF, M>> {
         self.0.get(index)
+=======
+        _guard: &'a Guard,
+    ) -> Option<&'a StoredPrefix<AF, M>> {
+        self.0.get(index)
+    }
+
+    pub(crate) fn empty(pow2_size: u8) -> Self {
+        PrefixSet(OnceBoxSlice::new(pow2_size))
+>>>>>>> Stashed changes
     }
 }
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
index 5e09615d..74375400 100644
--- a/src/local_array/store/custom_alloc.rs
+++ b/src/local_array/store/custom_alloc.rs
@@ -198,6 +198,10 @@ use std::marker::PhantomData;
 
 use crate::{local_array::tree::*, stats::CreatedNodes};
 use crate::{
+<<<<<<< Updated upstream
+=======
+    local_array::store::oncebox::{OnceBox, OnceBoxSlice},
+>>>>>>> Stashed changes
     local_array::{bit_span::BitSpan, store::errors::PrefixStoreError},
     prefix_record::PublicRecord,
 };
@@ -879,6 +883,7 @@ impl<
                         );
                     }
 
+<<<<<<< Updated upstream
                     let (mui_count, retry_count) =
                         locked_prefix.record_map.upsert_record(
                             prefix,
@@ -886,6 +891,14 @@ impl<
                             &self.persistence,
                             self.config.persist_strategy,
                         )?;
+=======
+                    // We're creating a StoredPrefix without our record first,
+                    // to avoid having to clone it on retry.
+                    let this_level =
+                        PB::get_bits_for_len(prefix.get_len(), level);
+                    let next_level =
+                        PB::get_bits_for_len(prefix.get_len(), level + 1);
+>>>>>>> Stashed changes
 
                     // See if someone beat us to creating the record.
                     if mui_count.is_some() {
@@ -896,7 +909,89 @@ impl<
                         self.counters.inc_prefixes_count(prefix.get_len());
                     }
 
+<<<<<<< Updated upstream
                     (mui_count, retry_count)
+=======
+                    let p = locked_prefix
+                        .get_or_init(1 << (next_level - this_level), || {
+                            new_stored_prefix
+                        });
+                    // let back_off = Backoff::new();
+
+                    let res = p.record_map.upsert_record(record);
+
+                    // loop {
+                    //     if let Ok(()) = set_res {
+                    //         break;
+                    //     } else {
+                    //         if let Some(stored_record) = locked_prefix.get() {
+                    //             res = stored_record
+                    //                 .record_map
+                    //                 .upsert_record(record.clone());
+                    //             // locked_prefix.set(*stored_record);
+                    //         };
+                    //     }
+                    //     back_off.snooze();
+                    // }
+
+                    // We're expecting an empty slot.
+                    // match atomic_stored_prefix.0.compare_exchange(
+                    //     Shared::null(),
+                    //     // tag with value 1, means the path selection is set to
+                    //     // outdated.
+                    //     Owned::new(new_stored_prefix).with_tag(1),
+                    //     Ordering::AcqRel,
+                    //     Ordering::Acquire,
+                    //     guard,
+                    // ) {
+                    //     // ...and we got an empty slot, the newly created
+                    //     // StoredPrefix is stored into it.
+                    //     Ok(spfx) => {
+                    //         if log_enabled!(log::Level::Info) {
+                    //             let StoredPrefix {
+                    //                 prefix,
+                    //                 record_map: stored_record,
+                    //                 ..
+                    //             } = unsafe { spfx.deref() };
+                    //             if log_enabled!(log::Level::Info) {
+                    //                 info!(
+                    //                         "{} store: Inserted new prefix record {}/{} with {:?}",
+                    //                         std::thread::current().name().unwrap(),
+                    //                         prefix.get_net().into_ipaddr(), prefix.get_len(),
+                    //                         stored_record
+                    //                     );
+                    //             }
+                    //         }
+
+                    self.counters.inc_prefixes_count(prefix.get_len());
+
+                    //         // ..and update the record_map with the actual record
+                    //         // we got from the user.
+                    //         unsafe { spfx.deref() }
+                    //             .record_map
+                    //             .upsert_record(record)
+                    //     }
+                    //     // ...somebody beat us to it, the slot's not empty
+                    //     // anymore, we'll have to do it again.
+                    //     Err(CompareExchangeError { current, new: _ }) => {
+                    //         if log_enabled!(log::Level::Debug) {
+                    //             debug!(
+                    //                     "{} store: Prefix can't be inserted as new {:?}",
+                    //                     std::thread::current().name().unwrap(),
+                    //                     current
+                    //                 );
+                    //         }
+                    //         retry_count += 1;
+                    //         let stored_prefix = unsafe { current.deref() };
+
+                    //         // update the record_map from the winning thread
+                    //         // with our caller's record.
+                    //         stored_prefix.set_ps_outdated(guard)?;
+                    //         stored_prefix.record_map.upsert_record(record)
+                    //     }
+                    // }
+                    res
+>>>>>>> Stashed changes
                 }
                 // There already is a StoredPrefix with a record at this
                 // location.
@@ -1079,7 +1174,14 @@ impl<
     pub(crate) fn non_recursive_retrieve_prefix_mut(
         &'a self,
         search_prefix_id: PrefixId<AF>,
+<<<<<<< Updated upstream
     ) -> (&'a StoredPrefix<AF, M>, bool) {
+=======
+    ) -> Result<
+        (&'a StoredPrefix<AF, M>, u8),
+        (&'a OnceBoxSlice<StoredPrefix<AF, M>>, u8),
+    > {
+>>>>>>> Stashed changes
         trace!("non_recursive_retrieve_prefix_mut_with_guard");
         let mut prefix_set = self
             .prefixes
@@ -1092,16 +1194,29 @@ impl<
             let index = Self::hash_prefix_id(search_prefix_id, level);
 
             // probe the slot with the index that's the result of the hashing.
+<<<<<<< Updated upstream
             // let locked_prefix = prefix_set.0.get(index);
             let stored_prefix = match prefix_set.0.get(index) {
                 Some(p) => {
                     trace!("prefix set found.");
                     (p, true)
                 }
+=======
+            // stored_prefix = Some(prefix_set.0[index]);
+            // let prefix_probe = if !prefixes.is_null() {
+            trace!("prefix set found.");
+            let locked_prefix = prefix_set.0.get(index);
+            let stored_prefix = match locked_prefix {
+                // None => {
+                // panic!("index for PrefixSet out of bounds. search_prefix_id {:?}, level {}", search_prefix_id, level);
+                // }
+                Some(p) => p,
+>>>>>>> Stashed changes
                 None => {
                     // We're at the end of the chain and haven't found our
                     // search_prefix_id anywhere. Return the end-of-the-chain
                     // StoredPrefix, so the caller can attach a new one.
+<<<<<<< Updated upstream
                     trace!(
                         "no record. returning last found record in level
                         {}, with index {}.",
@@ -1125,6 +1240,13 @@ impl<
                             .0,
                         false,
                     )
+=======
+                    // trace!("no prefix set.");
+                    trace!("no record. returning last found record.");
+                    return Err((&prefix_set.0, level));
+                    // .map(|sp| (sp, level)
+                    // .ok_or(locked_prefix)
+>>>>>>> Stashed changes
                 }
             };
 
@@ -1172,7 +1294,17 @@ impl<
             // HASHING FUNCTION
             let index = Self::hash_prefix_id(id, level);
 
+<<<<<<< Updated upstream
             if let Some(stored_prefix) = prefix_set.0.get(index) {
+=======
+            // let mut prefixes = prefix_set; //.0.load(Ordering::Acquire, guard);
+
+            // if !prefixes.is_none() {
+            // let prefix_ref = prefixes.0[index];
+            if let Some(stored_prefix) = prefix_set.0.get(index)
+            // unsafe { prefix_ref }.get_stored_prefix(guard)
+            {
+>>>>>>> Stashed changes
                 if id == stored_prefix.get_prefix_id() {
                     trace!("found requested prefix {:?}", id);
                     parents[level as usize] = Some((prefix_set, index));
@@ -1220,7 +1352,18 @@ impl<
                 // HASHING FUNCTION
                 let index = Self::hash_prefix_id(prefix_id, level);
 
+<<<<<<< Updated upstream
                 if let Some(stored_prefix) = prefix_set.0.get(index) {
+=======
+                // let prefixes = prefix_set.0.load(Ordering::SeqCst, guard);
+                // let tag = prefixes.tag();
+                // let prefix_ref = unsafe { &prefixes.deref()[index] };
+
+                if let Some(stored_prefix) =
+                    // unsafe { prefix_ref }.get_stored_prefix(guard)
+                    prefix_set.0.get(index)
+                {
+>>>>>>> Stashed changes
                     if prefix_id == stored_prefix.prefix {
                         trace!("found requested prefix {:?}", prefix_id,);
                         return Some((stored_prefix, 0));
diff --git a/src/local_array/store/default_store.rs b/src/local_array/store/default_store.rs
deleted file mode 100644
index 03b4444a..00000000
--- a/src/local_array/store/default_store.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-use crate::prelude::multi::*;
-use crate::prelude::*;
-use std::fmt;
-
-// The default stride sizes for IPv4, IPv6, resp.
-#[create_store((
-    ([5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 5, 18),
-    ([4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-    4, 4, 4, 4, 4, 4, 4, 4, 4], 17, 30)
-))]
-struct DefaultStore;
-
-/// Try some
-impl<M: Meta> DefaultStore<M> {
-    pub fn try_default() -> Result<Self, PrefixStoreError> {
-        let config = StoreConfig {
-            persist_strategy: PersistStrategy::MemoryOnly,
-            persist_path: "/tmp/rotonda/".to_string(),
-        };
-        Self::new_with_config(config)
-            .map_err(|_| PrefixStoreError::StoreNotReadyError)
-    }
-}
-
-impl<
-        M: Meta,
-        NB: NodeBuckets<IPv4>,
-        PB: PrefixBuckets<IPv4, M>,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > fmt::Display
-    for CustomAllocStorage<IPv4, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
-{
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "CustomAllocStorage<IPv4, {}>",
-            std::any::type_name::<M>()
-        )
-    }
-}
-
-impl<
-        M: Meta,
-        NB: NodeBuckets<IPv6>,
-        PB: PrefixBuckets<IPv6, M>,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > fmt::Display
-    for CustomAllocStorage<IPv6, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
-{
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "CustomAllocStorage<IPv6, {}>",
-            std::any::type_name::<M>()
-        )
-    }
-}
diff --git a/src/local_array/store/macros.rs b/src/local_array/store/macros.rs
index 2e5d8fab..8b263bc8 100644
--- a/src/local_array/store/macros.rs
+++ b/src/local_array/store/macros.rs
@@ -155,6 +155,7 @@ macro_rules! retrieve_node_mut_closure {
                 let index = Self::hash_node_id($id, level);
                 let node;
 
+<<<<<<< Updated upstream
                 match nodes.0.get(index) {
                     // This arm only ever gets called in multi-threaded code
                     // where our thread (running this code *now*), andgot ahead
@@ -189,6 +190,18 @@ macro_rules! retrieve_node_mut_closure {
                             return Some(SizedStrideRef::$stride(&node.node));
                         };
                     },
+=======
+                // Read the node from the block pointed to by the Atomic
+                // pointer.
+                // assert!(nodes.0.get(index).is_some());
+                // let stored_node = unsafe {
+                //     &mut nodes.0[index].assume_init_ref()
+                // };
+                // let this_node = stored_node.load(Ordering::Acquire, guard);
+
+                match nodes.0.get(index) {
+                    None => None,
+>>>>>>> Stashed changes
                     Some(this_node) => {
                         node = this_node;
                         if $id == this_node.node_id {
@@ -255,6 +268,10 @@ macro_rules! store_node_closure {
             multi_uniq_id: u32,
             mut level: u8,
             retry_count: u32| {
+<<<<<<< Updated upstream
+=======
+                // println!("-");
+>>>>>>> Stashed changes
                 let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level);
                 trace!("{:032b}", $id.get_id().0);
                 trace!("id {:?}", $id.get_id());
@@ -263,11 +280,22 @@ macro_rules! store_node_closure {
                 // HASHING FUNCTION
                 let index = Self::hash_node_id($id, level);
 
+<<<<<<< Updated upstream
                 match nodes.0.get(index) {
                     None => {
                         // No node exists, so we create one here.
                         let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level + 1);
+=======
+                // No node exists, so we create one here.
+                let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level + 1);
+
+                let node_set_len = next_level - this_level;
+>>>>>>> Stashed changes
+
+                let ptrbitarr = new_node.ptrbitarr.load();
+                let pfxbitarr = new_node.pfxbitarr.load();
 
+                let stored_node = nodes.0.get_or_init(index, || {
                         if log_enabled!(log::Level::Trace) {
                             trace!("Empty node found, creating new node {} len{} lvl{}",
                                 $id, $id.get_id().1, level + 1
@@ -282,6 +310,7 @@ macro_rules! store_node_closure {
 
                         trace!("multi uniq id {}", multi_uniq_id);
 
+<<<<<<< Updated upstream
                         let node_set = NodeSet::init(next_level - this_level);
 
                         let ptrbitarr = new_node.ptrbitarr.load();
@@ -308,8 +337,24 @@ macro_rules! store_node_closure {
                             if !its_us && pfxbitarr != 0 {
                                 stored_node.node.pfxbitarr.merge_with(pfxbitarr);
                             }
+=======
+                        // let node_set = if next_level > 0 {
+                        //     NodeSet::init((1 << (next_level - this_level)) as usize )
+                        // } else {
+                        //     NodeSet(
+                        //         Box::new([]),
+                        //         std::sync::RwLock::new(RoaringBitmap::new())
+                        //     )
+                        // }
+                        StoredNode {
+                            node_id: $id,
+                            node: TreeBitMapNode::<AF, $stride>::empty(),
+                            node_set: NodeSet::init(node_set_len)
+>>>>>>> Stashed changes
                         }
+                    });
 
+<<<<<<< Updated upstream
                         return Ok(($id, retry_count));
                     }
                     Some(stored_node) => {
@@ -369,7 +414,39 @@ macro_rules! store_node_closure {
                                 // There's no next level!
                                 _ => panic!("out of storage levels, current level is {}", level),
                             }
+=======
+                if stored_node.node_id == $id {
+                    stored_node.node_set.update_rbm_index(
+                        multi_uniq_id, $guard
+                    )?;
+
+                    stored_node.node.ptrbitarr.merge_with(ptrbitarr);
+                    stored_node.node.pfxbitarr.merge_with(pfxbitarr);
+
+                    Ok(($id, retry_count))
+                } else {
+                    // it's not "our" node, make a (recursive)
+                    // call to create it.
+                    level += 1;
+                    trace!("Collision with node_id {}, move to next level: {} len{} next_lvl{} index {}",
+                        stored_node.node_id, $id, $id.get_id().1, level, index
+                    );
+
+                    return match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
+                        // on to the next level!
+                        next_bit_shift if next_bit_shift > 0 => {
+                            (search_level.f)(
+                                search_level,
+                                &stored_node.node_set,
+                                new_node,
+                                multi_uniq_id,
+                                level,
+                                retry_count
+                            )
+>>>>>>> Stashed changes
                         }
+                        // There's no next level!
+                        _ => panic!("out of storage levels, current level is {}", level),
                     }
                 }
             }
diff --git a/src/local_array/store/mod.rs b/src/local_array/store/mod.rs
deleted file mode 100644
index c115a5ba..00000000
--- a/src/local_array/store/mod.rs
+++ /dev/null
@@ -1,11 +0,0 @@
-pub mod custom_alloc;
-pub mod errors;
-pub mod iterators;
-
-pub(crate) mod atomic_types;
-pub(crate) mod default_store;
-pub(crate) mod oncebox;
-
-pub use default_store::DefaultStore;
-#[macro_use]
-mod macros;
diff --git a/src/local_array/store/oncebox.rs b/src/local_array/store/oncebox.rs
index 9752ba5f..228c8b60 100644
--- a/src/local_array/store/oncebox.rs
+++ b/src/local_array/store/oncebox.rs
@@ -61,6 +61,7 @@ impl<T> Drop for OnceBox<T> {
     }
 }
 
+<<<<<<< Updated upstream
 #[derive(Debug, Default)]
 pub struct OnceBoxSlice<T> {
     ptr: AtomicPtr<OnceBox<T>>,
@@ -72,6 +73,19 @@ impl<T> OnceBoxSlice<T> {
         Self {
             ptr: AtomicPtr::new(null_mut()),
             p2_size,
+=======
+#[derive(Debug)]
+pub struct OnceBoxSlice<T> {
+    ptr: AtomicPtr<OnceBox<T>>,
+    pow2_size: u8,
+}
+
+impl<T> OnceBoxSlice<T> {
+    pub fn new(size: u8) -> Self {
+        Self {
+            ptr: AtomicPtr::new(null_mut()),
+            pow2_size: 1 << size as usize,
+>>>>>>> Stashed changes
         }
     }
 
@@ -84,24 +98,39 @@ impl<T> OnceBoxSlice<T> {
         if ptr.is_null() {
             None
         } else {
+<<<<<<< Updated upstream
             let slice =
                 unsafe { slice::from_raw_parts(ptr, 1 << self.p2_size) };
+=======
+            let slice = unsafe {
+                slice::from_raw_parts(ptr, 1 << self.pow2_size as usize)
+            };
+>>>>>>> Stashed changes
             slice.get(idx).and_then(|inner| inner.get())
         }
     }
 
+<<<<<<< Updated upstream
     pub fn get_or_init(
         &self,
         idx: usize,
         create: impl FnOnce() -> T,
     ) -> (&T, bool) {
         // assert!(idx < (1 << self.p2_size));
+=======
+    pub fn is_null(&self) -> bool {
+        self.ptr.load(Ordering::Relaxed).is_null()
+    }
+
+    pub fn get_or_init(&self, idx: usize, create: impl FnOnce() -> T) -> &T {
+>>>>>>> Stashed changes
         let slice = self.get_or_make_slice();
         slice[idx].get_or_init(create)
     }
 
     fn get_or_make_slice(&self) -> &[OnceBox<T>] {
         let ptr = self.ptr.load(Ordering::Relaxed);
+<<<<<<< Updated upstream
         if !ptr.is_null() {
             return unsafe { slice::from_raw_parts(ptr, 1 << self.p2_size) };
         }
@@ -109,6 +138,17 @@ impl<T> OnceBoxSlice<T> {
         // Create a slice, set it, get again.
         let mut vec = Vec::with_capacity(1 << self.p2_size);
         for _ in 0..(1 << self.p2_size) {
+=======
+        if ptr != null_mut() {
+            return unsafe {
+                slice::from_raw_parts(ptr, 1 << self.pow2_size as usize)
+            };
+        }
+
+        // Create a slice, set it, get again.
+        let mut vec = Vec::with_capacity(1 << self.pow2_size as usize);
+        for _ in 0..(1 << self.pow2_size) {
+>>>>>>> Stashed changes
             vec.push(OnceBox::new())
         }
         // Convert Vec<[OnceBox<T>] -> Box<[OnceBox<T>] -> &mut [OnceBox<T>]
@@ -132,14 +172,22 @@ impl<T> OnceBoxSlice<T> {
                 let _ = unsafe {
                     Box::from_raw(slice::from_raw_parts_mut(
                         ptr,
+<<<<<<< Updated upstream
                         1 << self.p2_size,
+=======
+                        self.pow2_size as usize,
+>>>>>>> Stashed changes
                     ))
                 };
                 current
             }
         };
 
+<<<<<<< Updated upstream
         unsafe { slice::from_raw_parts(res, 1 << self.p2_size) }
+=======
+        unsafe { slice::from_raw_parts(res, self.pow2_size as usize) }
+>>>>>>> Stashed changes
     }
 }
 
@@ -150,7 +198,11 @@ impl<T> Drop for OnceBoxSlice<T> {
             let _ = unsafe {
                 Box::from_raw(slice::from_raw_parts_mut(
                     ptr,
+<<<<<<< Updated upstream
                     1 << self.p2_size,
+=======
+                    self.pow2_size as usize,
+>>>>>>> Stashed changes
                 ))
             };
         }
diff --git a/src/local_array/tests.rs b/src/local_array/tests.rs
index 58d0d86a..06d5c441 100644
--- a/src/local_array/tests.rs
+++ b/src/local_array/tests.rs
@@ -4,7 +4,7 @@ use std::error::Error;
 //------------ AddressFamily bit flippers -----------------------------------
 #[test]
 fn test_af_1() -> Result<(), Box<dyn Error>> {
-    use crate::local_array::tree::StrideNodeId;
+    use crate::prelude::multi::StrideNodeId;
     use crate::AddressFamily;
     use crate::IPv4;
 
@@ -33,7 +33,7 @@ fn test_af_1() -> Result<(), Box<dyn Error>> {
 
 #[test]
 fn test_af_2() -> Result<(), Box<dyn Error>> {
-    use crate::local_array::tree::StrideNodeId;
+    use crate::prelude::multi::StrideNodeId;
     use crate::IPv4;
 
     let bit_addr: IPv4 = 0b1111_1111_1111_1111_1111_1111_1111_1111;
diff --git a/src/local_array/types.rs b/src/local_array/types.rs
new file mode 100644
index 00000000..13f3b91b
--- /dev/null
+++ b/src/local_array/types.rs
@@ -0,0 +1,119 @@
+use crate::AddressFamily;
+
+use super::errors::PrefixStoreError;
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
+pub enum RouteStatus {
+    Active,
+    InActive,
+    Withdrawn,
+}
+
+impl std::fmt::Display for RouteStatus {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RouteStatus::Active => write!(f, "active"),
+            RouteStatus::InActive => write!(f, "inactive"),
+            RouteStatus::Withdrawn => write!(f, "withdrawn"),
+        }
+    }
+}
+
+impl From<RouteStatus> for u8 {
+    fn from(value: RouteStatus) -> Self {
+        match value {
+            RouteStatus::Active => 1,
+            RouteStatus::InActive => 2,
+            RouteStatus::Withdrawn => 3,
+        }
+    }
+}
+
+impl TryFrom<u8> for RouteStatus {
+    type Error = PrefixStoreError;
+
+    fn try_from(value: u8) -> Result<Self, Self::Error> {
+        match value {
+            1 => Ok(RouteStatus::Active),
+            2 => Ok(RouteStatus::InActive),
+            3 => Ok(RouteStatus::Withdrawn),
+            _ => Err(PrefixStoreError::StoreNotReadyError),
+        }
+    }
+}
+
+//------------ PrefixId ------------------------------------------------------
+
+#[derive(Hash, Eq, PartialEq, Debug, Copy, Clone)]
+pub struct PrefixId<AF: AddressFamily>(Option<(AF, u8)>);
+
+impl<AF: AddressFamily> PrefixId<AF> {
+    pub fn new(net: AF, len: u8) -> Self {
+        PrefixId(Some((net, len)))
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_none()
+    }
+
+    pub fn get_net(&self) -> AF {
+        self.0.unwrap().0
+    }
+
+    pub fn get_len(&self) -> u8 {
+        self.0.unwrap().1
+    }
+
+    // This should never fail, since there shouldn't be a invalid prefix in
+    // this prefix id in the first place.
+    pub fn into_pub(&self) -> inetnum::addr::Prefix {
+        inetnum::addr::Prefix::new(
+            self.get_net().into_ipaddr(),
+            self.get_len(),
+        )
+        .unwrap_or_else(|p| panic!("can't convert {:?} into prefix.", p))
+    }
+
+    // Increment the length of the prefix without changing the bits part.
+    // This is used to iterate over more-specific prefixes for this prefix,
+    // since the more specifics iterator includes the requested `base_prefix`
+    // itself.
+    pub fn inc_len(self) -> Self {
+        Self(self.0.map(|(net, len)| (net, len + 1)))
+    }
+
+    pub fn as_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
+        match self.0 {
+            Some(r) => r.0.as_prefix_bytes(r.1),
+            _ => [255; PREFIX_SIZE],
+        }
+    }
+}
+
+impl<AF: AddressFamily> std::default::Default for PrefixId<AF> {
+    fn default() -> Self {
+        PrefixId(None)
+    }
+}
+
+impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
+    fn from(value: inetnum::addr::Prefix) -> Self {
+        Self(Some((AF::from_ipaddr(value.addr()), value.len())))
+    }
+}
+
+impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<PrefixId<AF>>
+    for [u8; PREFIX_SIZE]
+{
+    fn from(value: PrefixId<AF>) -> Self {
+        value.as_bytes::<PREFIX_SIZE>()
+    }
+}
+
+impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<[u8; PREFIX_SIZE]>
+    for PrefixId<AF>
+{
+    fn from(value: [u8; PREFIX_SIZE]) -> Self {
+        PrefixId(Some(AF::from_prefix_bytes(value)))
+    }
+}
diff --git a/src/local_vec/tree.rs b/src/local_vec/tree.rs
index bc004f6e..4b87f028 100644
--- a/src/local_vec/tree.rs
+++ b/src/local_vec/tree.rs
@@ -13,7 +13,7 @@ use crate::stride::*;
 use crate::synth_int::{U256, U512};
 use crate::{
     af::{AddressFamily, Zero},
-    custom_alloc::UpsertReport,
+    rib::UpsertReport,
 };
 
 #[cfg(feature = "cli")]
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index fb3081ec..ed612b42 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -2,8 +2,9 @@ use std::fmt;
 use std::fmt::Debug;
 use std::{cmp::Ordering, sync::Arc};
 
-use crate::local_array::store::atomic_types::RouteStatus;
-use crate::{af::AddressFamily, local_array::node::PrefixId};
+use crate::af::AddressFamily;
+use crate::local_array::types::RouteStatus;
+use crate::prelude::multi::PrefixId;
 use inetnum::addr::Prefix;
 
 //------------ InternalPrefixRecord -----------------------------------------
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index 5d5be38e..bc703324 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -15,16 +15,16 @@ pub mod multi {
 
     pub use crossbeam_epoch::{self as epoch, Guard};
 
-    pub use crate::local_array::store::atomic_types::RouteStatus;
-    pub use crate::local_array::store::atomic_types::{
+    pub use crate::local_array::errors::PrefixStoreError;
+    pub use crate::local_array::in_memory::atomic_types::{
         NodeBuckets, NodeSet, PrefixBuckets, PrefixSet,
     };
-    pub use crate::local_array::store::errors::PrefixStoreError;
-    pub use crate::local_array::tree::{PrefixId, StrideNodeId, TreeBitMap};
+    pub use crate::local_array::in_memory::tree::StrideNodeId;
+    pub use crate::local_array::types::{PrefixId, RouteStatus};
     pub use crate::prefix_record::PublicRecord as Record;
 
-    pub use crate::custom_alloc::CustomAllocStorage;
-    pub use crate::custom_alloc::{
+    pub use crate::rib::Rib;
+    pub use crate::rib::{
         Counters, PersistStrategy, StoreConfig, StoreStats, UpsertReport,
     };
 
diff --git a/src/rotonda_store.rs b/src/rotonda_store.rs
index 8187332c..3fd262d6 100644
--- a/src/rotonda_store.rs
+++ b/src/rotonda_store.rs
@@ -10,13 +10,13 @@ use inetnum::addr::Prefix;
 
 pub use crate::af::{AddressFamily, IPv4, IPv6};
 
-pub use crate::local_array::store::custom_alloc;
+pub use crate::local_array::rib::rib;
 
 pub const RECORDS_MAX_NUM: usize = 3;
 
 //------------ The publicly available Rotonda Stores ------------------------
 
-pub use crate::local_array::store::DefaultStore as MultiThreadedStore;
+pub use crate::local_array::rib::DefaultStore as MultiThreadedStore;
 pub use crate::local_vec::store::Store as SingleThreadedStore;
 
 //------------ Types for strides displaying/monitoring ----------------------

From 6c86f6a294bb9fd9de3f0a11b15633c4fd46d36b Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 17 Dec 2024 16:23:32 +0100
Subject: [PATCH 022/147] CustomAllocStorage -> Rib refactor (big) part 2

---
 examples/full_table_multiple_trees_json.rs |   5 +-
 examples/numbers_treebitmap.rs             |   3 +-
 proc_macros/src/lib.rs                     | 161 ++++++++++-----------
 tests/best-path.rs                         |   4 +-
 4 files changed, 80 insertions(+), 93 deletions(-)

diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index 05d186cc..333d75cb 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -51,7 +51,8 @@ fn main() -> Result<(), Box<dyn Error>> {
         println!("[");
         for n in 1..6 {
             let mut rec_vec: Vec<PrefixRecord<PrefixAs>> = vec![];
-            let tree_bitmap = MyStore::<PrefixAs>::try_default()?;
+            let config = StoreConfig::default();
+            let tree_bitmap = MyStore::<PrefixAs>::new_with_config(config)?;
 
             if let Err(err) = load_prefixes(&mut rec_vec) {
                 println!("error running example: {}", err);
@@ -110,7 +111,6 @@ fn main() -> Result<(), Box<dyn Error>> {
                 "\"strides v4 \": {:?},",
                 &tree_bitmap
                     .v4
-                    .store
                     .get_stride_sizes()
                     .iter()
                     .map_while(|s| if s > &0 { Some(*s) } else { None })
@@ -120,7 +120,6 @@ fn main() -> Result<(), Box<dyn Error>> {
                 "\"strides v6 \": {:?},",
                 &tree_bitmap
                     .v6
-                    .store
                     .get_stride_sizes()
                     .iter()
                     .map_while(|s| if s > &0 { Some(*s) } else { None })
diff --git a/examples/numbers_treebitmap.rs b/examples/numbers_treebitmap.rs
index 32aeac28..db10a92e 100644
--- a/examples/numbers_treebitmap.rs
+++ b/examples/numbers_treebitmap.rs
@@ -63,8 +63,9 @@ fn main() -> Result<(), Box<dyn Error>> {
 
     for _strides in strides_vec.iter() {
         let mut pfxs: Vec<PrefixRecord<PrefixAs>> = vec![];
+        let config = StoreConfig::default();
         let tree_bitmap: MyStore<PrefixAs> =
-            MyStore::<PrefixAs>::try_default()?;
+            MyStore::<PrefixAs>::new_with_config(config)?;
 
         if let Err(err) = load_prefixes(&mut pfxs) {
             println!("error running example: {}", err);
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 1dae186b..554bf33a 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -327,7 +327,7 @@ pub fn stride_sizes(
     };
 
     let type_alias = quote! {
-        type #type_name<M> = TreeBitMap<#ip_af, M, #buckets_name<#ip_af>, #prefixes_buckets_name<#ip_af, M>, #prefix_size, #key_size>;
+        type #type_name<M> = Rib<#ip_af, M, #buckets_name<#ip_af>, #prefixes_buckets_name<#ip_af, M>, #prefix_size, #key_size>;
     };
 
     let result = quote! {
@@ -529,7 +529,7 @@ pub fn create_store(
             ///
             /// let _: Vec<_> = (0..16)
             ///      .map(|_| {
-            ///         let tree_bitmap = tree_bitmap.clone();
+            ///         let tree_bitmap = in_memory_tree.clone();
             ///
             ///         thread::spawn(move || {
             ///              let pfxs = [
@@ -545,7 +545,7 @@ pub fn create_store(
             ///
             ///              for pfx in pfxs.into_iter() {
             ///                  println!("insert {}", pfx.unwrap());
-            ///                  tree_bitmap.insert(
+            ///                  in_memory_tree.insert(
             ///                      &pfx.unwrap(),
             ///                      Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
             ///                      None
@@ -957,7 +957,7 @@ pub fn create_store(
                     let (left, right) = match search_pfx.addr() {
                         std::net::IpAddr::V4(addr) => {
                             (
-                                Some(self.v4.store.less_specific_prefix_iter(
+                                Some(self.v4.less_specific_prefix_iter(
                                         PrefixId::<IPv4>::new(
                                             addr.into(),
                                             search_pfx.len(),
@@ -974,7 +974,7 @@ pub fn create_store(
                         std::net::IpAddr::V6(addr) => {
                             (
                                 None,
-                                Some(self.v6.store.less_specific_prefix_iter(
+                                Some(self.v6.less_specific_prefix_iter(
                                         PrefixId::<IPv6>::new(
                                             addr.into(),
                                             search_pfx.len(),
@@ -1045,41 +1045,37 @@ pub fn create_store(
 
                 let (left, right) = match search_pfx.addr() {
                     std::net::IpAddr::V4(addr) => {
-                        let bmin = unsafe {
-                            self.v4.store.withdrawn_muis_bmin.load(
-                                Ordering::Acquire, guard
-                            ).deref()
-                        };
+
+                        let bmin = self.v6.in_memory_tree
+                            .withdrawn_muis_bmin(guard);
+
                         if mui.is_some() && bmin.contains(mui.unwrap()) {
-                                (None, None)
-                            } else {
-                                (
-                                    Some(self.v4.store.more_specific_prefix_iter_from(
-                                            PrefixId::<IPv4>::new(
-                                                addr.into(),
-                                                search_pfx.len(),
-                                            ),
-                                            mui,
-                                            include_withdrawn,
-                                            guard
-                                        ).map(|p| PrefixRecord::from(p))
-                                    ),
-                                    None
-                                )
+                          (None, None)
+                         } else {
+                                (Some(self.v4.more_specific_prefix_iter_from(
+                                        PrefixId::<IPv4>::new(
+                                            addr.into(),
+                                            search_pfx.len(),
+                                        ),
+                                        mui,
+                                        include_withdrawn,
+                                        guard
+                                    ).map(|p| PrefixRecord::from(p))
+                                ),
+                                None)
                             }
                         }
                     std::net::IpAddr::V6(addr) => {
-                        let bmin = unsafe {
-                            self.v6.store.withdrawn_muis_bmin.load(
-                                Ordering::Acquire, guard
-                            ).deref()
-                        };
+
+                        let bmin = self.v6.in_memory_tree
+                            .withdrawn_muis_bmin(guard);
+
                         if mui.is_some() && bmin.contains(mui.unwrap()) {
-                            (None, None)
+                         (None, None)
                         } else {
                             (
                                 None,
-                                Some(self.v6.store.more_specific_prefix_iter_from(
+                                Some(self.v6.more_specific_prefix_iter_from(
                                         PrefixId::<IPv6>::new(
                                             addr.into(),
                                             search_pfx.len(),
@@ -1103,17 +1099,14 @@ pub fn create_store(
                 guard: &'a Guard
             ) -> impl Iterator<Item=PrefixRecord<M>> +'a {
 
-                let bmin = unsafe {
-                    self.v4.store.withdrawn_muis_bmin.load(
-                        Ordering::Acquire, guard
-                    ).deref()
-                };
+                let bmin =
+                    self.v4.in_memory_tree.withdrawn_muis_bmin(guard);
 
                 if bmin.contains(mui) && !include_withdrawn {
                     None
                 } else {
                     Some(
-                        self.v4.store.more_specific_prefix_iter_from(
+                        self.v4.more_specific_prefix_iter_from(
                                 PrefixId::<IPv4>::new(
                                     0,
                                     0,
@@ -1122,8 +1115,8 @@ pub fn create_store(
                                 include_withdrawn,
                                 guard
                             ).map(|p| PrefixRecord::from(p))
-                    )
-                }.into_iter().flatten()
+                    )}
+                .into_iter().flatten()
             }
 
             pub fn iter_records_for_mui_v6(
@@ -1133,17 +1126,13 @@ pub fn create_store(
                 guard: &'a Guard
             ) -> impl Iterator<Item=PrefixRecord<M>> +'a {
 
-                let bmin = unsafe {
-                    self.v4.store.withdrawn_muis_bmin.load(
-                        Ordering::Acquire, guard
-                    ).deref()
-                };
+                let bmin = self.v6.in_memory_tree.withdrawn_muis_bmin(guard);
 
                 if bmin.contains(mui) && !include_withdrawn {
                     None
                 } else {
                     Some(
-                        self.v6.store.more_specific_prefix_iter_from(
+                        self.v6.more_specific_prefix_iter_from(
                                 PrefixId::<IPv6>::new(
                                     0,
                                     0,
@@ -1152,8 +1141,8 @@ pub fn create_store(
                                 include_withdrawn,
                                 guard
                             ).map(|p| PrefixRecord::from(p))
-                    )
-                }.into_iter().flatten()
+                    )}
+                .into_iter().flatten()
             }
 
             /// Insert or replace a Record into the Store
@@ -1239,10 +1228,10 @@ pub fn create_store(
             pub fn prefixes_iter(
                 &'a self,
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v4.store.prefixes_iter()
+                self.v4.prefixes_iter()
                     .map(|p| PrefixRecord::from(p))
                     .chain(
-                        self.v6.store.prefixes_iter()
+                        self.v6.prefixes_iter()
                         .map(|p| PrefixRecord::from(p))
                     )
             }
@@ -1290,7 +1279,7 @@ pub fn create_store(
             pub fn prefixes_iter_v4(
                 &'a self,
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v4.store.prefixes_iter()
+                self.v4.prefixes_iter()
                     .map(|p| PrefixRecord::from(p))
             }
 
@@ -1337,7 +1326,7 @@ pub fn create_store(
             pub fn prefixes_iter_v6(
                 &'a self,
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v6.store.prefixes_iter()
+                self.v6.prefixes_iter()
                     .map(|p| PrefixRecord::from(p))
             }
 
@@ -1353,17 +1342,15 @@ pub fn create_store(
                 let guard = &epoch::pin();
                 match prefix.addr() {
                     std::net::IpAddr::V4(addr) => {
-                        self.v4.store.mark_mui_as_withdrawn_for_prefix(
+                        self.v4.mark_mui_as_withdrawn_for_prefix(
                             PrefixId::<IPv4>::from(*prefix),
                             mui,
-                            // &guard
                         )
                     }
                     std::net::IpAddr::V6(addr) => {
-                        self.v6.store.mark_mui_as_withdrawn_for_prefix(
+                        self.v6.mark_mui_as_withdrawn_for_prefix(
                             PrefixId::<IPv6>::from(*prefix),
                             mui,
-                            // &guard
                         )
                     }
                 }
@@ -1381,13 +1368,13 @@ pub fn create_store(
                 let guard = &epoch::pin();
                 match prefix.addr() {
                     std::net::IpAddr::V4(addr) => {
-                        self.v4.store.mark_mui_as_active_for_prefix(
+                        self.v4.mark_mui_as_active_for_prefix(
                             PrefixId::<IPv4>::from(*prefix),
                             mui,
                         )
                     }
                     std::net::IpAddr::V6(addr) => {
-                        self.v6.store.mark_mui_as_active_for_prefix(
+                        self.v6.mark_mui_as_active_for_prefix(
                             PrefixId::<IPv6>::from(*prefix),
                             mui,
                         )
@@ -1405,7 +1392,7 @@ pub fn create_store(
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
 
-                self.v4.store.mark_mui_as_active(
+                self.v4.in_memory_tree.mark_mui_as_active(
                     mui,
                     &guard
                 )
@@ -1423,7 +1410,7 @@ pub fn create_store(
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
 
-                self.v4.store.mark_mui_as_withdrawn(
+                self.v4.in_memory_tree.mark_mui_as_withdrawn(
                     mui,
                     &guard
                 )
@@ -1439,7 +1426,7 @@ pub fn create_store(
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
 
-                self.v6.store.mark_mui_as_active(
+                self.v6.in_memory_tree.mark_mui_as_active(
                     mui,
                     &guard
                 )
@@ -1457,7 +1444,7 @@ pub fn create_store(
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
 
-                self.v6.store.mark_mui_as_withdrawn(
+                self.v6.in_memory_tree.mark_mui_as_withdrawn(
                     mui,
                     &guard
                 )
@@ -1477,11 +1464,11 @@ pub fn create_store(
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
 
-                let res_v4 = self.v4.store.mark_mui_as_withdrawn(
+                let res_v4 = self.v4.in_memory_tree.mark_mui_as_withdrawn(
                     mui,
                     &guard
                 );
-                let res_v6 = self.v6.store.mark_mui_as_withdrawn(
+                let res_v6 = self.v6.in_memory_tree.mark_mui_as_withdrawn(
                     mui,
                     &guard
                 );
@@ -1499,7 +1486,7 @@ pub fn create_store(
             ) -> bool {
                 let guard = &epoch::pin();
 
-                self.v4.store.mui_is_withdrawn(mui, guard)
+                self.v4.in_memory_tree.mui_is_withdrawn(mui, guard)
             }
 
             // Whether the global status for IPv6 prefixes and the specified
@@ -1510,7 +1497,7 @@ pub fn create_store(
             ) -> bool {
                 let guard = &epoch::pin();
 
-                self.v6.store.mui_is_withdrawn(mui, guard)
+                self.v6.in_memory_tree.mui_is_withdrawn(mui, guard)
             }
 
             /// Returns the number of all prefixes in the store.
@@ -1518,8 +1505,8 @@ pub fn create_store(
             /// Note that this method will actually traverse the complete
             /// tree.
             pub fn prefixes_count(&self) -> usize {
-                self.v4.store.get_prefixes_count()
-                + self.v6.store.get_prefixes_count()
+                self.v4.get_prefixes_count()
+                + self.v6.get_prefixes_count()
             }
 
             /// Returns the number of all IPv4 prefixes in the store.
@@ -1528,7 +1515,7 @@ pub fn create_store(
             /// number in the store, due to contention at the time of
             /// reading the value.
             pub fn prefixes_v4_count(&self) -> usize {
-                self.v4.store.get_prefixes_count()
+                self.v4.get_prefixes_count()
             }
 
             /// Returns the number of all IPv4 prefixes with the
@@ -1538,7 +1525,7 @@ pub fn create_store(
             /// number in the store, due to contention at the time of
             /// reading the value.
             pub fn prefixes_v4_count_for_len(&self, len: u8) -> usize {
-                self.v4.store.get_prefixes_count_for_len(len)
+                self.v4.get_prefixes_count_for_len(len)
             }
 
             /// Returns the number of all IPv6 prefixes in the store.
@@ -1547,7 +1534,7 @@ pub fn create_store(
             /// number in the store, due to contention at the time of
             /// reading the value.
             pub fn prefixes_v6_count(&self) -> usize {
-                self.v6.store.get_prefixes_count()
+                self.v6.get_prefixes_count()
             }
 
             /// Returns the number of all IPv6 prefixes with the
@@ -1557,7 +1544,7 @@ pub fn create_store(
             /// number in the store, due to contention at the time of
             /// reading the value.
             pub fn prefixes_v6_count_for_len(&self, len: u8) -> usize {
-                self.v6.store.get_prefixes_count_for_len(len)
+                self.v6.get_prefixes_count_for_len(len)
             }
 
             /// Returns the number of nodes in the store.
@@ -1566,8 +1553,8 @@ pub fn create_store(
             /// number in the store, due to contention at the time of
             /// reading the value.
             pub fn nodes_count(&self) -> usize {
-                self.v4.store.get_nodes_count()
-                + self.v6.store.get_nodes_count()
+                self.v4.get_nodes_count()
+                + self.v6.get_nodes_count()
             }
 
             /// Returns the number of IPv4 nodes in the store.
@@ -1576,7 +1563,7 @@ pub fn create_store(
             /// number in the store, due to contention at the time of
             /// reading the value.
             pub fn nodes_v4_count(&self) -> usize {
-                self.v4.store.get_nodes_count()
+                self.v4.get_nodes_count()
             }
 
             /// Returns the number of IPv6 nodes in the store.
@@ -1585,7 +1572,7 @@ pub fn create_store(
             /// number in the store, due to contention at the time of
             /// reading the value.
             pub fn nodes_v6_count(&self) -> usize {
-                self.v6.store.get_nodes_count()
+                self.v6.get_nodes_count()
             }
 
             /// Print the store statistics to the standard output.
@@ -1601,25 +1588,25 @@ pub fn create_store(
             // The Store statistics.
             pub fn stats(&self) -> StoreStats {
                 StoreStats {
-                    v4: self.v4.store.counters.get_prefix_stats(),
-                    v6: self.v6.store.counters.get_prefix_stats(),
+                    v4: self.v4.counters.get_prefix_stats(),
+                    v6: self.v6.counters.get_prefix_stats(),
                 }
             }
 
             // Disk Persistence
 
             pub fn persist_strategy(&self) -> PersistStrategy {
-                self.v4.store.config.persist_strategy()
+                self.v4.config.persist_strategy()
             }
 
             pub fn get_records_for_prefix(&self, prefix: &Prefix) ->
                 Vec<Record<M>> {
                 match prefix.is_v4() {
-                    true => self.v4.store.persistence.as_ref().map_or(vec![],
+                    true => self.v4.persist_tree.as_ref().map_or(vec![],
                         |p| p.get_records_for_prefix(
                             PrefixId::<IPv4>::from(*prefix)
                         )),
-                    false => self.v6.store.persistence.as_ref().map_or(vec![],
+                    false => self.v6.persist_tree.as_ref().map_or(vec![],
                         |p| p.get_records_for_prefix(
                             PrefixId::<IPv6>::from(*prefix)
                         ))
@@ -1630,12 +1617,12 @@ pub fn create_store(
             /// with their values to disk
             pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
                 if let Some(persistence) =
-                    self.v4.store.persistence.as_ref() {
+                    self.v4.persist_tree.as_ref() {
                         persistence.flush_to_disk()?;
                 }
 
                 if let Some(persistence) =
-                    self.v6.store.persistence.as_ref() {
+                    self.v6.persist_tree.as_ref() {
                         persistence.flush_to_disk()?;
                 }
 
@@ -1646,16 +1633,16 @@ pub fn create_store(
             /// to disk, for IPv4 and IPv6 respectively.
             pub fn approx_persisted_items(&self) -> (usize, usize) {
                 (
-                    self.v4.store.persistence.as_ref().map_or(0, |p| p.approximate_len()),
-                    self.v6.store.persistence.as_ref().map_or(0, |p| p.approximate_len())
+                    self.v4.persist_tree.as_ref().map_or(0, |p| p.approximate_len()),
+                    self.v6.persist_tree.as_ref().map_or(0, |p| p.approximate_len())
                 )
             }
 
             /// Return an estimation of the disk space currently used by the
             /// store in bytes.
             pub fn disk_space(&self) -> u64 {
-                self.v4.store.persistence.as_ref().map_or(0, |p| p.disk_space()) +
-                self.v6.store.persistence.as_ref().map_or(0, |p| p.disk_space())
+                self.v4.persist_tree.as_ref().map_or(0, |p| p.disk_space()) +
+                self.v6.persist_tree.as_ref().map_or(0, |p| p.disk_space())
             }
         }
     };
diff --git a/tests/best-path.rs b/tests/best-path.rs
index 9b94341d..fa37ce2d 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -1,10 +1,10 @@
 use inetnum::addr::Prefix;
 use inetnum::asn::Asn;
-use rotonda_store::custom_alloc::PersistStrategy;
-use rotonda_store::custom_alloc::StoreConfig;
 use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::Record;
 use rotonda_store::prelude::multi::RouteStatus;
+use rotonda_store::rib::PersistStrategy;
+use rotonda_store::rib::StoreConfig;
 use rotonda_store::MatchOptions;
 use rotonda_store::Meta;
 use rotonda_store::MultiThreadedStore;

From 299127d36214d2b6bedcef048e170d1bd264f55f Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 17 Dec 2024 21:01:59 +0100
Subject: [PATCH 023/147] saner pub

---
 examples/numbers_treebitmap.rs            |   1 -
 proc_macros/src/lib.rs                    |  62 +--
 src/local_array/in_memory/atomic_types.rs |   3 +-
 src/local_array/in_memory/tree.rs         | 172 ++++----
 src/local_array/iterators.rs              |   7 +-
 src/local_array/mod.rs                    |   1 +
 src/local_array/persist/lsm_tree.rs       | 231 ++++++++++
 src/local_array/persist/mod.rs            |   1 +
 src/local_array/query.rs                  |  79 ++--
 src/local_array/rib/rib.rs                | 516 +++++++++-------------
 10 files changed, 584 insertions(+), 489 deletions(-)
 create mode 100644 src/local_array/persist/lsm_tree.rs
 create mode 100644 src/local_array/persist/mod.rs

diff --git a/examples/numbers_treebitmap.rs b/examples/numbers_treebitmap.rs
index db10a92e..d2f5d99a 100644
--- a/examples/numbers_treebitmap.rs
+++ b/examples/numbers_treebitmap.rs
@@ -8,7 +8,6 @@ use std::ffi::OsString;
 use std::fs::File;
 use std::net::{IpAddr, Ipv4Addr};
 use std::process;
-use std::sync::atomic::Ordering;
 
 #[create_store((
     ([4, 4, 4, 4, 4, 4, 4, 4], 5, 17),
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 554bf33a..c505baa0 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -529,7 +529,7 @@ pub fn create_store(
             ///
             /// let _: Vec<_> = (0..16)
             ///      .map(|_| {
-            ///         let tree_bitmap = in_memory_tree.clone();
+            ///         let tree_bitmap = clone();
             ///
             ///         thread::spawn(move || {
             ///              let pfxs = [
@@ -545,7 +545,7 @@ pub fn create_store(
             ///
             ///              for pfx in pfxs.into_iter() {
             ///                  println!("insert {}", pfx.unwrap());
-            ///                  in_memory_tree.insert(
+            ///                  insert(
             ///                      &pfx.unwrap(),
             ///                      Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
             ///                      None
@@ -1046,8 +1046,7 @@ pub fn create_store(
                 let (left, right) = match search_pfx.addr() {
                     std::net::IpAddr::V4(addr) => {
 
-                        let bmin = self.v6.in_memory_tree
-                            .withdrawn_muis_bmin(guard);
+                        let bmin = self.v6.withdrawn_muis_bmin(guard);
 
                         if mui.is_some() && bmin.contains(mui.unwrap()) {
                           (None, None)
@@ -1067,8 +1066,7 @@ pub fn create_store(
                         }
                     std::net::IpAddr::V6(addr) => {
 
-                        let bmin = self.v6.in_memory_tree
-                            .withdrawn_muis_bmin(guard);
+                        let bmin = self.v6.withdrawn_muis_bmin(guard);
 
                         if mui.is_some() && bmin.contains(mui.unwrap()) {
                          (None, None)
@@ -1100,7 +1098,7 @@ pub fn create_store(
             ) -> impl Iterator<Item=PrefixRecord<M>> +'a {
 
                 let bmin =
-                    self.v4.in_memory_tree.withdrawn_muis_bmin(guard);
+                    self.v4.withdrawn_muis_bmin(guard);
 
                 if bmin.contains(mui) && !include_withdrawn {
                     None
@@ -1126,7 +1124,7 @@ pub fn create_store(
                 guard: &'a Guard
             ) -> impl Iterator<Item=PrefixRecord<M>> +'a {
 
-                let bmin = self.v6.in_memory_tree.withdrawn_muis_bmin(guard);
+                let bmin = self.v6.withdrawn_muis_bmin(guard);
 
                 if bmin.contains(mui) && !include_withdrawn {
                     None
@@ -1392,7 +1390,7 @@ pub fn create_store(
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
 
-                self.v4.in_memory_tree.mark_mui_as_active(
+                self.v4.mark_mui_as_active(
                     mui,
                     &guard
                 )
@@ -1410,7 +1408,7 @@ pub fn create_store(
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
 
-                self.v4.in_memory_tree.mark_mui_as_withdrawn(
+                self.v4.mark_mui_as_withdrawn(
                     mui,
                     &guard
                 )
@@ -1426,7 +1424,7 @@ pub fn create_store(
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
 
-                self.v6.in_memory_tree.mark_mui_as_active(
+                self.v6.mark_mui_as_active(
                     mui,
                     &guard
                 )
@@ -1444,7 +1442,7 @@ pub fn create_store(
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
 
-                self.v6.in_memory_tree.mark_mui_as_withdrawn(
+                self.v6.mark_mui_as_withdrawn(
                     mui,
                     &guard
                 )
@@ -1464,11 +1462,11 @@ pub fn create_store(
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
 
-                let res_v4 = self.v4.in_memory_tree.mark_mui_as_withdrawn(
+                let res_v4 = self.v4.mark_mui_as_withdrawn(
                     mui,
                     &guard
                 );
-                let res_v6 = self.v6.in_memory_tree.mark_mui_as_withdrawn(
+                let res_v6 = self.v6.mark_mui_as_withdrawn(
                     mui,
                     &guard
                 );
@@ -1486,7 +1484,7 @@ pub fn create_store(
             ) -> bool {
                 let guard = &epoch::pin();
 
-                self.v4.in_memory_tree.mui_is_withdrawn(mui, guard)
+                self.v4.mui_is_withdrawn(mui, guard)
             }
 
             // Whether the global status for IPv6 prefixes and the specified
@@ -1497,7 +1495,7 @@ pub fn create_store(
             ) -> bool {
                 let guard = &epoch::pin();
 
-                self.v6.in_memory_tree.mui_is_withdrawn(mui, guard)
+                self.v6.mui_is_withdrawn(mui, guard)
             }
 
             /// Returns the number of all prefixes in the store.
@@ -1596,35 +1594,22 @@ pub fn create_store(
             // Disk Persistence
 
             pub fn persist_strategy(&self) -> PersistStrategy {
-                self.v4.config.persist_strategy()
+                self.config.persist_strategy()
             }
 
             pub fn get_records_for_prefix(&self, prefix: &Prefix) ->
                 Vec<Record<M>> {
                 match prefix.is_v4() {
-                    true => self.v4.persist_tree.as_ref().map_or(vec![],
-                        |p| p.get_records_for_prefix(
-                            PrefixId::<IPv4>::from(*prefix)
-                        )),
-                    false => self.v6.persist_tree.as_ref().map_or(vec![],
-                        |p| p.get_records_for_prefix(
-                            PrefixId::<IPv6>::from(*prefix)
-                        ))
+                    true => self.v4.get_records_for_prefix(prefix),
+                    false => self.v6.get_records_for_prefix(prefix)
                 }
             }
 
             /// Persist all the non-unique (prefix, mui, ltime) tuples
             /// with their values to disk
-            pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
-                if let Some(persistence) =
-                    self.v4.persist_tree.as_ref() {
-                        persistence.flush_to_disk()?;
-                }
-
-                if let Some(persistence) =
-                    self.v6.persist_tree.as_ref() {
-                        persistence.flush_to_disk()?;
-                }
+            pub fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
+                self.v4.flush_to_disk()?;
+                self.v6.flush_to_disk()?;
 
                 Ok(())
             }
@@ -1633,16 +1618,15 @@ pub fn create_store(
             /// to disk, for IPv4 and IPv6 respectively.
             pub fn approx_persisted_items(&self) -> (usize, usize) {
                 (
-                    self.v4.persist_tree.as_ref().map_or(0, |p| p.approximate_len()),
-                    self.v6.persist_tree.as_ref().map_or(0, |p| p.approximate_len())
+                    self.v4.approx_persisted_items(),
+                    self.v6.approx_persisted_items()
                 )
             }
 
             /// Return an estimation of the disk space currently used by the
             /// store in bytes.
             pub fn disk_space(&self) -> u64 {
-                self.v4.persist_tree.as_ref().map_or(0, |p| p.disk_space()) +
-                self.v6.persist_tree.as_ref().map_or(0, |p| p.disk_space())
+                self.v4.disk_space() + self.v6.disk_space()
             }
         }
     };
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index f5b9870d..254fd2d3 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -13,11 +13,12 @@ use log::{debug, log_enabled, trace};
 use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
 
+use crate::local_array::persist::lsm_tree::PersistTree;
 use crate::local_array::types::{PrefixId, RouteStatus};
 // use crate::local_array::in_memory_tree::*;
 use crate::prefix_record::PublicRecord;
 use crate::prelude::Meta;
-use crate::rib::{PersistStrategy, PersistTree};
+use crate::rib::PersistStrategy;
 use crate::AddressFamily;
 
 use super::super::errors::PrefixStoreError;
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 1acb7562..937c674d 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -1,13 +1,11 @@
 use crate::prefix_record::{Meta, PublicRecord};
 use crate::prelude::multi::PrefixId;
-use crossbeam_epoch::{self as epoch, Atomic, Guard, Owned};
+use crossbeam_epoch::{self as epoch, Atomic};
 use log::{error, log_enabled, trace};
 use roaring::RoaringBitmap;
 
 use std::hash::Hash;
-use std::sync::atomic::{
-    AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering,
-};
+use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
 use std::{fmt::Debug, marker::PhantomData};
 
 use super::atomic_types::{NodeBuckets, PrefixBuckets};
@@ -204,7 +202,7 @@ pub struct TreeBitMap<
 > {
     pub(crate) node_buckets: NB,
     pub(crate) prefix_buckets: PB,
-    withdrawn_muis_bmin: Atomic<RoaringBitmap>,
+    pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     _af: PhantomData<AF>,
     _m: PhantomData<M>,
 }
@@ -226,102 +224,91 @@ impl<
         }
     }
 
-    pub fn withdrawn_muis_bmin<'a>(
-        &'a self,
-        guard: &'a Guard,
-    ) -> &'a RoaringBitmap {
-        unsafe {
-            self.withdrawn_muis_bmin
-                .load(Ordering::Acquire, guard)
-                .deref()
-        }
-    }
-
     // Change the status of the mui globally to Withdrawn. Iterators and match
     // functions will by default not return any records for this mui.
-    pub fn mark_mui_as_withdrawn(
-        &self,
-        mui: u32,
-        guard: &Guard,
-    ) -> Result<(), PrefixStoreError> {
-        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
-
-        let mut new = unsafe { current.as_ref() }.unwrap().clone();
-        new.insert(mui);
-
-        #[allow(clippy::assigning_clones)]
-        loop {
-            match self.withdrawn_muis_bmin.compare_exchange(
-                current,
-                Owned::new(new),
-                Ordering::AcqRel,
-                Ordering::Acquire,
-                guard,
-            ) {
-                Ok(_) => return Ok(()),
-                Err(updated) => {
-                    new =
-                        unsafe { updated.current.as_ref() }.unwrap().clone();
-                }
-            }
-        }
-    }
+    // pub fn mark_mui_as_withdrawn(
+    //     &self,
+    //     mui: u32,
+    //     guard: &Guard,
+    // ) -> Result<(), PrefixStoreError> {
+    //     let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
+
+    //     let mut new = unsafe { current.as_ref() }.unwrap().clone();
+    //     new.insert(mui);
+
+    //     #[allow(clippy::assigning_clones)]
+    //     loop {
+    //         match self.withdrawn_muis_bmin.compare_exchange(
+    //             current,
+    //             Owned::new(new),
+    //             Ordering::AcqRel,
+    //             Ordering::Acquire,
+    //             guard,
+    //         ) {
+    //             Ok(_) => return Ok(()),
+    //             Err(updated) => {
+    //                 new =
+    //                     unsafe { updated.current.as_ref() }.unwrap().clone();
+    //             }
+    //         }
+    //     }
+    // }
 
     // Change the status of the mui globally to Active. Iterators and match
     // functions will default to the status on the record itself.
-    pub fn mark_mui_as_active(
-        &self,
-        mui: u32,
-        guard: &Guard,
-    ) -> Result<(), PrefixStoreError> {
-        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
-
-        let mut new = unsafe { current.as_ref() }.unwrap().clone();
-        new.remove(mui);
-
-        #[allow(clippy::assigning_clones)]
-        loop {
-            match self.withdrawn_muis_bmin.compare_exchange(
-                current,
-                Owned::new(new),
-                Ordering::AcqRel,
-                Ordering::Acquire,
-                guard,
-            ) {
-                Ok(_) => return Ok(()),
-                Err(updated) => {
-                    new =
-                        unsafe { updated.current.as_ref() }.unwrap().clone();
-                }
-            }
-        }
-    }
+    // pub fn mark_mui_as_active(
+    //     &self,
+    //     mui: u32,
+    //     guard: &Guard,
+    // ) -> Result<(), PrefixStoreError> {
+    //     let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
+
+    //     let mut new = unsafe { current.as_ref() }.unwrap().clone();
+    //     new.remove(mui);
+
+    //     #[allow(clippy::assigning_clones)]
+    //     loop {
+    //         match self.withdrawn_muis_bmin.compare_exchange(
+    //             current,
+    //             Owned::new(new),
+    //             Ordering::AcqRel,
+    //             Ordering::Acquire,
+    //             guard,
+    //         ) {
+    //             Ok(_) => return Ok(()),
+    //             Err(updated) => {
+    //                 new =
+    //                     unsafe { updated.current.as_ref() }.unwrap().clone();
+    //             }
+    //         }
+    //     }
+    // }
 
     // Whether this mui is globally withdrawn. Note that this overrules
     // (by default) any (prefix, mui) combination in iterators and match
     // functions.
-    pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
-        unsafe {
-            self.withdrawn_muis_bmin
-                .load(Ordering::Acquire, guard)
-                .as_ref()
-        }
-        .unwrap()
-        .contains(mui)
-    }
+    // pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
+    //     unsafe {
+    //         self.withdrawn_muis_bmin
+    //             .load(Ordering::Acquire, guard)
+    //             .as_ref()
+    //     }
+    //     .unwrap()
+    //     .contains(mui)
+    // }
 
     // Whether this mui is globally active. Note that the local statuses of
     // records (prefix, mui) may be set to withdrawn in iterators and match
     // functions.
-    pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
-        !unsafe {
-            self.withdrawn_muis_bmin
-                .load(Ordering::Acquire, guard)
-                .as_ref()
-        }
-        .unwrap()
-        .contains(mui)
-    }
+    // pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
+    //     !unsafe {
+    //         self.withdrawn_muis_bmin
+    //             .load(Ordering::Acquire, guard)
+    //             .as_ref()
+    //     }
+    //     .unwrap()
+    //     .contains(mui)
+    // }
 }
 
 impl<
@@ -515,15 +502,12 @@ impl<
         &self,
         record: PublicRecord<M>,
         guard: &epoch::Guard,
-        // user_data: Option<&<M as MergeUpdate>::UserDataIn>,
     ) -> Result<UpsertReport, PrefixStoreError> {
         trace!("Updating the default route...");
 
-        if let Some(root_node) = self.retrieve_node_mut(
-            self.get_root_node_id(),
-            record.multi_uniq_id,
-            // guard,
-        ) {
+        if let Some(root_node) = self
+            .retrieve_node_mut(self.get_root_node_id(), record.multi_uniq_id)
+        {
             match root_node {
                 SizedStrideRef::Stride3(_) => {
                     self.in_memory_tree
diff --git a/src/local_array/iterators.rs b/src/local_array/iterators.rs
index f9176e9e..c5b9abe9 100644
--- a/src/local_array/iterators.rs
+++ b/src/local_array/iterators.rs
@@ -6,7 +6,6 @@
 // storage (and some over the TreeBitMap nodes, the parent of the store),
 // as such all the iterators here are composed of iterators over the
 // individual nodes. The Node Iterators live in the node.rs file.
-use std::sync::atomic::Ordering;
 
 use super::in_memory::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
 use super::in_memory::tree::{Stride3, Stride4, Stride5, StrideNodeId};
@@ -817,8 +816,7 @@ impl<
                     }
                 };
 
-                let global_withdrawn_bmin =
-                    self.in_memory_tree.withdrawn_muis_bmin(guard);
+                let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
 
                 Some(MoreSpecificPrefixIter {
                     store: self,
@@ -864,8 +862,7 @@ impl<
                 .in_memory_tree
                 .prefix_buckets
                 .get_root_prefix_set(cur_len);
-            let global_withdrawn_bmin =
-                self.in_memory_tree.withdrawn_muis_bmin(guard);
+            let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
 
             Some(LessSpecificPrefixIter {
                 prefixes: &self.in_memory_tree.prefix_buckets,
diff --git a/src/local_array/mod.rs b/src/local_array/mod.rs
index b123f8ff..41b51128 100644
--- a/src/local_array/mod.rs
+++ b/src/local_array/mod.rs
@@ -1,4 +1,5 @@
 pub(crate) mod in_memory;
+pub(crate) mod persist;
 mod tests;
 
 pub(crate) mod bit_span;
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
new file mode 100644
index 00000000..64574963
--- /dev/null
+++ b/src/local_array/persist/lsm_tree.rs
@@ -0,0 +1,231 @@
+//------------ PersistTree ---------------------------------------------------
+
+use std::marker::PhantomData;
+use std::path::Path;
+
+use lsm_tree::AbstractTree;
+
+use crate::local_array::types::{PrefixId, RouteStatus};
+use crate::{AddressFamily, MatchType, Meta, PublicRecord, QueryResult};
+
+pub struct PersistTree<
+    AF: AddressFamily,
+    // The size in bytes of the prefix in the peristed storage (disk), this
+    // amounnts to the bytes for the addres (4 for IPv4, 16 for IPv6) and 1
+    // bytefor the prefix length.
+    const PREFIX_SIZE: usize,
+    // The size in bytes of the complete key in the persisted storage, this
+    // is PREFIX_SIZE bytes (4; 16) + mui size (4) + ltime (8)
+    const KEY_SIZE: usize,
+>(lsm_tree::Tree, PhantomData<AF>);
+
+impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
+    PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
+{
+    pub fn new(
+        persist_path: &Path,
+    ) -> PersistTree<AF, PREFIX_SIZE, KEY_SIZE> {
+        PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>(
+            lsm_tree::Config::new(persist_path).open().unwrap(),
+            PhantomData,
+        )
+    }
+
+    pub fn insert(&self, key: [u8; KEY_SIZE], value: &[u8]) -> (u32, u32) {
+        self.0.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
+    }
+
+    pub fn get_records_for_prefix<M: Meta>(
+        &self,
+        prefix: PrefixId<AF>,
+    ) -> Vec<PublicRecord<M>> {
+        let prefix_b = &prefix.as_bytes::<PREFIX_SIZE>();
+        (*self.0.prefix(prefix_b))
+            .into_iter()
+            .map(|kv| {
+                let kv = kv.unwrap();
+                let (_, mui, ltime, status) = Self::parse_key(kv.0.as_ref());
+                PublicRecord::new(
+                    mui,
+                    ltime,
+                    status.try_into().unwrap(),
+                    kv.1.as_ref().to_vec().into(),
+                )
+            })
+            .collect::<Vec<_>>()
+    }
+
+    pub fn get_records_for_key<M: Meta + From<Vec<u8>>>(
+        &self,
+        key: &[u8],
+    ) -> Vec<(inetnum::addr::Prefix, PublicRecord<M>)> {
+        (*self.0.prefix(key))
+            .into_iter()
+            .map(|kv| {
+                let kv = kv.unwrap();
+                let (pfx, mui, ltime, status) =
+                    Self::parse_key(kv.0.as_ref());
+
+                (
+                    PrefixId::<AF>::from(pfx).into_pub(),
+                    PublicRecord::new(
+                        mui,
+                        ltime,
+                        status.try_into().unwrap(),
+                        kv.1.as_ref().to_vec().into(),
+                    ),
+                )
+            })
+            .collect::<Vec<_>>()
+    }
+
+    fn match_prefix_in_persisted_store<'a, M: Meta>(
+        &'a self,
+        search_pfx: PrefixId<AF>,
+        mui: Option<u32>,
+    ) -> QueryResult<M> {
+        let key: Vec<u8> = if let Some(mui) = mui {
+            PersistTree::<AF,
+        PREFIX_SIZE, KEY_SIZE>::prefix_mui_persistence_key(search_pfx, mui)
+        } else {
+            search_pfx.as_bytes::<PREFIX_SIZE>().to_vec()
+        };
+
+        QueryResult {
+            prefix: Some(search_pfx.into_pub()),
+            match_type: MatchType::ExactMatch,
+            prefix_meta: self
+                .get_records_for_key(&key)
+                .into_iter()
+                .map(|(_, rec)| rec)
+                .collect::<Vec<_>>(),
+            less_specifics: None,
+            more_specifics: None,
+        }
+    }
+
+    pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
+        let segment = self.0.flush_active_memtable(0);
+
+        if let Ok(Some(segment)) = segment {
+            self.0.register_segments(&[segment])?;
+            self.0.compact(
+                std::sync::Arc::new(lsm_tree::compaction::Leveled::default()),
+                0,
+            )?;
+        };
+
+        Ok(())
+    }
+
+    pub fn approximate_len(&self) -> usize {
+        self.0.approximate_len()
+    }
+
+    pub fn disk_space(&self) -> u64 {
+        self.0.disk_space()
+    }
+
+    #[cfg(feature = "persist")]
+    pub fn persistence_key(
+        // PREFIX_SIZE bytes
+        prefix_id: PrefixId<AF>,
+        // 4 bytes
+        mui: u32,
+        // 8 bytes
+        ltime: u64,
+        // 1 byte
+        status: RouteStatus,
+    ) -> [u8; KEY_SIZE] {
+        assert!(KEY_SIZE > PREFIX_SIZE);
+        let key = &mut [0_u8; KEY_SIZE];
+
+        // prefix 5 or 17 bytes
+        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
+
+        // mui 4 bytes
+        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
+            .first_chunk_mut::<4>()
+            .unwrap() = mui.to_le_bytes();
+
+        // ltime 8 bytes
+        *key[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
+            .first_chunk_mut::<8>()
+            .unwrap() = ltime.to_le_bytes();
+
+        // status 1 byte
+        key[PREFIX_SIZE + 12] = status.into();
+
+        *key
+    }
+
+    #[cfg(feature = "persist")]
+    pub fn prefix_mui_persistence_key(
+        prefix_id: PrefixId<AF>,
+        mui: u32,
+    ) -> Vec<u8> {
+        let mut key = vec![];
+        // prefix 5 or 17 bytes
+        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
+
+        // mui 4 bytes
+        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
+            .first_chunk_mut::<4>()
+            .unwrap() = mui.to_le_bytes();
+
+        key
+    }
+
+    #[cfg(feature = "persist")]
+    pub fn parse_key(bytes: &[u8]) -> ([u8; PREFIX_SIZE], u32, u64, u8) {
+        (
+            // prefix 5 or 17 bytes
+            *bytes.first_chunk::<PREFIX_SIZE>().unwrap(),
+            // mui 4 bytes
+            u32::from_le_bytes(
+                *bytes[PREFIX_SIZE..PREFIX_SIZE + 4]
+                    .first_chunk::<4>()
+                    .unwrap(),
+            ),
+            // ltime 8 bytes
+            u64::from_le_bytes(
+                *bytes[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
+                    .first_chunk::<8>()
+                    .unwrap(),
+            ),
+            // status 1 byte
+            bytes[PREFIX_SIZE + 12],
+        )
+    }
+
+    #[cfg(feature = "persist")]
+    pub fn parse_prefix(bytes: &[u8]) -> [u8; PREFIX_SIZE] {
+        *bytes.first_chunk::<PREFIX_SIZE>().unwrap()
+    }
+
+    #[cfg(feature = "persist")]
+    pub(crate) fn persist_record<M: Meta>(
+        &self,
+        prefix: PrefixId<AF>,
+        mui: u32,
+        record: &PublicRecord<M>,
+    ) {
+        self.insert(
+            PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
+                prefix,
+                mui,
+                record.ltime,
+                record.status,
+            ),
+            record.meta.as_ref(),
+        );
+    }
+}
+
+impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
+    std::fmt::Debug for PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
+{
+    fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        todo!()
+    }
+}
diff --git a/src/local_array/persist/mod.rs b/src/local_array/persist/mod.rs
new file mode 100644
index 00000000..2b0475be
--- /dev/null
+++ b/src/local_array/persist/mod.rs
@@ -0,0 +1 @@
+pub(crate) mod lsm_tree;
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 7d5426fa..24678f4f 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -1,5 +1,3 @@
-use std::sync::atomic::Ordering;
-
 use crossbeam_epoch::{self as epoch};
 use epoch::Guard;
 
@@ -7,8 +5,9 @@ use crate::af::AddressFamily;
 use crate::local_array::in_memory::atomic_types::{
     NodeBuckets, PrefixBuckets,
 };
+use crate::local_array::persist::lsm_tree::PersistTree;
 use crate::prefix_record::{Meta, PublicRecord};
-use crate::rib::{PersistStrategy, PersistTree, Rib};
+use crate::rib::{PersistStrategy, Rib};
 use inetnum::addr::Prefix;
 
 use crate::QueryResult;
@@ -116,7 +115,7 @@ where
         impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a,
         PrefixStoreError,
     > {
-        let bmin = self.in_memory_tree.withdrawn_muis_bmin(guard);
+        let bmin = self.withdrawn_muis_bmin(guard);
 
         if mui.is_some() && bmin.contains(mui.unwrap()) {
             Err(PrefixStoreError::PrefixNotFound)
@@ -169,7 +168,7 @@ where
                         // the local statuses of the records with muis
                         // that appear in the specified bitmap index.
                         pfx.record_map.as_records_with_rewritten_status(
-                            self.in_memory_tree.withdrawn_muis_bmin(guard),
+                            self.withdrawn_muis_bmin(guard),
                             RouteStatus::Withdrawn,
                         )
                     },
@@ -254,40 +253,40 @@ where
         }
     }
 
-    fn match_prefix_in_persisted_store(
-        &'a self,
-        search_pfx: PrefixId<AF>,
-        mui: Option<u32>,
-    ) -> QueryResult<M> {
-        let key: Vec<u8> = if let Some(mui) = mui {
-            PersistTree::<AF,
-        PREFIX_SIZE, KEY_SIZE>::prefix_mui_persistence_key(search_pfx, mui)
-        } else {
-            search_pfx.as_bytes::<PREFIX_SIZE>().to_vec()
-        };
-
-        if let Some(persist) = &self.persist_tree {
-            QueryResult {
-                prefix: Some(search_pfx.into_pub()),
-                match_type: MatchType::ExactMatch,
-                prefix_meta: persist
-                    .get_records_for_key(&key)
-                    .into_iter()
-                    .map(|(_, rec)| rec)
-                    .collect::<Vec<_>>(),
-                less_specifics: None,
-                more_specifics: None,
-            }
-        } else {
-            QueryResult {
-                prefix: None,
-                match_type: MatchType::EmptyMatch,
-                prefix_meta: vec![],
-                less_specifics: None,
-                more_specifics: None,
-            }
-        }
-    }
+    // fn match_prefix_in_persisted_store(
+    //     &'a self,
+    //     search_pfx: PrefixId<AF>,
+    //     mui: Option<u32>,
+    // ) -> QueryResult<M> {
+    //     let key: Vec<u8> = if let Some(mui) = mui {
+    //         PersistTree::<AF,
+    //     PREFIX_SIZE, KEY_SIZE>::prefix_mui_persistence_key(search_pfx, mui)
+    //     } else {
+    //         search_pfx.as_bytes::<PREFIX_SIZE>().to_vec()
+    //     };
+
+    //     if let Some(persist) = &self.persist_tree {
+    //         QueryResult {
+    //             prefix: Some(search_pfx.into_pub()),
+    //             match_type: MatchType::ExactMatch,
+    //             prefix_meta: persist
+    //                 .get_records_for_key(&key)
+    //                 .into_iter()
+    //                 .map(|(_, rec)| rec)
+    //                 .collect::<Vec<_>>(),
+    //             less_specifics: None,
+    //             more_specifics: None,
+    //         }
+    //     } else {
+    //         QueryResult {
+    //             prefix: None,
+    //             match_type: MatchType::EmptyMatch,
+    //             prefix_meta: vec![],
+    //             less_specifics: None,
+    //             more_specifics: None,
+    //         }
+    //     }
+    // }
 
     pub fn best_path(
         &'a self,
@@ -896,7 +895,7 @@ where
         mui: Option<u32>,
         guard: &Guard,
     ) -> Vec<PublicRecord<M>> {
-        let bmin = self.in_memory_tree.withdrawn_muis_bmin(guard);
+        let bmin = self.withdrawn_muis_bmin(guard);
 
         pfx.record_map.get_filtered_records(mui, bmin)
     }
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 94c0d2e6..20798eb4 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -183,18 +183,18 @@
 // an actual memory leak in the mt-prefix-store (and even more if they can
 // produce a fix for it).
 
+use super::super::persist::lsm_tree::PersistTree;
 use crate::local_array::in_memory::atomic_stride::AtomicBitmap;
 use std::path::Path;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
+use inetnum::addr::Prefix;
 use log::{debug, info, log_enabled, trace};
 
 use crossbeam_epoch as epoch;
 use crossbeam_utils::Backoff;
 use epoch::{Guard, Owned};
-use lsm_tree::AbstractTree;
-
-use std::marker::PhantomData;
+use roaring::RoaringBitmap;
 
 use crate::local_array::in_memory::tree::{
     SizedStrideNode, Stride, Stride3, Stride4, Stride5, StrideNodeId,
@@ -207,7 +207,6 @@ use crate::{
     prefix_record::PublicRecord,
 };
 
-use super::super::types::RouteStatus;
 use crate::local_array::in_memory::atomic_types::{
     NodeBuckets, StoredPrefix,
 };
@@ -223,11 +222,33 @@ pub use crate::local_array::query;
 
 use crate::{
     impl_search_level, impl_search_level_for_mui, retrieve_node_mut_closure,
-    store_node_closure, Meta,
+    store_node_closure, MatchType, Meta, QueryResult,
 };
 
 use crate::AddressFamily;
 
+//------------ StoreConfig ---------------------------------------------------
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum PersistStrategy {
+    WriteAhead,
+    PersistHistory,
+    MemoryOnly,
+    PersistOnly,
+}
+
+#[derive(Debug, Clone)]
+pub struct StoreConfig {
+    pub persist_strategy: PersistStrategy,
+    pub persist_path: String,
+}
+
+impl StoreConfig {
+    pub fn persist_strategy(&self) -> PersistStrategy {
+        self.persist_strategy
+    }
+}
+
 //------------ Counters -----------------------------------------------------
 
 #[derive(Debug)]
@@ -315,220 +336,10 @@ pub struct UpsertReport {
     pub mui_count: usize,
 }
 
-//------------ PersistTree ---------------------------------------------------
-
-pub struct PersistTree<
-    AF: AddressFamily,
-    // The size in bytes of the prefix in the peristed storage (disk), this
-    // amounnts to the bytes for the addres (4 for IPv4, 16 for IPv6) and 1
-    // bytefor the prefix length.
-    const PREFIX_SIZE: usize,
-    // The size in bytes of the complete key in the persisted storage, this
-    // is PREFIX_SIZE bytes (4; 16) + mui size (4) + ltime (8)
-    const KEY_SIZE: usize,
->(lsm_tree::Tree, PhantomData<AF>);
-
-impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
-    PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
-{
-    pub fn insert(&self, key: [u8; KEY_SIZE], value: &[u8]) -> (u32, u32) {
-        self.0.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
-    }
-
-    pub fn get_records_for_prefix<M: Meta>(
-        &self,
-        prefix: PrefixId<AF>,
-    ) -> Vec<PublicRecord<M>> {
-        let prefix_b = &prefix.as_bytes::<PREFIX_SIZE>();
-        (*self.0.prefix(prefix_b))
-            .into_iter()
-            .map(|kv| {
-                let kv = kv.unwrap();
-                let (_, mui, ltime, status) = Self::parse_key(kv.0.as_ref());
-                PublicRecord::new(
-                    mui,
-                    ltime,
-                    status.try_into().unwrap(),
-                    kv.1.as_ref().to_vec().into(),
-                )
-            })
-            .collect::<Vec<_>>()
-    }
-
-    pub fn get_records_for_key<M: Meta + From<Vec<u8>>>(
-        &self,
-        key: &[u8],
-    ) -> Vec<(inetnum::addr::Prefix, PublicRecord<M>)> {
-        (*self.0.prefix(key))
-            .into_iter()
-            .map(|kv| {
-                let kv = kv.unwrap();
-                let (pfx, mui, ltime, status) =
-                    Self::parse_key(kv.0.as_ref());
-
-                (
-                    PrefixId::<AF>::from(pfx).into_pub(),
-                    PublicRecord::new(
-                        mui,
-                        ltime,
-                        status.try_into().unwrap(),
-                        kv.1.as_ref().to_vec().into(),
-                    ),
-                )
-            })
-            .collect::<Vec<_>>()
-    }
-
-    pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
-        let segment = self.0.flush_active_memtable(0);
-
-        if let Ok(Some(segment)) = segment {
-            self.0.register_segments(&[segment])?;
-            self.0.compact(
-                std::sync::Arc::new(lsm_tree::compaction::Leveled::default()),
-                0,
-            )?;
-        };
-
-        Ok(())
-    }
-
-    pub fn approximate_len(&self) -> usize {
-        self.0.approximate_len()
-    }
-
-    pub fn disk_space(&self) -> u64 {
-        self.0.disk_space()
-    }
-
-    #[cfg(feature = "persist")]
-    pub fn persistence_key(
-        // PREFIX_SIZE bytes
-        prefix_id: PrefixId<AF>,
-        // 4 bytes
-        mui: u32,
-        // 8 bytes
-        ltime: u64,
-        // 1 byte
-        status: RouteStatus,
-    ) -> [u8; KEY_SIZE] {
-        assert!(KEY_SIZE > PREFIX_SIZE);
-        let key = &mut [0_u8; KEY_SIZE];
-
-        // prefix 5 or 17 bytes
-        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
-
-        // mui 4 bytes
-        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
-            .first_chunk_mut::<4>()
-            .unwrap() = mui.to_le_bytes();
-
-        // ltime 8 bytes
-        *key[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
-            .first_chunk_mut::<8>()
-            .unwrap() = ltime.to_le_bytes();
-
-        // status 1 byte
-        key[PREFIX_SIZE + 12] = status.into();
-
-        *key
-    }
-
-    #[cfg(feature = "persist")]
-    pub fn prefix_mui_persistence_key(
-        prefix_id: PrefixId<AF>,
-        mui: u32,
-    ) -> Vec<u8> {
-        let mut key = vec![];
-        // prefix 5 or 17 bytes
-        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
-
-        // mui 4 bytes
-        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
-            .first_chunk_mut::<4>()
-            .unwrap() = mui.to_le_bytes();
-
-        key
-    }
-
-    #[cfg(feature = "persist")]
-    pub fn parse_key(bytes: &[u8]) -> ([u8; PREFIX_SIZE], u32, u64, u8) {
-        (
-            // prefix 5 or 17 bytes
-            *bytes.first_chunk::<PREFIX_SIZE>().unwrap(),
-            // mui 4 bytes
-            u32::from_le_bytes(
-                *bytes[PREFIX_SIZE..PREFIX_SIZE + 4]
-                    .first_chunk::<4>()
-                    .unwrap(),
-            ),
-            // ltime 8 bytes
-            u64::from_le_bytes(
-                *bytes[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
-                    .first_chunk::<8>()
-                    .unwrap(),
-            ),
-            // status 1 byte
-            bytes[PREFIX_SIZE + 12],
-        )
-    }
-
-    #[cfg(feature = "persist")]
-    pub fn parse_prefix(bytes: &[u8]) -> [u8; PREFIX_SIZE] {
-        *bytes.first_chunk::<PREFIX_SIZE>().unwrap()
-    }
-
-    #[cfg(feature = "persist")]
-    pub(crate) fn persist_record<M: Meta>(
-        &self,
-        prefix: PrefixId<AF>,
-        mui: u32,
-        record: &PublicRecord<M>,
-    ) {
-        self.insert(
-            PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
-                prefix,
-                mui,
-                record.ltime,
-                record.status,
-            ),
-            record.meta.as_ref(),
-        );
-    }
-}
-
-impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
-    std::fmt::Debug for PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
-{
-    fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        todo!()
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub enum PersistStrategy {
-    WriteAhead,
-    PersistHistory,
-    MemoryOnly,
-    PersistOnly,
-}
-
-#[derive(Debug, Clone)]
-pub struct StoreConfig {
-    pub persist_strategy: PersistStrategy,
-    pub persist_path: String,
-}
-
-impl StoreConfig {
-    pub fn persist_strategy(&self) -> PersistStrategy {
-        self.persist_strategy
-    }
-}
-
-// ----------- CustomAllocStorage -------------------------------------------
+// ----------- Rib -----------------------------------------------------------
 //
-// CustomAllocStorage is a storage backend that uses a custom allocator, that
-// consists of arrays that point to other arrays on collision.
+// A Routing Information Base that consists of multiple different trees for
+// in-memory and on-disk (persisted storage).
 #[derive(Debug)]
 pub struct Rib<
     AF: AddressFamily,
@@ -539,9 +350,9 @@ pub struct Rib<
     const KEY_SIZE: usize,
 > {
     pub config: StoreConfig,
-    pub(crate) in_memory_tree: TreeBitMap<AF, M, NB, PB>,
+    pub(in crate::local_array) in_memory_tree: TreeBitMap<AF, M, NB, PB>,
     #[cfg(feature = "persist")]
-    pub persist_tree: Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
+    persist_tree: Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
     // Global Roaring BitMap INdex that stores MUIs.
     pub counters: Counters,
 }
@@ -562,21 +373,18 @@ impl<
     ) -> Result<Self, Box<dyn std::error::Error>> {
         info!("store: initialize store {}", AF::BITS);
 
-        let persistence = match config.persist_strategy {
+        let persist_tree = match config.persist_strategy {
             PersistStrategy::MemoryOnly => None,
             _ => {
-                let persist_path = Path::new(&config.persist_path);
-                Some(PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>(
-                    lsm_tree::Config::new(persist_path).open()?,
-                    PhantomData,
-                ))
+                let persist_path = &Path::new(&config.persist_path);
+                Some(PersistTree::new(persist_path))
             }
         };
 
         let store = Rib {
             config,
             in_memory_tree: TreeBitMap::<AF, M, NB, PB>::init(),
-            persist_tree: persistence,
+            persist_tree,
             counters: Counters::default(),
         };
 
@@ -943,6 +751,18 @@ impl<
         })
     }
 
+    pub fn withdrawn_muis_bmin(
+        &'a self,
+        guard: &'a Guard,
+    ) -> &'a RoaringBitmap {
+        unsafe {
+            self.in_memory_tree
+                .withdrawn_muis_bmin
+                .load(Ordering::Acquire, guard)
+                .deref()
+        }
+    }
+
     // Change the status of the record for the specified (prefix, mui)
     // combination  to Withdrawn.
     pub fn mark_mui_as_withdrawn_for_prefix(
@@ -983,96 +803,97 @@ impl<
 
     // Change the status of the mui globally to Withdrawn. Iterators and match
     // functions will by default not return any records for this mui.
-    // pub fn mark_mui_as_withdrawn(
-    //     &self,
-    //     mui: u32,
-    //     guard: &Guard,
-    // ) -> Result<(), PrefixStoreError> {
-    //     let current = self
-    //         .in_memory_tree
-    //         .withdrawn_muis_bmin
-    //         .load(Ordering::Acquire, guard);
-
-    //     let mut new = unsafe { current.as_ref() }.unwrap().clone();
-    //     new.insert(mui);
-
-    //     #[allow(clippy::assigning_clones)]
-    //     loop {
-    //         match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
-    //             current,
-    //             Owned::new(new),
-    //             Ordering::AcqRel,
-    //             Ordering::Acquire,
-    //             guard,
-    //         ) {
-    //             Ok(_) => return Ok(()),
-    //             Err(updated) => {
-    //                 new =
-    //                     unsafe { updated.current.as_ref() }.unwrap().clone();
-    //             }
-    //         }
-    //     }
-    // }
+    pub fn mark_mui_as_withdrawn(
+        &self,
+        mui: u32,
+        guard: &Guard,
+    ) -> Result<(), PrefixStoreError> {
+        let current = self
+            .in_memory_tree
+            .withdrawn_muis_bmin
+            .load(Ordering::Acquire, guard);
+
+        let mut new = unsafe { current.as_ref() }.unwrap().clone();
+        new.insert(mui);
+
+        #[allow(clippy::assigning_clones)]
+        loop {
+            match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
+                current,
+                Owned::new(new),
+                Ordering::AcqRel,
+                Ordering::Acquire,
+                guard,
+            ) {
+                Ok(_) => return Ok(()),
+                Err(updated) => {
+                    new =
+                        unsafe { updated.current.as_ref() }.unwrap().clone();
+                }
+            }
+        }
+    }
 
     // Change the status of the mui globally to Active. Iterators and match
     // functions will default to the status on the record itself.
-    // pub fn mark_mui_as_active(
-    //     &self,
-    //     mui: u32,
-    //     guard: &Guard,
-    // ) -> Result<(), PrefixStoreError> {
-    //     let current = self
-    //         .in_memory_tree
-    //         .withdrawn_muis_bmin
-    //         .load(Ordering::Acquire, guard);
-
-    //     let mut new = unsafe { current.as_ref() }.unwrap().clone();
-    //     new.remove(mui);
-
-    //     #[allow(clippy::assigning_clones)]
-    //     loop {
-    //         match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
-    //             current,
-    //             Owned::new(new),
-    //             Ordering::AcqRel,
-    //             Ordering::Acquire,
-    //             guard,
-    //         ) {
-    //             Ok(_) => return Ok(()),
-    //             Err(updated) => {
-    //                 new =
-    //                     unsafe { updated.current.as_ref() }.unwrap().clone();
-    //             }
-    //         }
-    //     }
-    // }
+    pub fn mark_mui_as_active(
+        &self,
+        mui: u32,
+        guard: &Guard,
+    ) -> Result<(), PrefixStoreError> {
+        let current = self
+            .in_memory_tree
+            .withdrawn_muis_bmin
+            .load(Ordering::Acquire, guard);
 
-    // Whether this mui is globally withdrawn. Note that this overrules (by
-    // default) any (prefix, mui) combination in iterators and match functions.
-    // pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
-    //     unsafe {
-    //         self.in_memory_tree
-    //             .withdrawn_muis_bmin
-    //             .load(Ordering::Acquire, guard)
-    //             .as_ref()
-    //     }
-    //     .unwrap()
-    //     .contains(mui)
-    // }
+        let mut new = unsafe { current.as_ref() }.unwrap().clone();
+        new.remove(mui);
+
+        #[allow(clippy::assigning_clones)]
+        loop {
+            match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
+                current,
+                Owned::new(new),
+                Ordering::AcqRel,
+                Ordering::Acquire,
+                guard,
+            ) {
+                Ok(_) => return Ok(()),
+                Err(updated) => {
+                    new =
+                        unsafe { updated.current.as_ref() }.unwrap().clone();
+                }
+            }
+        }
+    }
+
+    // Whether this mui is globally withdrawn. Note that this overrules
+    // (by default) any (prefix, mui) combination in iterators and match
+    // functions.
+    pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
+        unsafe {
+            self.in_memory_tree
+                .withdrawn_muis_bmin
+                .load(Ordering::Acquire, guard)
+                .as_ref()
+        }
+        .unwrap()
+        .contains(mui)
+    }
 
     // Whether this mui is globally active. Note that the local statuses of
     // records (prefix, mui) may be set to withdrawn in iterators and match
     // functions.
-    // pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
-    //     !unsafe {
-    //         self.in_memory_tree
-    //             .withdrawn_muis_bmin
-    //             .load(Ordering::Acquire, guard)
-    //             .as_ref()
-    //     }
-    //     .unwrap()
-    //     .contains(mui)
-    // }
+    pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
+        !unsafe {
+            self.in_memory_tree
+                .withdrawn_muis_bmin
+                .load(Ordering::Acquire, guard)
+                .as_ref()
+        }
+        .unwrap()
+        .contains(mui)
+    }
 
     // This function is used by the upsert_prefix function above.
     //
@@ -1328,6 +1149,83 @@ impl<
         panic!("prefix length for {:?} is too long", prefix);
     }
 
+    //-------- Persistence ---------------------------------------------------
+
+    pub fn persist_strategy(&self) -> PersistStrategy {
+        self.config.persist_strategy
+    }
+
+    pub fn match_prefix_in_persisted_store(
+        &'a self,
+        search_pfx: PrefixId<AF>,
+        mui: Option<u32>,
+    ) -> QueryResult<M> {
+        let key: Vec<u8> = if let Some(mui) = mui {
+            PersistTree::<AF,
+        PREFIX_SIZE, KEY_SIZE>::prefix_mui_persistence_key(search_pfx, mui)
+        } else {
+            search_pfx.as_bytes::<PREFIX_SIZE>().to_vec()
+        };
+
+        if let Some(persist) = &self.persist_tree {
+            QueryResult {
+                prefix: Some(search_pfx.into_pub()),
+                match_type: MatchType::ExactMatch,
+                prefix_meta: persist
+                    .get_records_for_key(&key)
+                    .into_iter()
+                    .map(|(_, rec)| rec)
+                    .collect::<Vec<_>>(),
+                less_specifics: None,
+                more_specifics: None,
+            }
+        } else {
+            QueryResult {
+                prefix: None,
+                match_type: MatchType::EmptyMatch,
+                prefix_meta: vec![],
+                less_specifics: None,
+                more_specifics: None,
+            }
+        }
+    }
+
+    pub fn get_records_for_prefix(
+        &self,
+        prefix: &Prefix,
+    ) -> Vec<PublicRecord<M>> {
+        if let Some(p) = &self.persist_tree {
+            p.get_records_for_prefix(PrefixId::from(*prefix))
+        } else {
+            vec![]
+        }
+    }
+
+    pub fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
+        if let Some(p) = &self.persist_tree {
+            p.flush_to_disk()
+                .map_err(|_| PrefixStoreError::PersistFailed)
+        } else {
+            Err(PrefixStoreError::PersistFailed)
+        }
+    }
+
+    pub fn approx_persisted_items(&self) -> usize {
+        if let Some(p) = &self.persist_tree {
+            p.approximate_len()
+        } else {
+            0
+        }
+    }
+
+    pub fn disk_space(&self) -> u64 {
+        if let Some(p) = &self.persist_tree {
+            p.disk_space()
+        } else {
+            0
+        }
+    }
+
     // ------- THE HASHING FUNCTION -----------------------------------------
 
     // Ok, so hashing is really hard, but we're keeping it simple, and

From 05f0f4e8164aa52e95330d4902f0dda431f7669b Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 18 Dec 2024 11:14:02 +0100
Subject: [PATCH 024/147] remove withdrawn_mui_bmin() calls from proc macro

---
 proc_macros/src/lib.rs       | 51 ++++++++++++++++--------------------
 src/local_array/iterators.rs | 40 ++--------------------------
 src/local_array/rib/rib.rs   |  4 +--
 3 files changed, 27 insertions(+), 68 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index c505baa0..97c96c6c 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -1046,30 +1046,28 @@ pub fn create_store(
                 let (left, right) = match search_pfx.addr() {
                     std::net::IpAddr::V4(addr) => {
 
-                        let bmin = self.v6.withdrawn_muis_bmin(guard);
-
-                        if mui.is_some() && bmin.contains(mui.unwrap()) {
-                          (None, None)
-                         } else {
-                                (Some(self.v4.more_specific_prefix_iter_from(
-                                        PrefixId::<IPv4>::new(
-                                            addr.into(),
-                                            search_pfx.len(),
-                                        ),
-                                        mui,
-                                        include_withdrawn,
-                                        guard
-                                    ).map(|p| PrefixRecord::from(p))
-                                ),
-                                None)
-                            }
+                        if mui.is_some_and(
+                            |m| { self.v6.mui_is_withdrawn(m, guard) }) {
+                                (None, None)
+                        } else {
+                            (Some(self.v4.more_specific_prefix_iter_from(
+                                    PrefixId::<IPv4>::new(
+                                        addr.into(),
+                                        search_pfx.len(),
+                                    ),
+                                    mui,
+                                    include_withdrawn,
+                                    guard
+                                ).map(|p| PrefixRecord::from(p))
+                            ),
+                            None)
                         }
+                    }
                     std::net::IpAddr::V6(addr) => {
 
-                        let bmin = self.v6.withdrawn_muis_bmin(guard);
-
-                        if mui.is_some() && bmin.contains(mui.unwrap()) {
-                         (None, None)
+                        if mui.is_some_and(
+                            |m| { self.v6.mui_is_withdrawn(m, guard) }) {
+                                (None, None)
                         } else {
                             (
                                 None,
@@ -1097,10 +1095,8 @@ pub fn create_store(
                 guard: &'a Guard
             ) -> impl Iterator<Item=PrefixRecord<M>> +'a {
 
-                let bmin =
-                    self.v4.withdrawn_muis_bmin(guard);
-
-                if bmin.contains(mui) && !include_withdrawn {
+                if self.v4.mui_is_withdrawn(mui, guard)
+                    && !include_withdrawn {
                     None
                 } else {
                     Some(
@@ -1124,9 +1120,8 @@ pub fn create_store(
                 guard: &'a Guard
             ) -> impl Iterator<Item=PrefixRecord<M>> +'a {
 
-                let bmin = self.v6.withdrawn_muis_bmin(guard);
-
-                if bmin.contains(mui) && !include_withdrawn {
+                if self.v6.mui_is_withdrawn(mui, guard)
+                    && !include_withdrawn {
                     None
                 } else {
                     Some(
diff --git a/src/local_array/iterators.rs b/src/local_array/iterators.rs
index c5b9abe9..254224a0 100644
--- a/src/local_array/iterators.rs
+++ b/src/local_array/iterators.rs
@@ -707,7 +707,7 @@ impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
     }
 }
 
-// ----------- Iterator initialization methods for CustomAllocStorage -------
+// ----------- Iterator initialization methods for Rib -----------------------
 
 // These are only the methods that are starting the iterations. All other
 // methods for Rib are in the main rib.rs file.
@@ -733,7 +733,7 @@ impl<
     ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
         trace!("more specifics for {:?}", start_prefix_id);
 
-        // A v4 /32 or a v4 /128 doesn't have more specific prefixes 🤓.
+        // A v4 /32 or a v6 /128 doesn't have more specific prefixes 🤓.
         if start_prefix_id.get_len() >= AF::BITS {
             None
         } else {
@@ -896,39 +896,3 @@ impl<
         }
     }
 }
-
-// ----------- InternalPrefixRecord -> RecordSet (public) -------------------
-
-// impl<'a, AF: AddressFamily, Meta: crate::prefix_record::Meta>
-//     std::iter::FromIterator<InternalPrefixRecord<AF, Meta>>
-//     for routecore::bgp::RecordSet<'a, Meta>
-// {
-//     fn from_iter<I: IntoIterator<Item = InternalPrefixRecord<AF, Meta>>>(
-//         iter: I,
-//     ) -> Self {
-//         let mut v4 = vec![];
-//         let mut v6 = vec![];
-//         for pfx in iter {
-//             let addr = pfx.net.into_ipaddr();
-//             match addr {
-//                 std::net::IpAddr::V4(_) => {
-//                     v4.push(
-//                         routecore::bgp::PrefixRecord::new_with_local_meta(
-//                             Prefix::new(addr, pfx.len).unwrap(),
-//                             pfx.meta,
-//                         ),
-//                     );
-//                 }
-//                 std::net::IpAddr::V6(_) => {
-//                     v6.push(
-//                         routecore::bgp::PrefixRecord::new_with_local_meta(
-//                             Prefix::new(addr, pfx.len).unwrap(),
-//                             pfx.meta,
-//                         ),
-//                     );
-//                 }
-//             }
-//         }
-//         Self { v4, v6 }
-//     }
-// }
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 20798eb4..0c3376ec 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -350,7 +350,7 @@ pub struct Rib<
     const KEY_SIZE: usize,
 > {
     pub config: StoreConfig,
-    pub(in crate::local_array) in_memory_tree: TreeBitMap<AF, M, NB, PB>,
+    pub(crate) in_memory_tree: TreeBitMap<AF, M, NB, PB>,
     #[cfg(feature = "persist")]
     persist_tree: Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
     // Global Roaring BitMap INdex that stores MUIs.
@@ -751,7 +751,7 @@ impl<
         })
     }
 
-    pub fn withdrawn_muis_bmin(
+    pub(crate) fn withdrawn_muis_bmin(
         &'a self,
         guard: &'a Guard,
     ) -> &'a RoaringBitmap {

From 0f8b81bd741ec26338766b94a8ba2554aabf66b1 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 18 Dec 2024 11:33:50 +0100
Subject: [PATCH 025/147] move helper fn

---
 src/local_array/query.rs   | 13 -------------
 src/local_array/rib/rib.rs | 13 +++++++++++++
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 24678f4f..e1ac3c3b 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -886,17 +886,4 @@ where
             },
         }
     }
-
-    // Helper to filter out records that are not-active (Inactive or
-    // Withdrawn), or whose mui appears in the global withdrawn index.
-    fn get_filtered_records(
-        &self,
-        pfx: &StoredPrefix<AF, M>,
-        mui: Option<u32>,
-        guard: &Guard,
-    ) -> Vec<PublicRecord<M>> {
-        let bmin = self.withdrawn_muis_bmin(guard);
-
-        pfx.record_map.get_filtered_records(mui, bmin)
-    }
 }
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 0c3376ec..300c6583 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -895,6 +895,19 @@ impl<
         .contains(mui)
     }
 
+    // Helper to filter out records that are not-active (Inactive or
+    // Withdrawn), or whose mui appears in the global withdrawn index.
+    pub fn get_filtered_records(
+        &self,
+        pfx: &StoredPrefix<AF, M>,
+        mui: Option<u32>,
+        guard: &Guard,
+    ) -> Vec<PublicRecord<M>> {
+        let bmin = self.withdrawn_muis_bmin(guard);
+
+        pfx.record_map.get_filtered_records(mui, bmin)
+    }
+
     // This function is used by the upsert_prefix function above.
     //
     // We're using a Chained Hash Table and this function returns one of:

From 5899b83341b67759511c06f0d08193b63515ef35 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 18 Dec 2024 14:38:53 +0100
Subject: [PATCH 026/147] split out treebitmap and rib

---
 src/local_array/in_memory/deprecated_query.rs |  709 ++++++++
 src/local_array/in_memory/macros.rs           |   98 +-
 src/local_array/in_memory/mod.rs              |    3 +
 src/local_array/in_memory/tree.rs             | 1139 ++++++++----
 src/local_array/iterators.rs                  |   13 +-
 src/local_array/query.rs                      |  619 +------
 src/local_array/rib/rib.rs                    | 1447 ++++++++-------
 src/local_array/store/atomic_types.rs         |  760 --------
 src/local_array/store/custom_alloc.rs         | 1553 -----------------
 src/local_array/store/macros.rs               |  456 -----
 src/local_array/store/oncebox.rs              |  210 ---
 11 files changed, 2268 insertions(+), 4739 deletions(-)
 create mode 100644 src/local_array/in_memory/deprecated_query.rs
 delete mode 100644 src/local_array/store/atomic_types.rs
 delete mode 100644 src/local_array/store/custom_alloc.rs
 delete mode 100644 src/local_array/store/macros.rs
 delete mode 100644 src/local_array/store/oncebox.rs

diff --git a/src/local_array/in_memory/deprecated_query.rs b/src/local_array/in_memory/deprecated_query.rs
new file mode 100644
index 00000000..467444b8
--- /dev/null
+++ b/src/local_array/in_memory/deprecated_query.rs
@@ -0,0 +1,709 @@
+use log::trace;
+
+use crate::af::AddressFamily;
+use crate::prelude::multi::PrefixSet;
+use crate::rib::Rib;
+use inetnum::addr::Prefix;
+
+use crate::{Meta, QueryResult};
+
+use crate::local_array::in_memory::node::TreeBitMapNode;
+use crate::{MatchOptions, MatchType};
+
+use super::super::in_memory::atomic_types::StoredPrefix;
+use super::super::in_memory::tree::{SizedStrideRef, Stride, StrideNodeId};
+use super::super::types::PrefixId;
+use super::atomic_types::{NodeBuckets, PrefixBuckets};
+use super::tree::TreeBitMap;
+
+#[allow(dead_code)]
+impl<'a, AF, M, NB, PB> TreeBitMap<AF, M, NB, PB>
+where
+    AF: AddressFamily,
+    M: Meta,
+    NB: NodeBuckets<AF>,
+    PB: PrefixBuckets<AF, M>,
+{
+    #[allow(clippy::type_complexity)]
+    fn retrieve_prefix(
+        &'a self,
+        prefix_id: PrefixId<AF>,
+    ) -> Option<(&'a StoredPrefix<AF, M>, usize)> {
+        struct SearchLevel<
+            's,
+            AF: AddressFamily,
+            M: crate::prefix_record::Meta,
+        > {
+            f: &'s dyn for<'a> Fn(
+                &SearchLevel<AF, M>,
+                &'a PrefixSet<AF, M>,
+                u8,
+            )
+                -> Option<(&'a StoredPrefix<AF, M>, usize)>,
+        }
+
+        let search_level = SearchLevel {
+            f: &|search_level: &SearchLevel<AF, M>,
+                 prefix_set: &PrefixSet<AF, M>,
+                 mut level: u8| {
+                // HASHING FUNCTION
+                let index =
+                    crate::local_array::in_memory::tree::TreeBitMap::<
+                        AF,
+                        M,
+                        NB,
+                        PB,
+                    >::hash_prefix_id(prefix_id, level);
+
+                if let Some(stored_prefix) = prefix_set.0.get(index) {
+                    if prefix_id == stored_prefix.prefix {
+                        trace!("found requested prefix {:?}", prefix_id,);
+                        return Some((stored_prefix, 0));
+                    };
+                    level += 1;
+
+                    (search_level.f)(
+                        search_level,
+                        &stored_prefix.next_bucket,
+                        level,
+                    );
+                }
+                None
+            },
+        };
+
+        (search_level.f)(
+            &search_level,
+            self.prefix_buckets.get_root_prefix_set(prefix_id.get_len()),
+            0,
+        )
+    }
+
+    // This function assembles all entries in the `pfx_vec` of all child nodes
+    // of the `start_node` into one vec, starting from itself and then
+    // recursively assembling adding all `pfx_vec`s of its children.
+    fn get_all_more_specifics_for_node(
+        &self,
+        start_node_id: StrideNodeId<AF>,
+        found_pfx_vec: &mut Vec<PrefixId<AF>>,
+    ) {
+        trace!("{:?}", self.retrieve_node(start_node_id));
+        match self.retrieve_node(start_node_id) {
+            Some(SizedStrideRef::Stride3(n)) => {
+                found_pfx_vec.extend(
+                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
+                );
+
+                for child_node in n.ptr_iter(start_node_id) {
+                    self.get_all_more_specifics_for_node(
+                        child_node,
+                        found_pfx_vec,
+                    );
+                }
+            }
+            Some(SizedStrideRef::Stride4(n)) => {
+                found_pfx_vec.extend(
+                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
+                );
+
+                for child_node in n.ptr_iter(start_node_id) {
+                    self.get_all_more_specifics_for_node(
+                        child_node,
+                        found_pfx_vec,
+                    );
+                }
+            }
+            Some(SizedStrideRef::Stride5(n)) => {
+                found_pfx_vec.extend(
+                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
+                );
+
+                for child_node in n.ptr_iter(start_node_id) {
+                    self.get_all_more_specifics_for_node(
+                        child_node,
+                        found_pfx_vec,
+                    );
+                }
+            }
+            _ => {
+                panic!("can't find node {}", start_node_id);
+            }
+        }
+    }
+
+    // This function assembles the prefixes of a child node starting on a
+    // specified bit position in a ptr_vec of `current_node` into a vec,
+    // then adds all prefixes of these children recursively into a vec and
+    // returns that.
+    fn get_all_more_specifics_from_nibble<S: Stride>(
+        &self,
+        current_node: &TreeBitMapNode<AF, S>,
+        nibble: u32,
+        nibble_len: u8,
+        base_prefix: StrideNodeId<AF>,
+    ) -> Option<Vec<PrefixId<AF>>> {
+        let (cnvec, mut msvec) = current_node.add_more_specifics_at(
+            nibble,
+            nibble_len,
+            base_prefix,
+        );
+
+        for child_node in cnvec.iter() {
+            self.get_all_more_specifics_for_node(*child_node, &mut msvec);
+        }
+        Some(msvec)
+    }
+
+    // In a LMP search we have to go over all the nibble lengths in the
+    // stride up until the value of the actual nibble length were looking for
+    // (until we reach stride length for all strides that aren't the last)
+    // and see if the prefix bit in that position is set. Note that this does
+    // not search for prefixes with length 0 (which would always match).
+    // So for matching a nibble 1010, we have to search for 1, 10, 101 and
+    // 1010 on resp. position 1, 5, 12 and 25:
+    //                       ↓          ↓                         ↓
+    // nibble              * 0 1 00 01 10 11 000 001 010 011 100 101 110 111
+    // nibble len offset   0 1    2            3
+    //
+    // (contd.)
+    // pfx bit arr (u32)     15   16   17   18   19   20   21   22   23   24
+    // nibble              0000 0001 0010 0011 0100 0101 0110 0111 1000 1001
+    // nibble len offset      4
+    //
+    // (contd.)               ↓
+    // pfx bit arr (u32)     25   26   27   28   29   30   31
+    // nibble              1010 1011 1100 1101 1110 1111    x
+    // nibble len offset      4(contd.)
+
+    fn match_prefix_by_tree_traversal(
+        &'a self,
+        search_pfx: PrefixId<AF>,
+        options: &MatchOptions,
+        // guard: &'a Guard,
+    ) -> QueryResult<M> {
+        // --- The Default Route Prefix -------------------------------------
+
+        // The Default Route Prefix unfortunately does not fit in tree as we
+        // have it. There's no room for it in the pfxbitarr of the root node,
+        // since that can only contain serial numbers for prefixes that are
+        // children of the root node. We, however, want the default prefix
+        // which lives on the root node itself! We are *not* going to return
+        // all of the prefixes in the tree as more-specifics.
+        if search_pfx.get_len() == 0 {
+            // match self.load_default_route_prefix_serial() {
+            //     0 => {
+            //         return QueryResult {
+            //             prefix: None,
+            //             prefix_meta: vec![],
+            //             match_type: MatchType::EmptyMatch,
+            //             less_specifics: None,
+            //             more_specifics: None,
+            //         };
+            //     }
+
+            //     _serial => {
+            let prefix_meta = self
+                .retrieve_prefix(PrefixId::new(AF::zero(), 0))
+                .map(|sp| sp.0.record_map.as_records())
+                .unwrap_or_default();
+            return QueryResult {
+                prefix: Prefix::new(
+                    search_pfx.get_net().into_ipaddr(),
+                    search_pfx.get_len(),
+                )
+                .ok(),
+                prefix_meta,
+                match_type: MatchType::ExactMatch,
+                less_specifics: None,
+                more_specifics: None,
+            };
+            // }
+            // }
+        }
+
+        let mut stride_end = 0;
+
+        let root_node_id = self.get_root_node_id();
+        let mut node = match self.get_stride_for_id(root_node_id) {
+            3 => self.retrieve_node(root_node_id).unwrap(),
+            4 => self.retrieve_node(root_node_id).unwrap(),
+            _ => self.retrieve_node(root_node_id).unwrap(),
+        };
+
+        let mut nibble;
+        let mut nibble_len;
+
+        //---- result values ------------------------------------------------
+
+        // These result values are kept in mutable variables, and assembled
+        // at the end into a QueryResult struct. This proved to result in the
+        // most efficient code, where we don't have to match on
+        // SizedStrideNode over and over. The `match_type` field in the
+        // QueryResult is computed at the end.
+
+        // The final prefix
+        let mut match_prefix_idx: Option<PrefixId<AF>> = None;
+
+        // The indexes of the less-specifics
+        let mut less_specifics_vec = if options.include_less_specifics {
+            Some(Vec::<PrefixId<AF>>::new())
+        } else {
+            None
+        };
+
+        // The indexes of the more-specifics.
+        let mut more_specifics_vec = if options.include_more_specifics {
+            Some(Vec::<PrefixId<AF>>::new())
+        } else {
+            None
+        };
+
+        //---- Stride Processing --------------------------------------------
+
+        // We're going to iterate over all the strides in the treebitmap (so
+        // up to the last bit in the max prefix length for that tree). When
+        // a final prefix is found or we get to the end of the strides,
+        // depending on the options.match_type (the type requested by the
+        // user). we ALWAYS break out of the loop. WE ALWAYS BREAK OUT OF THE
+        // LOOP. Just before breaking some processing is done inside the loop
+        // before the break (looking up more-specifics mainly), which looks a
+        // bit repetitious, but again it's been done like that to avoid
+        // having to match over a SizedStrideNode again in the
+        // `post-processing` section.
+
+        for stride in self.get_stride_sizes() {
+            stride_end += stride;
+
+            let last_stride = search_pfx.get_len() < stride_end;
+
+            nibble_len = if last_stride {
+                stride + search_pfx.get_len() - stride_end
+            } else {
+                *stride
+            };
+
+            // Shift left and right to set the bits to zero that are not
+            // in the nibble we're handling here.
+            nibble = AddressFamily::get_nibble(
+                search_pfx.get_net(),
+                stride_end - stride,
+                nibble_len,
+            );
+
+            match node {
+                SizedStrideRef::Stride3(current_node) => {
+                    let search_fn = match options.match_type {
+                        MatchType::ExactMatch => {
+                            if options.include_less_specifics {
+                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
+                            } else {
+                                TreeBitMapNode::search_stride_for_exact_match_at
+                            }
+                        }
+                        MatchType::LongestMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                        MatchType::EmptyMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                    };
+
+                    // This whole match assumes that:
+                    // - if the first value in the return tuple of
+                    //   `search_fn` holds a value, then we need to continue
+                    //   searching by following the node contained in the
+                    //   value.
+                    // - The second value in the tuple holds the prefix that
+                    //   was found.
+                    // The less_specifics_vec is mutated by `search_fn` to
+                    // hold the prefixes found along the way, in the cases
+                    // where `include_less_specifics` was requested by the
+                    // user.
+                    match search_fn(
+                        current_node,
+                        search_pfx,
+                        nibble,
+                        nibble_len,
+                        stride_end - stride,
+                        &mut less_specifics_vec,
+                    ) {
+                        // This and the next match will handle all
+                        // intermediary nodes, but they might also handle
+                        // exit nodes.
+                        (Some(n), Some(pfx_idx)) => {
+                            match_prefix_idx = Some(pfx_idx);
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        (Some(n), None) => {
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        // This handles exact and longest matches: there are
+                        // no more children, but there is a prefix on this
+                        // node.
+                        (None, Some(pfx_idx)) => {
+                            if options.include_more_specifics {
+                                more_specifics_vec = self
+                                    .get_all_more_specifics_from_nibble(
+                                        current_node,
+                                        nibble,
+                                        nibble_len,
+                                        StrideNodeId::new_with_cleaned_id(
+                                            search_pfx.get_net(),
+                                            stride_end - stride,
+                                        ),
+                                    );
+                            }
+                            match_prefix_idx = Some(pfx_idx);
+                            break;
+                        }
+                        // This handles cases where there's no prefix (and no
+                        // child) for exact match or longest match, the empty
+                        // match - which doesn't care about actually finding
+                        // a prefix - just continues in search of
+                        // more-specifics.
+                        (None, None) => {
+                            match options.match_type {
+                                MatchType::EmptyMatch => {
+                                    // To make sure we don't process this
+                                    // match arm more then once, we return
+                                    // early here.
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+
+                                    match_prefix_idx = None;
+                                    break;
+                                }
+                                MatchType::LongestMatch => {}
+                                MatchType::ExactMatch => {
+                                    match_prefix_idx = None;
+                                }
+                            }
+                            break;
+                        }
+                    }
+                }
+                //---- From here only repetitions for all strides -----------
+                // For comments see the code above for the Stride3 arm.
+                SizedStrideRef::Stride4(current_node) => {
+                    let search_fn = match options.match_type {
+                        MatchType::ExactMatch => {
+                            if options.include_less_specifics {
+                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
+                            } else {
+                                TreeBitMapNode::search_stride_for_exact_match_at
+                            }
+                        }
+                        MatchType::LongestMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                        MatchType::EmptyMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                    };
+                    match search_fn(
+                        current_node,
+                        search_pfx,
+                        nibble,
+                        nibble_len,
+                        stride_end - stride,
+                        &mut less_specifics_vec,
+                    ) {
+                        (Some(n), Some(pfx_idx)) => {
+                            match_prefix_idx = Some(pfx_idx);
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        (Some(n), None) => {
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        (None, Some(pfx_idx)) => {
+                            if options.include_more_specifics {
+                                more_specifics_vec = self
+                                    .get_all_more_specifics_from_nibble(
+                                        current_node,
+                                        nibble,
+                                        nibble_len,
+                                        StrideNodeId::new_with_cleaned_id(
+                                            search_pfx.get_net(),
+                                            stride_end - stride,
+                                        ),
+                                    );
+                            }
+                            match_prefix_idx = Some(pfx_idx);
+                            break;
+                        }
+                        (None, None) => {
+                            match options.match_type {
+                                MatchType::EmptyMatch => {
+                                    // To make sure we don't process this match arm more then once, we
+                                    // return early here.
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+
+                                    match_prefix_idx = None;
+                                    break;
+                                }
+                                MatchType::LongestMatch => {}
+                                MatchType::ExactMatch => {
+                                    match_prefix_idx = None;
+                                }
+                            }
+                            break;
+                        }
+                    }
+                }
+                SizedStrideRef::Stride5(current_node) => {
+                    let search_fn = match options.match_type {
+                        MatchType::ExactMatch => {
+                            if options.include_less_specifics {
+                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
+                            } else {
+                                TreeBitMapNode::search_stride_for_exact_match_at
+                            }
+                        }
+                        MatchType::LongestMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                        MatchType::EmptyMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                    };
+                    match search_fn(
+                        current_node,
+                        search_pfx,
+                        nibble,
+                        nibble_len,
+                        stride_end - stride,
+                        &mut less_specifics_vec,
+                    ) {
+                        (Some(n), Some(pfx_idx)) => {
+                            match_prefix_idx = Some(pfx_idx);
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        (Some(n), None) => {
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        (None, Some(pfx_idx)) => {
+                            if options.include_more_specifics {
+                                more_specifics_vec = self
+                                    .get_all_more_specifics_from_nibble(
+                                        current_node,
+                                        nibble,
+                                        nibble_len,
+                                        StrideNodeId::new_with_cleaned_id(
+                                            search_pfx.get_net(),
+                                            stride_end - stride,
+                                        ),
+                                    );
+                            }
+                            match_prefix_idx = Some(pfx_idx);
+                            break;
+                        }
+                        (None, None) => {
+                            match options.match_type {
+                                MatchType::EmptyMatch => {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+
+                                    match_prefix_idx = None;
+                                    break;
+                                }
+                                MatchType::LongestMatch => {}
+                                MatchType::ExactMatch => {
+                                    match_prefix_idx = None;
+                                }
+                            }
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        //------------------ end of Stride branch arm repetition ------------
+
+        //------------------ post-processing --------------------------------
+
+        // If the above loop finishes (so not hitting a break) we have
+        // processed all strides and have found a child node and maybe a
+        // prefix. Now we will look up more-specifics for longest-matching
+        // prefixes that were found in the last stride only. Note that still
+        // any of the match_types (as specified by the user, not the return
+        // type) may end up here.
+
+        let mut match_type: MatchType = MatchType::EmptyMatch;
+        let prefix = None;
+        if let Some(pfx_idx) = match_prefix_idx {
+            match_type = match self.retrieve_prefix(pfx_idx) {
+                Some(prefix) => {
+                    if prefix.0.prefix.get_len() == search_pfx.get_len() {
+                        MatchType::ExactMatch
+                    } else {
+                        MatchType::LongestMatch
+                    }
+                }
+                None => MatchType::EmptyMatch,
+            };
+        };
+
+        QueryResult {
+            prefix: prefix.map(|pfx: (&StoredPrefix<AF, M>, usize)| {
+                pfx.0.prefix.into_pub()
+            }),
+            prefix_meta: prefix
+                .map(|pfx| pfx.0.record_map.as_records())
+                .unwrap_or_default(),
+            match_type,
+            less_specifics: if options.include_less_specifics {
+                less_specifics_vec
+                    .unwrap()
+                    .iter()
+                    .filter_map(move |p| {
+                        self.retrieve_prefix(*p).map(|p| {
+                            Some((p.0.prefix, p.0.record_map.as_records()))
+                        })
+                    })
+                    .collect()
+            } else {
+                None
+            },
+            more_specifics: if options.include_more_specifics {
+                more_specifics_vec.map(|vec| {
+                    vec.into_iter()
+                        .map(|p| {
+                            self.retrieve_prefix(p)
+                                .unwrap_or_else(|| {
+                                    panic!(
+                                        "more specific {:?} does not exist",
+                                        p
+                                    )
+                                })
+                                .0
+                        })
+                        .map(|sp| (sp.prefix, sp.record_map.as_records()))
+                        .collect()
+                })
+            } else {
+                None
+            },
+        }
+    }
+}
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index 6c0f7ed2..b911c124 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -47,68 +47,116 @@ macro_rules! insert_match {
             // retry_count from this macro.
             let local_retry_count = 0;
             // retrieve_node_mut updates the bitmap index if necessary.
-            if let Some(current_node) = $self.retrieve_node_mut($cur_i, $record.multi_uniq_id) {
+            if let Some(current_node) = $self.in_memory_tree.retrieve_node_mut(
+                $cur_i, $record.multi_uniq_id) {
                 match current_node {
                     $(
                         SizedStrideRef::$variant(current_node) => {
-                            // eval_node_or_prefix_at mutates the node to reflect changes
-                            // in the ptrbitarr & pfxbitarr.
+                            // eval_node_or_prefix_at mutates the node to
+                            // reflect changes in the ptrbitarr & pfxbitarr.
                             match current_node.eval_node_or_prefix_at(
                                 $nibble,
                                 $nibble_len,
-                                // All the bits of the search prefix, but with a length set to
-                                // the start of the current stride.
-                                StrideNodeId::dangerously_new_with_id_as_is($pfx.get_net(), $truncate_len),
+                                // All the bits of the search prefix, but with
+                                // a length set to the start of the current
+                                // stride.
+                                StrideNodeId::dangerously_new_with_id_as_is(
+                                    $pfx.get_net(), $truncate_len),
                                 // the length of THIS stride
                                 $stride_len,
                                 // the length of the next stride
-                                $self.get_stride_sizes().get(($level + 1) as usize),
+                                $self
+                                    .in_memory_tree
+                                    .get_stride_sizes()
+                                    .get(($level + 1) as usize),
                                 $is_last_stride,
                             ) {
                                 (NewNodeOrIndex::NewNode(n), retry_count) => {
-                                    // Stride3 logs to stats[0], Stride4 logs to stats[1], etc.
+                                    // Stride3 logs to stats[0], Stride4 logs
+                                    // to stats[1], etc.
                                     // $self.stats[$stats_level].inc($level);
 
-                                    // get a new identifier for the node we're going to create.
-                                    let new_id = $self.acquire_new_node_id(($pfx.get_net(), $truncate_len + $nibble_len));
+                                    // get a new identifier for the node we're
+                                    // going to create.
+                                        let new_id =
+                                            StrideNodeId::new_with_cleaned_id(
+                                                $pfx.get_net(),
+                                                $truncate_len + $nibble_len
+                                           );
 
                                     // store the new node in the global
                                     // store. It returns the created id
                                     // and the number of retries before
                                     // success.
-                                    match $self.store_node(new_id, $record.multi_uniq_id, n)  {
+                                    match $self.in_memory_tree.store_node(
+                                        new_id,
+                                        $record.multi_uniq_id, n
+                                    ) {
                                         Ok((node_id, s_retry_count)) => {
-                                            Ok((node_id, $acc_retry_count + s_retry_count + retry_count))
+                                            Ok((
+                                                node_id,
+                                                $acc_retry_count +
+                                                s_retry_count +
+                                                retry_count
+                                            ))
                                         },
                                         Err(err) => {
                                             Err(err)
                                         }
                                     }
                                 }
-                                (NewNodeOrIndex::ExistingNode(node_id), retry_count) => {
-                                    // $self.store.update_node($cur_i,SizedStrideRefMut::$variant(current_node));
+                                (NewNodeOrIndex::ExistingNode(node_id),
+                                    retry_count
+                                ) => {
                                     if log_enabled!(log::Level::Trace) {
                                         if local_retry_count > 0 {
-                                            trace!("{} contention: Node already exists {}",
-                                            std::thread::current().name().unwrap_or("unnamed-thread"), node_id
+                                            trace!("{} contention: Node \
+                                                 already exists {}",
+                                            std::thread::current()
+                                                .name()
+                                                .unwrap_or("unnamed-thread"),
+                                                node_id
                                             )
                                         }
                                     }
-                                    Ok((node_id, $acc_retry_count + local_retry_count + retry_count))
+                                    Ok((
+                                        node_id,
+                                        $acc_retry_count +
+                                        local_retry_count +
+                                        retry_count
+                                    ))
                                 },
                                 (NewNodeOrIndex::NewPrefix, retry_count) => {
-                                    return $self.upsert_prefix($pfx, $record, $update_path_selections, $guard)
-                                        .and_then(|mut r| {
-                                            r.cas_count += $acc_retry_count as usize + local_retry_count as usize + retry_count as usize;
+                                    return $self.upsert_prefix(
+                                        $pfx,
+                                        $record,
+                                        $update_path_selections,
+                                        $guard
+                                    ).and_then(|mut r| {
+                                            r.cas_count +=
+                                                $acc_retry_count as usize +
+                                                local_retry_count as usize +
+                                                retry_count as usize;
                                             Ok(r)
                                         })
                                     // Log
-                                    // $self.stats[$stats_level].inc_prefix_count($level);
+                                    // $self.stats[$stats_level].
+                                    //inc_prefix_count($level);
                                 }
-                                (NewNodeOrIndex::ExistingPrefix, retry_count) => {
-                                    return $self.upsert_prefix($pfx, $record, $update_path_selections, $guard)
-                                        .and_then(|mut r| {
-                                            r.cas_count += $acc_retry_count as usize + local_retry_count as usize + retry_count as usize;
+                                (
+                                    NewNodeOrIndex::ExistingPrefix,
+                                     retry_count
+                                 ) =>
+                                     {
+                                        return $self.upsert_prefix(
+                                            $pfx,
+                                            $record,
+                                            $update_path_selections,$guard
+                                        ).and_then(|mut r| {
+                                            r.cas_count +=
+                                                $acc_retry_count as usize +
+                                                local_retry_count as usize +
+                                                retry_count as usize;
                                             Ok(r)
                                         })
                                 }
diff --git a/src/local_array/in_memory/mod.rs b/src/local_array/in_memory/mod.rs
index 0cb20754..164b8dc7 100644
--- a/src/local_array/in_memory/mod.rs
+++ b/src/local_array/in_memory/mod.rs
@@ -4,5 +4,8 @@ pub(crate) mod node;
 mod oncebox;
 pub(crate) mod tree;
 
+#[deprecated]
+mod deprecated_query;
+
 #[macro_use]
 mod macros;
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 937c674d..ed3351be 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -1,17 +1,209 @@
-use crate::prefix_record::{Meta, PublicRecord};
-use crate::prelude::multi::PrefixId;
-use crossbeam_epoch::{self as epoch, Atomic};
-use log::{error, log_enabled, trace};
+use crate::local_array::bit_span::BitSpan;
+// ----------- THE STORE ----------------------------------------------------
+//
+// The CustomAllocStore provides in-memory storage for the BitTreeMapNodes
+// and for prefixes and their meta-data. The storage for node is on the
+// `buckets` field, and the prefixes are stored in, well, the `prefixes`
+// field. They are both organised in the same way, as chained hash tables,
+// one per (prefix|node)-length. The hashing function (that is detailed
+// lower down in this file), basically takes the address part of the
+// node|prefix and uses `(node|prefix)-address part % bucket size`
+// as its index.
+//
+// Both the prefixes and the buckets field have one bucket per (prefix|node)
+// -length that start out with a fixed-size array. The size of the arrays is
+// set in the rotonda_macros/maps.rs file.
+//
+// For lower (prefix|node)-lengths the number of elements in the array is
+// equal to the number of prefixes in that length, so there's exactly one
+// element per (prefix|node). For greater lengths there will be collisions,
+// in that case the stored (prefix|node) will have a reference to another
+// bucket (also of a fixed size), that holds a (prefix|node) that collided
+// with the one that was already stored. A (node|prefix) lookup will have to
+// go over all (node|prefix) buckets until it matches the requested (node|
+// prefix) or it reaches the end of the chain.
+//
+// The chained (node|prefixes) are occupied at a first-come, first-serve
+// basis, and are not re-ordered on new insertions of (node|prefixes). This
+// may change in the future, since it prevents iterators from being ordered.
+//
+// One of the nice things of having one table per (node|prefix)-length is that
+// a search can start directly at the prefix-length table it wishes, and go
+// go up and down into other tables if it needs to (e.g., because more- or
+// less-specifics were asked for). In contrast if you do a lookup by
+// traversing the tree of nodes, we would always have to go through the root-
+// node first and then go up the tree to the requested node. The lower nodes
+// of the tree (close to the root) would be a formidable bottle-neck then.
+//
+// Currently, the meta-data is an atomically stored value, that is required to
+// implement the `Meta` and the `Clone` trait. New meta-data
+// instances are stored atomically without further ado, but updates to a
+// piece of meta-data are done by merging the previous meta-data with the new
+// meta-data, through use of the `MergeUpdate` trait.
+//
+// The `upsert_prefix` methods retrieve only the most recent insert
+// for a prefix (for now).
+//
+// Future work could have a user-configurable retention strategy that allows
+// the meta-data to be stored as a linked-list of references, where each
+// meta-data object has a reference to its predecessor.
+//
+// Prefix example
+//
+//         (level 0 arrays)         prefixes  bucket
+//                                    /len     size
+//         ┌──┐
+// len /0  │ 0│                        1        1     ■
+//         └──┘                                       │
+//         ┌──┬──┐                                    │
+// len /1  │00│01│                     2        2     │
+//         └──┴──┘                                 perfect
+//         ┌──┬──┬──┬──┐                             hash
+// len /2  │  │  │  │  │               4        4     │
+//         └──┴──┴──┴──┘                              │
+//         ┌──┬──┬──┬──┬──┬──┬──┬──┐                  │
+// len /3  │  │  │  │  │  │  │  │  │   8        8     ■
+//         └──┴──┴──┴──┴──┴──┴──┴──┘
+//         ┌──┬──┬──┬──┬──┬──┬──┬──┐                        ┌────────────┐
+// len /4  │  │  │  │  │  │  │  │  │   8        16 ◀────────│ collision  │
+//         └──┴──┴──┴┬─┴──┴──┴──┴──┘                        └────────────┘
+//                   └───┐
+//                       │              ┌─collision─────────┐
+//                   ┌───▼───┐          │                   │
+//                   │       │ ◀────────│ 0x0100 and 0x0101 │
+//                   │ 0x010 │          └───────────────────┘
+//                   │       │
+//                   ├───────┴──────────────┬──┬──┐
+//                   │ StoredPrefix 0x0101  │  │  │
+//                   └──────────────────────┴─┬┴─┬┘
+//                                            │  │
+//                       ┌────────────────────┘  └──┐
+//            ┌──────────▼──────────┬──┐          ┌─▼┬──┐
+//         ┌─▶│ metadata (current)  │  │          │ 0│ 1│ (level 1 array)
+//         │  └─────────────────────┴──┘          └──┴──┘
+//    merge└─┐                        │             │
+//    update │           ┌────────────┘             │
+//           │┌──────────▼──────────┬──┐        ┌───▼───┐
+//         ┌─▶│ metadata (previous) │  │        │       │
+//         │  └─────────────────────┴──┘        │  0x0  │
+//    merge└─┐                        │         │       │
+//    update │           ┌────────────┘         ├───────┴──────────────┬──┐
+//           │┌──────────▼──────────┬──┐        │ StoredPrefix 0x0110  │  │
+//            │ metadata (oldest)   │  │        └──────────────────────┴──┘
+//            └─────────────────────┴──┘                                 │
+//                                                         ┌─────────────┘
+//                                              ┌──────────▼──────────────┐
+//                                              │ metadata (current)      │
+//                                              └─────────────────────────┘
+
+// Note about the memory usage of the data-structures of the Buckets
+//
+// As said, the prefixes and nodes are stored in buckets. A bucket right now
+// is of type `[MaybeUnit<Atomic<StoredPrefix>>]`, this has the advantage
+// that the length can be variable, based on the stride size for that level.
+// It saves us to have to implement a generic something.
+// Another advantage is the fixed place in which an atomic StoredPrefix
+// lives: this makes compare-and-swapping it relatively straight forward.
+// Each accessing thread would try to read/write the exact same entry in the
+// array, so shouldn't be any 'rug pulling' on the whole array.
+//
+// A disadvantage is that this is a fixed size, sparse array the moment it
+// is created. Theoretically, an `Atomic<Vec<StoredPrefix>`
+// would not have this disadvantage. Manipulating the whole vec atomically
+// though is very tricky (we would have to atomically compare-and-swap the
+// whole vec each time the prefix meta-data is changed) and inefficient,
+// since we would have to either keep the vec sorted on `PrefixId` at all
+// times, or, we would have to inspect each value in the vec on *every* read
+// or write. the StoredPrefix (this is a challenge in itself, since the
+// StoredPrefix needs to be read atomically to retrieve the PrefixId).
+// Compare-and-swapping a whole vec most probably would need a hash over the
+// vec to determine whether it was changed. I gave up on this approach,
+//
+// Another approach to try to limit the memory use is to try to use other
+// indexes in the same array on collision (the array mentioned above), before
+// heading off and following the reference to the next bucket. This would
+// limit the amount of (sparse) arrays being created for a typical prefix
+// treebitmap, at the cost of longer average search times. Two
+// implementations of this approach are Cuckoo hashing[^1], and Skip Lists.
+// Skip lists[^2] are a probabilistic data-structure, famously used by Redis,
+// (and by TiKv). I haven't tries either of these. Crossbeam has a SkipList
+// implementation, that wasn't ready at the time I wrote this. Cuckoo
+// hashing has the advantage of being easier to understand/implement. Maybe
+// Cuckoo hashing can also be combined with Fibonacci hashing[^3]. Note that
+// Robin Hood hashing maybe faster than Cuckoo hashing for reads, but it
+// requires shifting around existing entries, which is rather costly to do
+// atomically (and complex).
+
+// [^1]: [https://en.wikipedia.org/wiki/Cuckoo_hashing]
+// [^3]: [https://docs.rs/crossbeam-skiplist/0.1.1/crossbeam_skiplist/]
+// [^3]: [https://probablydance.com/2018/06/16/fibonacci-hashing-
+//  the-optimization-that-the-world-forgot-or-a-better-alternative-
+//  to-integer-modulo/]
+
+// Notes on memory leaks in Rotonda-store
+//
+// Both valgrind and miri report memory leaks on the multi-threaded prefix
+// store. Valgrind only reports it when it a binary stops using the tree,
+// while still keeping it around. An interrupted use of the mt-prefix-store
+// does not report any memory leaks. Miri is persistent in reporting memory
+// leaks in the mt-prefix-store. They both report the memory leaks in the same
+// location: the init method of the node- and prefix-buckets.
+//
+// I have reasons to believe these reported memory leaks aren't real, or that
+// crossbeam-epoch leaks a bit of memory when creating a new `Atomic`
+// instance. Since neither prefix nor node buckets can or should be dropped
+// this is not a big issue anyway, it just means that an `Atomic` occupies
+// more memory than it could in an optimal situation. Since we're not storing
+// the actual meta-data in an `Atomic` (it is stored in an `flurry Map`), this
+// means memory usage won't grow on updating the meta-data on a prefix,
+// (unless the meta-data itself grows of course, but that's up to the user).
+//
+// To get a better understanding on the nature of the reported memory leaks I
+// have created a branch (`vec_set`) that replaces the dynamically sized array
+// with a (equally sparse) Vec, that is not filled with `Atomic:::null()`, but
+// with `Option<StoredPrefix` instead, in order to see if this would eliminate
+// the memory leaks reporting. It did not. Valgrind still reports the memory
+// leaks at the same location, although they're now reported as `indirectly
+// leaked`, instead of `directly`. Miri is unchanged. The attentive reader may
+// now suspect that the `Atomic` inside the `AtomicStoredPrefix` may be the
+// culprit, but: a. Both miri and valgrind report the leaks still in the same
+// place: the creation of the buckets, not the creation of the stored
+//    prefixes. b. Tests that look at the memory usage of the prefix stores
+// under heavy modification of existing prefixes do not exhibit memory leaking
+//    behavior. c. Tests that look at memory usage under addition of new
+//    prefixes exhibit linear incrementation of memory usage (which is
+// expected). d. Tests that look at memory usage under contention do not
+//    exhibit increased memory usage either.
+//
+// My strong suspicion is that both Miri and Valgrind report the sparse slots
+// in the buckets as leaks, no matter whether they're `Atomic::null()`, or
+// `None` values, probably as a result of the way `crossbeam-epoch` indexes
+// into these, with pointer arithmetic (unsafe as hell).
+//
+// I would be super grateful if somebody would prove me wrong and can point to
+// an actual memory leak in the mt-prefix-store (and even more if they can
+// produce a fix for it).
+
+use crate::local_array::in_memory::atomic_types::StoredNode;
+use crate::prefix_record::Meta;
+use crate::prelude::multi::{NodeSet, PrefixId};
+use crossbeam_epoch::Atomic;
+use crossbeam_utils::Backoff;
+use log::{debug, log_enabled, trace};
 use roaring::RoaringBitmap;
 
-use std::hash::Hash;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
 use std::{fmt::Debug, marker::PhantomData};
 
-use super::atomic_types::{NodeBuckets, PrefixBuckets};
+use super::atomic_types::{
+    NodeBuckets, PrefixBuckets, PrefixSet, StoredPrefix,
+};
 use crate::af::AddressFamily;
-use crate::insert_match;
-use crate::rib::{Rib, StoreConfig, UpsertReport};
+use crate::rib::{Counters, Rib};
+use crate::{
+    impl_search_level, impl_search_level_for_mui, retrieve_node_mut_closure,
+    store_node_closure,
+};
 
 use super::super::errors::PrefixStoreError;
 pub(crate) use super::atomic_stride::*;
@@ -163,33 +355,33 @@ impl<AF: AddressFamily> std::convert::From<StrideNodeId<AF>>
 
 //------------------------- Node Collections --------------------------------
 
-#[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy, Clone)]
-pub enum StrideType {
-    Stride3,
-    Stride4,
-    Stride5,
-}
-
-impl From<u8> for StrideType {
-    fn from(level: u8) -> Self {
-        match level {
-            3 => StrideType::Stride3,
-            4 => StrideType::Stride4,
-            5 => StrideType::Stride5,
-            _ => panic!("Invalid stride level {}", level),
-        }
-    }
-}
-
-impl std::fmt::Display for StrideType {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            StrideType::Stride3 => write!(f, "S3"),
-            StrideType::Stride4 => write!(f, "S4"),
-            StrideType::Stride5 => write!(f, "S5"),
-        }
-    }
-}
+// #[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy, Clone)]
+// pub enum StrideType {
+//     Stride3,
+//     Stride4,
+//     Stride5,
+// }
+
+// impl From<u8> for StrideType {
+//     fn from(level: u8) -> Self {
+//         match level {
+//             3 => StrideType::Stride3,
+//             4 => StrideType::Stride4,
+//             5 => StrideType::Stride5,
+//             _ => panic!("Invalid stride level {}", level),
+//         }
+//     }
+// }
+
+// impl std::fmt::Display for StrideType {
+//     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+//         match self {
+//             StrideType::Stride3 => write!(f, "S3"),
+//             StrideType::Stride4 => write!(f, "S4"),
+//             StrideType::Stride5 => write!(f, "S5"),
+//         }
+//     }
+// }
 
 //--------------------- TreeBitMap ------------------------------------------
 
@@ -202,7 +394,9 @@ pub struct TreeBitMap<
 > {
     pub(crate) node_buckets: NB,
     pub(crate) prefix_buckets: PB,
+    // Global Roaring BitMap INdex that stores MUIs.
     pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
+    counters: Counters,
     _af: PhantomData<AF>,
     _m: PhantomData<M>,
 }
@@ -214,127 +408,17 @@ impl<
         PB: PrefixBuckets<AF, M>,
     > TreeBitMap<AF, M, NB, PB>
 {
-    pub(crate) fn init() -> Self {
-        Self {
+    pub(crate) fn new() -> Result<Self, Box<dyn std::error::Error>> {
+        let tree_bitmap = Self {
             node_buckets: NodeBuckets::init(),
             prefix_buckets: PB::init(),
             withdrawn_muis_bmin: RoaringBitmap::new().into(),
+            counters: Counters::default(),
             _af: PhantomData,
             _m: PhantomData,
-        }
-    }
-
-    // Change the status of the mui globally to Withdrawn. Iterators and match
-    // functions will by default not return any records for this mui.
-    // pub fn mark_mui_as_withdrawn(
-    //     &self,
-    //     mui: u32,
-    //     guard: &Guard,
-    // ) -> Result<(), PrefixStoreError> {
-    //     let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
-
-    //     let mut new = unsafe { current.as_ref() }.unwrap().clone();
-    //     new.insert(mui);
-
-    //     #[allow(clippy::assigning_clones)]
-    //     loop {
-    //         match self.withdrawn_muis_bmin.compare_exchange(
-    //             current,
-    //             Owned::new(new),
-    //             Ordering::AcqRel,
-    //             Ordering::Acquire,
-    //             guard,
-    //         ) {
-    //             Ok(_) => return Ok(()),
-    //             Err(updated) => {
-    //                 new =
-    //                     unsafe { updated.current.as_ref() }.unwrap().clone();
-    //             }
-    //         }
-    //     }
-    // }
-
-    // Change the status of the mui globally to Active. Iterators and match
-    // functions will default to the status on the record itself.
-    // pub fn mark_mui_as_active(
-    //     &self,
-    //     mui: u32,
-    //     guard: &Guard,
-    // ) -> Result<(), PrefixStoreError> {
-    //     let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
-
-    //     let mut new = unsafe { current.as_ref() }.unwrap().clone();
-    //     new.remove(mui);
-
-    //     #[allow(clippy::assigning_clones)]
-    //     loop {
-    //         match self.withdrawn_muis_bmin.compare_exchange(
-    //             current,
-    //             Owned::new(new),
-    //             Ordering::AcqRel,
-    //             Ordering::Acquire,
-    //             guard,
-    //         ) {
-    //             Ok(_) => return Ok(()),
-    //             Err(updated) => {
-    //                 new =
-    //                     unsafe { updated.current.as_ref() }.unwrap().clone();
-    //             }
-    //         }
-    //     }
-    // }
-
-    // Whether this mui is globally withdrawn. Note that this overrules
-    // (by default) any (prefix, mui) combination in iterators and match
-    // functions.
-    // pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
-    //     unsafe {
-    //         self.withdrawn_muis_bmin
-    //             .load(Ordering::Acquire, guard)
-    //             .as_ref()
-    //     }
-    //     .unwrap()
-    //     .contains(mui)
-    // }
-
-    // Whether this mui is globally active. Note that the local statuses of
-    // records (prefix, mui) may be set to withdrawn in iterators and match
-    // functions.
-    // pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
-    //     !unsafe {
-    //         self.withdrawn_muis_bmin
-    //             .load(Ordering::Acquire, guard)
-    //             .as_ref()
-    //     }
-    //     .unwrap()
-    //     .contains(mui)
-    // }
-}
+        };
 
-impl<
-        AF: AddressFamily,
-        M: Meta,
-        NB: NodeBuckets<AF>,
-        PB: PrefixBuckets<AF, M>,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
-{
-    pub fn new(
-        config: StoreConfig,
-    ) -> Result<
-        Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
-        Box<dyn std::error::Error>,
-    > {
-        let root_node = match Rib::<
-            AF,
-            M,
-            NB,
-            PB,
-            PREFIX_SIZE,
-            KEY_SIZE,
-        >::get_first_stride_size()
-        {
+        let root_node = match Self::get_first_stride_size() {
             3 => SizedStrideNode::Stride3(TreeBitMapNode {
                 ptrbitarr: AtomicStride2(AtomicU8::new(0)),
                 pfxbitarr: AtomicStride3(AtomicU16::new(0)),
@@ -358,263 +442,574 @@ impl<
             }
         };
 
-        Rib::<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>::init(root_node, config)
+        let _retry_count = tree_bitmap.store_node(
+            StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0),
+            0_u32,
+            root_node,
+        )?;
+
+        Ok(tree_bitmap)
     }
 
-    // Partition for stride 4
-    //
-    // ptr bits never happen in the first half of the bitmap for the stride-size. Consequently the ptrbitarr can be an integer type
-    // half the size of the pfxbitarr.
+    // Create a new node in the store with payload `next_node`.
     //
-    // ptr bit arr (u16)                                                        0    1    2    3    4    5    6    7    8    9   10   11   12   13   14   15    x
-    // pfx bit arr (u32)   0 1 2  3  4  5  6   7   8   9  10  11  12  13  14   15   16   17   18   19   20   21   22   23   24   25   26   27   28   29   30   31
-    // nibble              * 0 1 00 01 10 11 000 001 010 011 100 101 110 111 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 1100 1101 1110 1111    x
-    // nibble len offset   0 1    2            3                                4
+    // Next node will be ignored if a node with the same `id` already exists,
+    // but the multi_uniq_id will be added to the rbm_index of the NodeSet.
     //
-    // stride 3: 1 + 2 + 4 + 8                              =  15 bits. 2^4 - 1 (1 << 4) - 1. ptrbitarr starts at pos  7 (1 << 3) - 1
-    // stride 4: 1 + 2 + 4 + 8 + 16                         =  31 bits. 2^5 - 1 (1 << 5) - 1. ptrbitarr starts at pos 15 (1 << 4) - 1
-    // stride 5: 1 + 2 + 4 + 8 + 16 + 32 + 64               =  63 bits. 2^6 - 1
-    // stride 6: 1 + 2 + 4 + 8 + 16 + 32 + 64               = 127 bits. 2^7 - 1
-    // stride 7: 1 + 2 + 4 + 8 + 16 + 32 + 64 = 128         = 256 bits. 2^8 - 1126
-    // stride 8: 1 + 2 + 4 + 8 + 16 + 32 + 64 + 128 + 256   = 511 bits. 2^9 - 1
-    //
-    // Ex.:
-    // pfx            65.0.0.252/30                                             0100_0001_0000_0000_0000_0000_1111_1100
-    //
-    // nibble 1       (pfx << 0) >> 28                                          0000_0000_0000_0000_0000_0000_0000_0100
-    // bit_pos        (1 << nibble length) - 1 + nibble                         0000_0000_0000_0000_0000_1000_0000_0000
-    //
-    // nibble 2       (pfx << 4) >> 24                                          0000_0000_0000_0000_0000_0000_0000_0001
-    // bit_pos        (1 << nibble length) - 1 + nibble                         0000_0000_0000_0000_1000_0000_0000_0000
-    // ...
-    // nibble 8       (pfx << 28) >> 0                                          0000_0000_0000_0000_0000_0000_0000_1100
-    // bit_pos        (1 << nibble length) - 1 + nibble = (1 << 2) - 1 + 2 = 5  0000_0010_0000_0000_0000_0000_0000_0000
-    // 5 - 5 - 5 - 4 - 4 - [4] - 5
-    // startpos (2 ^ nibble length) - 1 + nibble as usize
-
-    pub fn insert(
+    // Returns: a tuple with the node_id of the created node and the number of
+    // retry_count
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn store_node(
         &self,
-        pfx: PrefixId<AF>,
-        record: PublicRecord<M>,
-        update_path_selections: Option<M::TBI>,
-    ) -> Result<UpsertReport, PrefixStoreError> {
-        let guard = &epoch::pin();
-
-        if pfx.get_len() == 0 {
-            let res = self.update_default_route_prefix_meta(record, guard)?;
-            return Ok(res);
+        id: StrideNodeId<AF>,
+        multi_uniq_id: u32,
+        next_node: SizedStrideNode<AF>,
+    ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
+        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+            f: &'s dyn Fn(
+                &SearchLevel<AF, S>,
+                &NodeSet<AF, S>,
+                TreeBitMapNode<AF, S>,
+                u32, // multi_uniq_id
+                u8,  // the store level
+                u32, // retry_count
+            ) -> Result<
+                (StrideNodeId<AF>, u32),
+                PrefixStoreError,
+            >,
         }
 
-        let mut stride_end: u8 = 0;
-        let mut cur_i = self.get_root_node_id();
-        let mut level: u8 = 0;
-        let mut acc_retry_count = 0;
+        let search_level_3 =
+            store_node_closure![Stride3; id; guard; back_off;];
+        let search_level_4 =
+            store_node_closure![Stride4; id; guard; back_off;];
+        let search_level_5 =
+            store_node_closure![Stride5; id; guard; back_off;];
+
+        if log_enabled!(log::Level::Trace) {
+            debug!(
+                "{} store: Store node {}: {:?} mui {}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                id,
+                next_node,
+                multi_uniq_id
+            );
+        }
+        self.counters.inc_nodes_count();
+
+        match next_node {
+            SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
+                &search_level_3,
+                self.node_buckets.get_store3(id),
+                new_node,
+                multi_uniq_id,
+                0,
+                0,
+            ),
+            SizedStrideNode::Stride4(new_node) => (search_level_4.f)(
+                &search_level_4,
+                self.node_buckets.get_store4(id),
+                new_node,
+                multi_uniq_id,
+                0,
+                0,
+            ),
+            SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
+                &search_level_5,
+                self.node_buckets.get_store5(id),
+                new_node,
+                multi_uniq_id,
+                0,
+                0,
+            ),
+        }
+    }
 
-        loop {
-            let stride = self.get_stride_sizes()[level as usize];
-            stride_end += stride;
-            let nibble_len = if pfx.get_len() < stride_end {
-                stride + pfx.get_len() - stride_end
-            } else {
-                stride
-            };
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn retrieve_node_mut(
+        &self,
+        id: StrideNodeId<AF>,
+        multi_uniq_id: u32,
+    ) -> Option<SizedStrideRef<'_, AF>> {
+        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+            f: &'s dyn for<'a> Fn(
+                &SearchLevel<AF, S>,
+                &'a NodeSet<AF, S>,
+                u8,
+            )
+                -> Option<SizedStrideRef<'a, AF>>,
+        }
 
-            let nibble = AF::get_nibble(
-                pfx.get_net(),
-                stride_end - stride,
-                nibble_len,
+        let search_level_3 =
+            retrieve_node_mut_closure![Stride3; id; multi_uniq_id;];
+        let search_level_4 =
+            retrieve_node_mut_closure![Stride4; id; multi_uniq_id;];
+        let search_level_5 =
+            retrieve_node_mut_closure![Stride5; id; multi_uniq_id;];
+
+        if log_enabled!(log::Level::Trace) {
+            trace!(
+                "{} store: Retrieve node mut {} from l{}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                id,
+                id.get_id().1
             );
-            let is_last_stride = pfx.get_len() <= stride_end;
-            let stride_start = stride_end - stride;
-            // let back_off = crossbeam_utils::Backoff::new();
-
-            // insert_match! returns the node_id of the next node to be
-            // traversed. It was created if it did not exist.
-            let node_result = insert_match![
-                // applicable to the whole outer match in the macro
-                self;
-                guard;
-                nibble_len;
-                nibble;
-                is_last_stride;
-                pfx;
-                record;
-                update_path_selections; // perform an update for the paths in this record
-                stride_start; // the length at the start of the stride a.k.a. start_bit
-                stride;
-                cur_i;
-                level;
-                acc_retry_count;
-                // Strides to create match arm for; stats level
-                Stride3; 0,
-                Stride4; 1,
-                Stride5; 2
-            ];
-
-            match node_result {
-                Ok((next_id, retry_count)) => {
-                    cur_i = next_id;
-                    level += 1;
-                    acc_retry_count += retry_count;
-                }
-                Err(err) => {
-                    if log_enabled!(log::Level::Error) {
-                        error!("{} failing to store (intermediate) node {}. Giving up this node. This shouldn't happen!",
-                            std::thread::current().name().unwrap_or("unnamed-thread"),
-                            cur_i,
-                        );
-                        error!(
-                            "{} {}",
-                            std::thread::current()
-                                .name()
-                                .unwrap_or("unnamed-thread"),
-                            err
-                        );
-                    }
-                }
-            }
+        }
+
+        match self.node_buckets.get_stride_for_id(id) {
+            3 => (search_level_3.f)(
+                &search_level_3,
+                self.node_buckets.get_store3(id),
+                0,
+            ),
+
+            4 => (search_level_4.f)(
+                &search_level_4,
+                self.node_buckets.get_store4(id),
+                0,
+            ),
+            _ => (search_level_5.f)(
+                &search_level_5,
+                self.node_buckets.get_store5(id),
+                0,
+            ),
         }
     }
 
-    // Yes, we're hating this. But, the root node has no room for a serial of
-    // the prefix 0/0 (the default route), which doesn't even matter, unless,
-    // UNLESS, somebody wants to store a default route. So we have to store a
-    // serial for this prefix. The normal place for a serial of any prefix is
-    // on the pfxvec of its paren. But, hey, guess what, the
-    // default-route-prefix lives *on* the root node, and, you know, the root
-    // node doesn't have a parent. We can:
-    // - Create a type RootTreeBitmapNode with a ptrbitarr with a size one
-    //   bigger than a "normal" TreeBitMapNod for the first stride size. no we
-    //   have to iterate over the root-node type in all matches on
-    //   stride_size, just because we have exactly one instance of the
-    //   RootTreeBitmapNode. So no.
-    // - Make the `get_pfx_index` method on the implementations of the
-    //   `Stride` trait check for a length of zero and branch if it is and
-    //   return the serial of the root node. Now each and every call to this
-    //   method will have to check a condition for exactly one instance of
-    //   RootTreeBitmapNode. So again, no.
-    // - The root node only gets used at the beginning of a search query or an
-    //   insert. So if we provide two specialised methods that will now how to
-    //   search for the default-route prefix and now how to set serial for
-    //  that prefix and make sure we start searching/inserting with one of
-    //   those specialized methods we're good to go.
-    fn update_default_route_prefix_meta(
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn retrieve_node(
         &self,
-        record: PublicRecord<M>,
-        guard: &epoch::Guard,
-    ) -> Result<UpsertReport, PrefixStoreError> {
-        trace!("Updating the default route...");
+        id: StrideNodeId<AF>,
+    ) -> Option<SizedStrideRef<'_, AF>> {
+        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+            f: &'s dyn for<'a> Fn(
+                &SearchLevel<AF, S>,
+                &'a NodeSet<AF, S>,
+                u8,
+            )
+                -> Option<SizedStrideRef<'a, AF>>,
+        }
 
-        if let Some(root_node) = self
-            .retrieve_node_mut(self.get_root_node_id(), record.multi_uniq_id)
-        {
-            match root_node {
-                SizedStrideRef::Stride3(_) => {
-                    self.in_memory_tree
-                        .node_buckets
-                        .get_store3(self.get_root_node_id())
-                        .update_rbm_index(record.multi_uniq_id)?;
-                }
-                SizedStrideRef::Stride4(_) => {
-                    self.in_memory_tree
-                        .node_buckets
-                        .get_store4(self.get_root_node_id())
-                        .update_rbm_index(record.multi_uniq_id)?;
+        let search_level_3 = impl_search_level![Stride3; id;];
+        let search_level_4 = impl_search_level![Stride4; id;];
+        let search_level_5 = impl_search_level![Stride5; id;];
+
+        if log_enabled!(log::Level::Trace) {
+            trace!(
+                "{} store: Retrieve node {} from l{}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                id,
+                id.get_id().1
+            );
+        }
+
+        match self.get_stride_for_id(id) {
+            3 => (search_level_3.f)(
+                &search_level_3,
+                self.node_buckets.get_store3(id),
+                0,
+            ),
+            4 => (search_level_4.f)(
+                &search_level_4,
+                self.node_buckets.get_store4(id),
+                0,
+            ),
+            _ => (search_level_5.f)(
+                &search_level_5,
+                self.node_buckets.get_store5(id),
+                0,
+            ),
+        }
+    }
+
+    // retrieve a node, but only its bitmap index contains the specified mui.
+    // Used for iterators per mui.
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn retrieve_node_for_mui(
+        &self,
+        id: StrideNodeId<AF>,
+        // The mui that is tested to be present in the nodes bitmap index
+        mui: u32,
+    ) -> Option<SizedStrideRef<'_, AF>> {
+        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+            f: &'s dyn for<'a> Fn(
+                &SearchLevel<AF, S>,
+                &'a NodeSet<AF, S>,
+                u8,
+            )
+                -> Option<SizedStrideRef<'a, AF>>,
+        }
+
+        let search_level_3 = impl_search_level_for_mui![Stride3; id; mui;];
+        let search_level_4 = impl_search_level_for_mui![Stride4; id; mui;];
+        let search_level_5 = impl_search_level_for_mui![Stride5; id; mui;];
+
+        if log_enabled!(log::Level::Trace) {
+            trace!(
+                "{} store: Retrieve node {} from l{} for mui {}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                id,
+                id.get_id().1,
+                mui
+            );
+        }
+
+        match self.get_stride_for_id(id) {
+            3 => (search_level_3.f)(
+                &search_level_3,
+                self.node_buckets.get_store3(id),
+                0,
+            ),
+            4 => (search_level_4.f)(
+                &search_level_4,
+                self.node_buckets.get_store4(id),
+                0,
+            ),
+            _ => (search_level_5.f)(
+                &search_level_5,
+                self.node_buckets.get_store5(id),
+                0,
+            ),
+        }
+    }
+
+    pub(crate) fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8 {
+        self.node_buckets.get_stride_for_id(id)
+    }
+
+    pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
+        StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0)
+    }
+
+    pub fn get_nodes_count(&self) -> usize {
+        self.counters.get_nodes_count()
+    }
+
+    // This function is used by the upsert_prefix function above.
+    //
+    // We're using a Chained Hash Table and this function returns one of:
+    // - a StoredPrefix that already exists for this search_prefix_id
+    // - the Last StoredPrefix in the chain.
+    // - an error, if no StoredPrefix whatsoever can be found in the store.
+    //
+    // The error condition really shouldn't happen, because that basically
+    // means the root node for that particular prefix length doesn't exist.
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn non_recursive_retrieve_prefix_mut(
+        &self,
+        search_prefix_id: PrefixId<AF>,
+    ) -> (&StoredPrefix<AF, M>, bool) {
+        trace!("non_recursive_retrieve_prefix_mut_with_guard");
+        let mut prefix_set = self
+            .prefix_buckets
+            .get_root_prefix_set(search_prefix_id.get_len());
+        let mut level: u8 = 0;
+
+        trace!("root prefix_set {:?}", prefix_set);
+        loop {
+            // HASHING FUNCTION
+            let index = TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(
+                search_prefix_id,
+                level,
+            );
+
+            // probe the slot with the index that's the result of the hashing.
+            // let locked_prefix = prefix_set.0.get(index);
+            let stored_prefix = match prefix_set.0.get(index) {
+                Some(p) => {
+                    trace!("prefix set found.");
+                    (p, true)
                 }
-                SizedStrideRef::Stride5(_) => {
-                    self.in_memory_tree
-                        .node_buckets
-                        .get_store5(self.get_root_node_id())
-                        .update_rbm_index(record.multi_uniq_id)?;
+                None => {
+                    // We're at the end of the chain and haven't found our
+                    // search_prefix_id anywhere. Return the end-of-the-chain
+                    // StoredPrefix, so the caller can attach a new one.
+                    trace!(
+                        "no record. returning last found record in level
+                        {}, with index {}.",
+                        level,
+                        index
+                    );
+                    let index = TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(
+                        search_prefix_id,
+                        level,
+                    );
+                    trace!("calculate next index {}", index);
+                    let var_name = (
+                        prefix_set
+                            .0
+                            .get_or_init(index, || {
+                                StoredPrefix::new::<PB>(
+                                    PrefixId::new(
+                                        search_prefix_id.get_net(),
+                                        search_prefix_id.get_len(),
+                                    ),
+                                    level,
+                                )
+                            })
+                            .0,
+                        false,
+                    );
+                    var_name
                 }
             };
-        };
 
-        self.upsert_prefix(
-            PrefixId::new(AF::zero(), 0),
-            record,
-            // Do not update the path selection for the default route.
-            None,
-            guard,
-        )
+            if search_prefix_id == stored_prefix.0.prefix {
+                // GOTCHA!
+                // Our search-prefix is stored here, so we're returning
+                // it, so its PrefixRecord can be updated by the caller.
+                trace!("found requested prefix {:?}", search_prefix_id);
+                return stored_prefix;
+            } else {
+                // A Collision. Follow the chain.
+                level += 1;
+                prefix_set = &stored_prefix.0.next_bucket;
+                continue;
+            }
+        }
     }
 
-    // This function assembles all entries in the `pfx_vec` of all child nodes
-    // of the `start_node` into one vec, starting from itself and then
-    // recursively assembling adding all `pfx_vec`s of its children.
-    fn get_all_more_specifics_for_node(
+    // This function is used by the match_prefix, and [more|less]_specifics
+    // public methods on the TreeBitMap (indirectly).
+    #[allow(clippy::type_complexity)]
+    pub fn non_recursive_retrieve_prefix(
         &self,
-        start_node_id: StrideNodeId<AF>,
-        found_pfx_vec: &mut Vec<PrefixId<AF>>,
+        id: PrefixId<AF>,
+    ) -> (
+        Option<&StoredPrefix<AF, M>>,
+        Option<(
+            PrefixId<AF>,
+            u8,
+            &PrefixSet<AF, M>,
+            [Option<(&PrefixSet<AF, M>, usize)>; 32],
+            usize,
+        )>,
     ) {
-        trace!("{:?}", self.retrieve_node(start_node_id));
-        match self.retrieve_node(start_node_id) {
-            Some(SizedStrideRef::Stride3(n)) => {
-                found_pfx_vec.extend(
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
-                );
+        let mut prefix_set =
+            self.prefix_buckets.get_root_prefix_set(id.get_len());
+        let mut parents = [None; 32];
+        let mut level: u8 = 0;
+        let backoff = Backoff::new();
 
-                for child_node in n.ptr_iter(start_node_id) {
-                    self.get_all_more_specifics_for_node(
-                        child_node,
-                        found_pfx_vec,
+        loop {
+            // The index of the prefix in this array (at this len and
+            // level) is calculated by performing the hash function
+            // over the prefix.
+
+            // HASHING FUNCTION
+            let index =
+                TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(id, level);
+
+            if let Some(stored_prefix) = prefix_set.0.get(index) {
+                if id == stored_prefix.get_prefix_id() {
+                    trace!("found requested prefix {:?}", id);
+                    parents[level as usize] = Some((prefix_set, index));
+                    return (
+                        Some(stored_prefix),
+                        Some((id, level, prefix_set, parents, index)),
                     );
-                }
-            }
-            Some(SizedStrideRef::Stride4(n)) => {
-                found_pfx_vec.extend(
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
-                );
+                };
 
-                for child_node in n.ptr_iter(start_node_id) {
-                    self.get_all_more_specifics_for_node(
-                        child_node,
-                        found_pfx_vec,
-                    );
-                }
+                // Advance to the next level.
+                prefix_set = &stored_prefix.next_bucket;
+                level += 1;
+                backoff.spin();
+                continue;
             }
-            Some(SizedStrideRef::Stride5(n)) => {
-                found_pfx_vec.extend(
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
-                );
 
-                for child_node in n.ptr_iter(start_node_id) {
-                    self.get_all_more_specifics_for_node(
-                        child_node,
-                        found_pfx_vec,
-                    );
-                }
-            }
-            _ => {
-                panic!("can't find node {}", start_node_id);
-            }
+            trace!("no prefix found for {:?}", id);
+            parents[level as usize] = Some((prefix_set, index));
+            return (None, Some((id, level, prefix_set, parents, index)));
+        }
+    }
+
+    #[allow(dead_code)]
+    fn remove_prefix(&mut self, index: PrefixId<AF>) -> Option<M> {
+        match index.is_empty() {
+            false => self.prefix_buckets.remove(index),
+            true => None,
         }
     }
 
-    // This function assembles the prefixes of a child node starting on a
-    // specified bit position in a ptr_vec of `current_node` into a vec,
-    // then adds all prefixes of these children recursively into a vec and
-    // returns that.
-    pub(crate) fn get_all_more_specifics_from_nibble<S: Stride>(
+    pub fn get_prefixes_count(&self) -> usize {
+        self.counters.get_prefixes_count().iter().sum()
+    }
+
+    pub fn get_prefixes_count_for_len(&self, len: u8) -> usize {
+        self.counters.get_prefixes_count()[len as usize]
+    }
+
+    // Stride related methods
+
+    pub fn get_stride_sizes(&self) -> &[u8] {
+        self.node_buckets.get_stride_sizes()
+    }
+
+    pub(crate) fn get_first_stride_size() -> u8 {
+        NB::get_first_stride_size()
+    }
+
+    // Calculates the id of the node that COULD host a prefix in its
+    // ptrbitarr.
+    pub(crate) fn get_node_id_for_prefix(
         &self,
-        current_node: &TreeBitMapNode<AF, S>,
-        nibble: u32,
-        nibble_len: u8,
-        base_prefix: StrideNodeId<AF>,
-    ) -> Option<Vec<PrefixId<AF>>> {
-        let (cnvec, mut msvec) = current_node.add_more_specifics_at(
-            nibble,
-            nibble_len,
-            base_prefix,
+        prefix: &PrefixId<AF>,
+    ) -> (StrideNodeId<AF>, BitSpan) {
+        let mut acc = 0;
+        for i in self.get_stride_sizes() {
+            acc += *i;
+            if acc >= prefix.get_len() {
+                let node_len = acc - i;
+                return (
+                    StrideNodeId::new_with_cleaned_id(
+                        prefix.get_net(),
+                        node_len,
+                    ),
+                    // NOT THE HASHING FUNCTION!
+                    // Do the right shift in a checked manner, for the sake
+                    // of 0/0. A search for 0/0 will perform a 0 << MAX_LEN,
+                    // which will panic in debug mode (undefined behaviour
+                    // in prod).
+                    BitSpan::new(
+                        ((prefix.get_net() << node_len).checked_shr_or_zero(
+                            (AF::BITS - (prefix.get_len() - node_len)).into(),
+                        ))
+                        .dangerously_truncate_to_u32(),
+                        prefix.get_len() - node_len,
+                    ),
+                );
+            }
+        }
+        panic!("prefix length for {:?} is too long", prefix);
+    }
+    // ------- THE HASHING FUNCTION -----------------------------------------
+
+    // Ok, so hashing is really hard, but we're keeping it simple, and
+    // because we're keeping it simple we're having lots of collisions, but
+    // we don't care!
+    //
+    // We're using a part of bitarray representation of the address part of
+    // a prefix the as the hash. Sounds complicated, but isn't.
+    // Suppose we have an IPv4 prefix, say 130.24.55.0/24.
+    // The address part is 130.24.55.0 or as a bitarray that would be:
+    //
+    // pos  0    4    8    12   16   20   24   28
+    // bit  1000 0010 0001 1000 0011 0111 0000 0000
+    //
+    // First, we're discarding the bits after the length of the prefix, so
+    // we'll have:
+    //
+    // pos  0    4    8    12   16   20
+    // bit  1000 0010 0001 1000 0011 0111
+    //
+    // Now we're dividing this bitarray into one or more levels. A level can
+    // be an arbitrary number of bits between 1 and the length of the prefix,
+    // but the number of bits summed over all levels should be exactly the
+    // prefix length. So in our case they should add up to 24. A possible
+    // division could be: 4, 4, 4, 4, 4, 4. Another one would be: 12, 12. The
+    // actual division being used is described in the function
+    // `<NB>::get_bits_for_len` in the `rotonda-macros` crate. Each level has
+    // its own hash, so for our example prefix this would be:
+    //
+    // pos   0    4    8    12   16   20
+    // level 0              1
+    // hash  1000 0010 0001 1000 0011 0111
+    //
+    // level 1 hash: 1000 0010 0001
+    // level 2 hash: 1000 0011 0011
+    //
+    // The hash is now converted to a usize integer, by shifting it all the
+    // way to the right in a u32 and then converting to a usize. Why a usize
+    // you ask? Because the hash is used by the CustomAllocStorage as the
+    // index to the array for that specific prefix length and level.
+    // So for our example this means that the hash on level 1 is now 0x821
+    // (decimal 2081) and the hash on level 2 is 0x833 (decimal 2099).
+    // Now, if we only consider the hash on level 1 and that we're going to
+    // use that as the index to the array that stores all prefixes, you'll
+    // notice very quickly that all prefixes starting with 130.[16..31] will
+    // cause a collision: they'll all point to the same array element. These
+    // collisions are resolved by creating a linked list from each array
+    // element, where each element in the list has an array of its own that
+    // uses the hash function with the level incremented.
+
+    pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
+        // And, this is all of our hashing function.
+        let last_level = if level > 0 {
+            <NB>::len_to_store_bits(id.get_id().1, level - 1)
+        } else {
+            0
+        };
+        let this_level = <NB>::len_to_store_bits(id.get_id().1, level);
+        trace!("bits division {}", this_level);
+        trace!(
+            "calculated index ({} << {}) >> {}",
+            id.get_id().0,
+            last_level,
+            ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
         );
+        // HASHING FUNCTION
+        ((id.get_id().0 << last_level)
+            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
+            .dangerously_truncate_to_u32() as usize
+    }
 
-        for child_node in cnvec.iter() {
-            self.get_all_more_specifics_for_node(*child_node, &mut msvec);
-        }
-        Some(msvec)
+    pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
+        // And, this is all of our hashing function.
+        let last_level = if level > 0 {
+            <PB>::get_bits_for_len(id.get_len(), level - 1)
+        } else {
+            0
+        };
+        let this_level = <PB>::get_bits_for_len(id.get_len(), level);
+        trace!(
+            "bits division {}; no of bits {}",
+            this_level,
+            this_level - last_level
+        );
+        trace!(
+            "calculated index ({} << {}) >> {}",
+            id.get_net(),
+            last_level,
+            ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
+        );
+        // HASHING FUNCTION
+        ((id.get_net() << last_level)
+            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
+            .dangerously_truncate_to_u32() as usize
     }
 }
 
+// Partition for stride 4
+//
+// ptr bits never happen in the first half of the bitmap for the stride-size. Consequently the ptrbitarr can be an integer type
+// half the size of the pfxbitarr.
+//
+// ptr bit arr (u16)                                                        0    1    2    3    4    5    6    7    8    9   10   11   12   13   14   15    x
+// pfx bit arr (u32)   0 1 2  3  4  5  6   7   8   9  10  11  12  13  14   15   16   17   18   19   20   21   22   23   24   25   26   27   28   29   30   31
+// nibble              * 0 1 00 01 10 11 000 001 010 011 100 101 110 111 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 1100 1101 1110 1111    x
+// nibble len offset   0 1    2            3                                4
+//
+// stride 3: 1 + 2 + 4 + 8                              =  15 bits. 2^4 - 1 (1 << 4) - 1. ptrbitarr starts at pos  7 (1 << 3) - 1
+// stride 4: 1 + 2 + 4 + 8 + 16                         =  31 bits. 2^5 - 1 (1 << 5) - 1. ptrbitarr starts at pos 15 (1 << 4) - 1
+// stride 5: 1 + 2 + 4 + 8 + 16 + 32 + 64               =  63 bits. 2^6 - 1
+// stride 6: 1 + 2 + 4 + 8 + 16 + 32 + 64               = 127 bits. 2^7 - 1
+// stride 7: 1 + 2 + 4 + 8 + 16 + 32 + 64 = 128         = 256 bits. 2^8 - 1126
+// stride 8: 1 + 2 + 4 + 8 + 16 + 32 + 64 + 128 + 256   = 511 bits. 2^9 - 1
+//
+// Ex.:
+// pfx            65.0.0.252/30                                             0100_0001_0000_0000_0000_0000_1111_1100
+//
+// nibble 1       (pfx << 0) >> 28                                          0000_0000_0000_0000_0000_0000_0000_0100
+// bit_pos        (1 << nibble length) - 1 + nibble                         0000_0000_0000_0000_0000_1000_0000_0000
+//
+// nibble 2       (pfx << 4) >> 24                                          0000_0000_0000_0000_0000_0000_0000_0001
+// bit_pos        (1 << nibble length) - 1 + nibble                         0000_0000_0000_0000_1000_0000_0000_0000
+// ...
+// nibble 8       (pfx << 28) >> 0                                          0000_0000_0000_0000_0000_0000_0000_1100
+// bit_pos        (1 << nibble length) - 1 + nibble = (1 << 2) - 1 + 2 = 5  0000_0010_0000_0000_0000_0000_0000_0000
+// 5 - 5 - 5 - 4 - 4 - [4] - 5
+// startpos (2 ^ nibble length) - 1 + nibble as usize
+
 // This implements the funky stats for a tree
 #[cfg(feature = "cli")]
 impl<
@@ -622,9 +1017,7 @@ impl<
         M: Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > std::fmt::Display for Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+    > std::fmt::Display for TreeBitMap<AF, M, NB, PB>
 {
     fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(_f, "{} prefixes created", self.get_prefixes_count())?;
diff --git a/src/local_array/iterators.rs b/src/local_array/iterators.rs
index 254224a0..43f54a99 100644
--- a/src/local_array/iterators.rs
+++ b/src/local_array/iterators.rs
@@ -340,6 +340,7 @@ impl<
                 if let Some(mui) = self.mui {
                     if let Some(p) = self
                         .store
+                        .in_memory_tree
                         .non_recursive_retrieve_prefix(
                             next_pfx.unwrap_or_else(|| {
                                 panic!(
@@ -376,6 +377,7 @@ impl<
                 } else {
                     return self
                         .store
+                        .in_memory_tree
                         .non_recursive_retrieve_prefix(
                             next_pfx.unwrap_or_else(|| {
                                 panic!(
@@ -434,9 +436,9 @@ impl<
 
             if let Some(next_ptr) = next_ptr {
                 let node = if self.mui.is_none() {
-                    self.store.retrieve_node(next_ptr)
+                    self.store.in_memory_tree.retrieve_node(next_ptr)
                 } else {
-                    self.store.retrieve_node_for_mui(
+                    self.store.in_memory_tree.retrieve_node_for_mui(
                         next_ptr,
                         self.mui.unwrap(),
                         // self.guard,
@@ -739,7 +741,7 @@ impl<
         } else {
             // calculate the node start_prefix_id lives in.
             let (start_node_id, start_bit_span) =
-                self.get_node_id_for_prefix(&start_prefix_id);
+                self.in_memory_tree.get_node_id_for_prefix(&start_prefix_id);
             trace!("start node {}", start_node_id);
 
             trace!(
@@ -762,9 +764,10 @@ impl<
             let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
 
             let node = if let Some(mui) = mui {
-                self.retrieve_node_for_mui(start_node_id, mui)
+                self.in_memory_tree
+                    .retrieve_node_for_mui(start_node_id, mui)
             } else {
-                self.retrieve_node(start_node_id)
+                self.in_memory_tree.retrieve_node(start_node_id)
             };
 
             if let Some(node) = node {
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index e1ac3c3b..54e0ee21 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -5,19 +5,15 @@ use crate::af::AddressFamily;
 use crate::local_array::in_memory::atomic_types::{
     NodeBuckets, PrefixBuckets,
 };
-use crate::local_array::persist::lsm_tree::PersistTree;
-use crate::prefix_record::{Meta, PublicRecord};
+use crate::prefix_record::PublicRecord;
 use crate::rib::{PersistStrategy, Rib};
 use inetnum::addr::Prefix;
 
-use crate::QueryResult;
+use crate::{Meta, QueryResult};
 
-use crate::local_array::in_memory::node::TreeBitMapNode;
 use crate::{MatchOptions, MatchType};
 
 use super::errors::PrefixStoreError;
-use super::in_memory::atomic_types::StoredPrefix;
-use super::in_memory::tree::{SizedStrideRef, StrideNodeId};
 use super::types::{PrefixId, RouteStatus};
 
 //------------ Prefix Matching ----------------------------------------------
@@ -37,7 +33,8 @@ where
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> QueryResult<M> {
-        let result = self.non_recursive_retrieve_prefix(prefix_id);
+        let result =
+            self.in_memory_tree.non_recursive_retrieve_prefix(prefix_id);
         let prefix = result.0;
         let more_specifics_vec = self.more_specific_prefix_iter_from(
             prefix_id,
@@ -72,7 +69,8 @@ where
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> QueryResult<M> {
-        let result = self.non_recursive_retrieve_prefix(prefix_id);
+        let result =
+            self.in_memory_tree.non_recursive_retrieve_prefix(prefix_id);
 
         let prefix = result.0;
         let less_specifics_vec = result.1.map(
@@ -151,8 +149,11 @@ where
     ) -> QueryResult<M> {
         // `non_recursive_retrieve_prefix` returns an exact match only, so no
         // longest matching prefix!
-        let mut stored_prefix =
-            self.non_recursive_retrieve_prefix(search_pfx).0.map(|pfx| {
+        let mut stored_prefix = self
+            .in_memory_tree
+            .non_recursive_retrieve_prefix(search_pfx)
+            .0
+            .map(|pfx| {
                 (
                     pfx.prefix,
                     if !options.include_withdrawn {
@@ -253,47 +254,13 @@ where
         }
     }
 
-    // fn match_prefix_in_persisted_store(
-    //     &'a self,
-    //     search_pfx: PrefixId<AF>,
-    //     mui: Option<u32>,
-    // ) -> QueryResult<M> {
-    //     let key: Vec<u8> = if let Some(mui) = mui {
-    //         PersistTree::<AF,
-    //     PREFIX_SIZE, KEY_SIZE>::prefix_mui_persistence_key(search_pfx, mui)
-    //     } else {
-    //         search_pfx.as_bytes::<PREFIX_SIZE>().to_vec()
-    //     };
-
-    //     if let Some(persist) = &self.persist_tree {
-    //         QueryResult {
-    //             prefix: Some(search_pfx.into_pub()),
-    //             match_type: MatchType::ExactMatch,
-    //             prefix_meta: persist
-    //                 .get_records_for_key(&key)
-    //                 .into_iter()
-    //                 .map(|(_, rec)| rec)
-    //                 .collect::<Vec<_>>(),
-    //             less_specifics: None,
-    //             more_specifics: None,
-    //         }
-    //     } else {
-    //         QueryResult {
-    //             prefix: None,
-    //             match_type: MatchType::EmptyMatch,
-    //             prefix_meta: vec![],
-    //             less_specifics: None,
-    //             more_specifics: None,
-    //         }
-    //     }
-    // }
-
     pub fn best_path(
         &'a self,
         search_pfx: PrefixId<AF>,
         guard: &Guard,
     ) -> Option<Result<PublicRecord<M>, PrefixStoreError>> {
-        self.non_recursive_retrieve_prefix(search_pfx)
+        self.in_memory_tree
+            .non_recursive_retrieve_prefix(search_pfx)
             .0
             .map(|p_rec| {
                 p_rec.get_path_selections(guard).best().map_or_else(
@@ -314,7 +281,8 @@ where
         tbi: &<M as Meta>::TBI,
         guard: &Guard,
     ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
-        self.non_recursive_retrieve_prefix(search_pfx)
+        self.in_memory_tree
+            .non_recursive_retrieve_prefix(search_pfx)
             .0
             .map_or(Err(PrefixStoreError::StoreNotReadyError), |p_rec| {
                 p_rec.calculate_and_store_best_backup(tbi, guard)
@@ -326,564 +294,11 @@ where
         search_pfx: PrefixId<AF>,
         guard: &Guard,
     ) -> Result<bool, PrefixStoreError> {
-        self.non_recursive_retrieve_prefix(search_pfx)
+        self.in_memory_tree
+            .non_recursive_retrieve_prefix(search_pfx)
             .0
             .map_or(Err(PrefixStoreError::StoreNotReadyError), |p| {
                 Ok(p.is_ps_outdated(guard))
             })
     }
-
-    // In a LMP search we have to go over all the nibble lengths in the
-    // stride up until the value of the actual nibble length were looking for
-    // (until we reach stride length for all strides that aren't the last)
-    // and see if the prefix bit in that position is set. Note that this does
-    // not search for prefixes with length 0 (which would always match).
-    // So for matching a nibble 1010, we have to search for 1, 10, 101 and
-    // 1010 on resp. position 1, 5, 12 and 25:
-    //                       ↓          ↓                         ↓
-    // nibble              * 0 1 00 01 10 11 000 001 010 011 100 101 110 111
-    // nibble len offset   0 1    2            3
-    //
-    // (contd.)
-    // pfx bit arr (u32)     15   16   17   18   19   20   21   22   23   24
-    // nibble              0000 0001 0010 0011 0100 0101 0110 0111 1000 1001
-    // nibble len offset      4
-    //
-    // (contd.)               ↓
-    // pfx bit arr (u32)     25   26   27   28   29   30   31
-    // nibble              1010 1011 1100 1101 1110 1111    x
-    // nibble len offset      4(contd.)
-
-    #[deprecated]
-    pub fn match_prefix_by_tree_traversal(
-        &'a self,
-        search_pfx: PrefixId<AF>,
-        options: &MatchOptions,
-        // guard: &'a Guard,
-    ) -> QueryResult<M> {
-        // --- The Default Route Prefix -------------------------------------
-
-        // The Default Route Prefix unfortunately does not fit in tree as we
-        // have it. There's no room for it in the pfxbitarr of the root node,
-        // since that can only contain serial numbers for prefixes that are
-        // children of the root node. We, however, want the default prefix
-        // which lives on the root node itself! We are *not* going to return
-        // all of the prefixes in the tree as more-specifics.
-        if search_pfx.get_len() == 0 {
-            // match self.load_default_route_prefix_serial() {
-            //     0 => {
-            //         return QueryResult {
-            //             prefix: None,
-            //             prefix_meta: vec![],
-            //             match_type: MatchType::EmptyMatch,
-            //             less_specifics: None,
-            //             more_specifics: None,
-            //         };
-            //     }
-
-            //     _serial => {
-            let prefix_meta = self
-                .retrieve_prefix(PrefixId::new(AF::zero(), 0))
-                .map(|sp| sp.0.record_map.as_records())
-                .unwrap_or_default();
-            return QueryResult {
-                prefix: Prefix::new(
-                    search_pfx.get_net().into_ipaddr(),
-                    search_pfx.get_len(),
-                )
-                .ok(),
-                prefix_meta,
-                match_type: MatchType::ExactMatch,
-                less_specifics: None,
-                more_specifics: None,
-            };
-            // }
-            // }
-        }
-
-        let mut stride_end = 0;
-
-        let root_node_id = self.get_root_node_id();
-        let mut node = match self.get_stride_for_id(root_node_id) {
-            3 => self.retrieve_node(root_node_id).unwrap(),
-            4 => self.retrieve_node(root_node_id).unwrap(),
-            _ => self.retrieve_node(root_node_id).unwrap(),
-        };
-
-        let mut nibble;
-        let mut nibble_len;
-
-        //---- result values ------------------------------------------------
-
-        // These result values are kept in mutable variables, and assembled
-        // at the end into a QueryResult struct. This proved to result in the
-        // most efficient code, where we don't have to match on
-        // SizedStrideNode over and over. The `match_type` field in the
-        // QueryResult is computed at the end.
-
-        // The final prefix
-        let mut match_prefix_idx: Option<PrefixId<AF>> = None;
-
-        // The indexes of the less-specifics
-        let mut less_specifics_vec = if options.include_less_specifics {
-            Some(Vec::<PrefixId<AF>>::new())
-        } else {
-            None
-        };
-
-        // The indexes of the more-specifics.
-        let mut more_specifics_vec = if options.include_more_specifics {
-            Some(Vec::<PrefixId<AF>>::new())
-        } else {
-            None
-        };
-
-        //---- Stride Processing --------------------------------------------
-
-        // We're going to iterate over all the strides in the treebitmap (so
-        // up to the last bit in the max prefix length for that tree). When
-        // a final prefix is found or we get to the end of the strides,
-        // depending on the options.match_type (the type requested by the
-        // user). we ALWAYS break out of the loop. WE ALWAYS BREAK OUT OF THE
-        // LOOP. Just before breaking some processing is done inside the loop
-        // before the break (looking up more-specifics mainly), which looks a
-        // bit repetitious, but again it's been done like that to avoid
-        // having to match over a SizedStrideNode again in the
-        // `post-processing` section.
-
-        for stride in self.get_stride_sizes() {
-            stride_end += stride;
-
-            let last_stride = search_pfx.get_len() < stride_end;
-
-            nibble_len = if last_stride {
-                stride + search_pfx.get_len() - stride_end
-            } else {
-                *stride
-            };
-
-            // Shift left and right to set the bits to zero that are not
-            // in the nibble we're handling here.
-            nibble = AddressFamily::get_nibble(
-                search_pfx.get_net(),
-                stride_end - stride,
-                nibble_len,
-            );
-
-            match node {
-                SizedStrideRef::Stride3(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-
-                    // This whole match assumes that:
-                    // - if the first value in the return tuple of
-                    //   `search_fn` holds a value, then we need to continue
-                    //   searching by following the node contained in the
-                    //   value.
-                    // - The second value in the tuple holds the prefix that
-                    //   was found.
-                    // The less_specifics_vec is mutated by `search_fn` to
-                    // hold the prefixes found along the way, in the cases
-                    // where `include_less_specifics` was requested by the
-                    // user.
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        // This and the next match will handle all
-                        // intermediary nodes, but they might also handle
-                        // exit nodes.
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx);
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        // This handles exact and longest matches: there are
-                        // no more children, but there is a prefix on this
-                        // node.
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                        StrideNodeId::new_with_cleaned_id(
-                                            search_pfx.get_net(),
-                                            stride_end - stride,
-                                        ),
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx);
-                            break;
-                        }
-                        // This handles cases where there's no prefix (and no
-                        // child) for exact match or longest match, the empty
-                        // match - which doesn't care about actually finding
-                        // a prefix - just continues in search of
-                        // more-specifics.
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    // To make sure we don't process this
-                                    // match arm more then once, we return
-                                    // early here.
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                //---- From here only repetitions for all strides -----------
-                // For comments see the code above for the Stride3 arm.
-                SizedStrideRef::Stride4(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx);
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                        StrideNodeId::new_with_cleaned_id(
-                                            search_pfx.get_net(),
-                                            stride_end - stride,
-                                        ),
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx);
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    // To make sure we don't process this match arm more then once, we
-                                    // return early here.
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                SizedStrideRef::Stride5(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx);
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                        StrideNodeId::new_with_cleaned_id(
-                                            search_pfx.get_net(),
-                                            stride_end - stride,
-                                        ),
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx);
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-            }
-        }
-        //------------------ end of Stride branch arm repetition ------------
-
-        //------------------ post-processing --------------------------------
-
-        // If the above loop finishes (so not hitting a break) we have
-        // processed all strides and have found a child node and maybe a
-        // prefix. Now we will look up more-specifics for longest-matching
-        // prefixes that were found in the last stride only. Note that still
-        // any of the match_types (as specified by the user, not the return
-        // type) may end up here.
-
-        let mut match_type: MatchType = MatchType::EmptyMatch;
-        let prefix = None;
-        if let Some(pfx_idx) = match_prefix_idx {
-            match_type = match self.retrieve_prefix(pfx_idx) {
-                Some(prefix) => {
-                    if prefix.0.prefix.get_len() == search_pfx.get_len() {
-                        MatchType::ExactMatch
-                    } else {
-                        MatchType::LongestMatch
-                    }
-                }
-                None => MatchType::EmptyMatch,
-            };
-        };
-
-        QueryResult {
-            prefix: prefix.map(|pfx: (&StoredPrefix<AF, M>, usize)| {
-                pfx.0.prefix.into_pub()
-            }),
-            prefix_meta: prefix
-                .map(|pfx| pfx.0.record_map.as_records())
-                .unwrap_or_default(),
-            match_type,
-            less_specifics: if options.include_less_specifics {
-                less_specifics_vec
-                    .unwrap()
-                    .iter()
-                    .filter_map(move |p| {
-                        self.retrieve_prefix(*p).map(|p| {
-                            Some((p.0.prefix, p.0.record_map.as_records()))
-                        })
-                    })
-                    .collect()
-            } else {
-                None
-            },
-            more_specifics: if options.include_more_specifics {
-                more_specifics_vec.map(|vec| {
-                    vec.into_iter()
-                        .map(|p| {
-                            self.retrieve_prefix(p)
-                                .unwrap_or_else(|| {
-                                    panic!(
-                                        "more specific {:?} does not exist",
-                                        p
-                                    )
-                                })
-                                .0
-                        })
-                        .map(|sp| (sp.prefix, sp.record_map.as_records()))
-                        .collect()
-                })
-            } else {
-                None
-            },
-        }
-    }
 }
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 300c6583..fe849321 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -1,229 +1,33 @@
-// ----------- THE STORE ----------------------------------------------------
-//
-// The CustomAllocStore provides in-memory storage for the BitTreeMapNodes
-// and for prefixes and their meta-data. The storage for node is on the
-// `buckets` field, and the prefixes are stored in, well, the `prefixes`
-// field. They are both organised in the same way, as chained hash tables,
-// one per (prefix|node)-length. The hashing function (that is detailed
-// lower down in this file), basically takes the address part of the
-// node|prefix and uses `(node|prefix)-address part % bucket size`
-// as its index.
-//
-// Both the prefixes and the buckets field have one bucket per (prefix|node)
-// -length that start out with a fixed-size array. The size of the arrays is
-// set in the rotonda_macros/maps.rs file.
-//
-// For lower (prefix|node)-lengths the number of elements in the array is
-// equal to the number of prefixes in that length, so there's exactly one
-// element per (prefix|node). For greater lengths there will be collisions,
-// in that case the stored (prefix|node) will have a reference to another
-// bucket (also of a fixed size), that holds a (prefix|node) that collided
-// with the one that was already stored. A (node|prefix) lookup will have to
-// go over all (node|prefix) buckets until it matches the requested (node|
-// prefix) or it reaches the end of the chain.
-//
-// The chained (node|prefixes) are occupied at a first-come, first-serve
-// basis, and are not re-ordered on new insertions of (node|prefixes). This
-// may change in the future, since it prevents iterators from being ordered.
-//
-// One of the nice things of having one table per (node|prefix)-length is that
-// a search can start directly at the prefix-length table it wishes, and go
-// go up and down into other tables if it needs to (e.g., because more- or
-// less-specifics were asked for). In contrast if you do a lookup by
-// traversing the tree of nodes, we would always have to go through the root-
-// node first and then go up the tree to the requested node. The lower nodes
-// of the tree (close to the root) would be a formidable bottle-neck then.
-//
-// Currently, the meta-data is an atomically stored value, that is required to
-// implement the `Meta` and the `Clone` trait. New meta-data
-// instances are stored atomically without further ado, but updates to a
-// piece of meta-data are done by merging the previous meta-data with the new
-// meta-data, through use of the `MergeUpdate` trait.
-//
-// The `upsert_prefix` methods retrieve only the most recent insert
-// for a prefix (for now).
-//
-// Future work could have a user-configurable retention strategy that allows
-// the meta-data to be stored as a linked-list of references, where each
-// meta-data object has a reference to its predecessor.
-//
-// Prefix example
-//
-//         (level 0 arrays)         prefixes  bucket
-//                                    /len     size
-//         ┌──┐
-// len /0  │ 0│                        1        1     ■
-//         └──┘                                       │
-//         ┌──┬──┐                                    │
-// len /1  │00│01│                     2        2     │
-//         └──┴──┘                                 perfect
-//         ┌──┬──┬──┬──┐                             hash
-// len /2  │  │  │  │  │               4        4     │
-//         └──┴──┴──┴──┘                              │
-//         ┌──┬──┬──┬──┬──┬──┬──┬──┐                  │
-// len /3  │  │  │  │  │  │  │  │  │   8        8     ■
-//         └──┴──┴──┴──┴──┴──┴──┴──┘
-//         ┌──┬──┬──┬──┬──┬──┬──┬──┐                        ┌────────────┐
-// len /4  │  │  │  │  │  │  │  │  │   8        16 ◀────────│ collision  │
-//         └──┴──┴──┴┬─┴──┴──┴──┴──┘                        └────────────┘
-//                   └───┐
-//                       │              ┌─collision─────────┐
-//                   ┌───▼───┐          │                   │
-//                   │       │ ◀────────│ 0x0100 and 0x0101 │
-//                   │ 0x010 │          └───────────────────┘
-//                   │       │
-//                   ├───────┴──────────────┬──┬──┐
-//                   │ StoredPrefix 0x0101  │  │  │
-//                   └──────────────────────┴─┬┴─┬┘
-//                                            │  │
-//                       ┌────────────────────┘  └──┐
-//            ┌──────────▼──────────┬──┐          ┌─▼┬──┐
-//         ┌─▶│ metadata (current)  │  │          │ 0│ 1│ (level 1 array)
-//         │  └─────────────────────┴──┘          └──┴──┘
-//    merge└─┐                        │             │
-//    update │           ┌────────────┘             │
-//           │┌──────────▼──────────┬──┐        ┌───▼───┐
-//         ┌─▶│ metadata (previous) │  │        │       │
-//         │  └─────────────────────┴──┘        │  0x0  │
-//    merge└─┐                        │         │       │
-//    update │           ┌────────────┘         ├───────┴──────────────┬──┐
-//           │┌──────────▼──────────┬──┐        │ StoredPrefix 0x0110  │  │
-//            │ metadata (oldest)   │  │        └──────────────────────┴──┘
-//            └─────────────────────┴──┘                                 │
-//                                                         ┌─────────────┘
-//                                              ┌──────────▼──────────────┐
-//                                              │ metadata (current)      │
-//                                              └─────────────────────────┘
-
-// Note about the memory usage of the data-structures of the Buckets
-//
-// As said, the prefixes and nodes are stored in buckets. A bucket right now
-// is of type `[MaybeUnit<Atomic<StoredPrefix>>]`, this has the advantage
-// that the length can be variable, based on the stride size for that level.
-// It saves us to have to implement a generic something.
-// Another advantage is the fixed place in which an atomic StoredPrefix
-// lives: this makes compare-and-swapping it relatively straight forward.
-// Each accessing thread would try to read/write the exact same entry in the
-// array, so shouldn't be any 'rug pulling' on the whole array.
-//
-// A disadvantage is that this is a fixed size, sparse array the moment it
-// is created. Theoretically, an `Atomic<Vec<StoredPrefix>`
-// would not have this disadvantage. Manipulating the whole vec atomically
-// though is very tricky (we would have to atomically compare-and-swap the
-// whole vec each time the prefix meta-data is changed) and inefficient,
-// since we would have to either keep the vec sorted on `PrefixId` at all
-// times, or, we would have to inspect each value in the vec on *every* read
-// or write. the StoredPrefix (this is a challenge in itself, since the
-// StoredPrefix needs to be read atomically to retrieve the PrefixId).
-// Compare-and-swapping a whole vec most probably would need a hash over the
-// vec to determine whether it was changed. I gave up on this approach,
-//
-// Another approach to try to limit the memory use is to try to use other
-// indexes in the same array on collision (the array mentioned above), before
-// heading off and following the reference to the next bucket. This would
-// limit the amount of (sparse) arrays being created for a typical prefix
-// treebitmap, at the cost of longer average search times. Two
-// implementations of this approach are Cuckoo hashing[^1], and Skip Lists.
-// Skip lists[^2] are a probabilistic data-structure, famously used by Redis,
-// (and by TiKv). I haven't tries either of these. Crossbeam has a SkipList
-// implementation, that wasn't ready at the time I wrote this. Cuckoo
-// hashing has the advantage of being easier to understand/implement. Maybe
-// Cuckoo hashing can also be combined with Fibonacci hashing[^3]. Note that
-// Robin Hood hashing maybe faster than Cuckoo hashing for reads, but it
-// requires shifting around existing entries, which is rather costly to do
-// atomically (and complex).
-
-// [^1]: [https://en.wikipedia.org/wiki/Cuckoo_hashing]
-// [^3]: [https://docs.rs/crossbeam-skiplist/0.1.1/crossbeam_skiplist/]
-// [^3]: [https://probablydance.com/2018/06/16/fibonacci-hashing-
-//  the-optimization-that-the-world-forgot-or-a-better-alternative-
-//  to-integer-modulo/]
-
-// Notes on memory leaks in Rotonda-store
-//
-// Both valgrind and miri report memory leaks on the multi-threaded prefix
-// store. Valgrind only reports it when it a binary stops using the tree,
-// while still keeping it around. An interrupted use of the mt-prefix-store
-// does not report any memory leaks. Miri is persistent in reporting memory
-// leaks in the mt-prefix-store. They both report the memory leaks in the same
-// location: the init method of the node- and prefix-buckets.
-//
-// I have reasons to believe these reported memory leaks aren't real, or that
-// crossbeam-epoch leaks a bit of memory when creating a new `Atomic`
-// instance. Since neither prefix nor node buckets can or should be dropped
-// this is not a big issue anyway, it just means that an `Atomic` occupies
-// more memory than it could in an optimal situation. Since we're not storing
-// the actual meta-data in an `Atomic` (it is stored in an `flurry Map`), this
-// means memory usage won't grow on updating the meta-data on a prefix,
-// (unless the meta-data itself grows of course, but that's up to the user).
-//
-// To get a better understanding on the nature of the reported memory leaks I
-// have created a branch (`vec_set`) that replaces the dynamically sized array
-// with a (equally sparse) Vec, that is not filled with `Atomic:::null()`, but
-// with `Option<StoredPrefix` instead, in order to see if this would eliminate
-// the memory leaks reporting. It did not. Valgrind still reports the memory
-// leaks at the same location, although they're now reported as `indirectly
-// leaked`, instead of `directly`. Miri is unchanged. The attentive reader may
-// now suspect that the `Atomic` inside the `AtomicStoredPrefix` may be the
-// culprit, but: a. Both miri and valgrind report the leaks still in the same
-// place: the creation of the buckets, not the creation of the stored
-//    prefixes. b. Tests that look at the memory usage of the prefix stores
-// under heavy modification of existing prefixes do not exhibit memory leaking
-//    behavior. c. Tests that look at memory usage under addition of new
-//    prefixes exhibit linear incrementation of memory usage (which is
-// expected). d. Tests that look at memory usage under contention do not
-//    exhibit increased memory usage either.
-//
-// My strong suspicion is that both Miri and Valgrind report the sparse slots
-// in the buckets as leaks, no matter whether they're `Atomic::null()`, or
-// `None` values, probably as a result of the way `crossbeam-epoch` indexes
-// into these, with pointer arithmetic (unsafe as hell).
-//
-// I would be super grateful if somebody would prove me wrong and can point to
-// an actual memory leak in the mt-prefix-store (and even more if they can
-// produce a fix for it).
-
 use super::super::persist::lsm_tree::PersistTree;
-use crate::local_array::in_memory::atomic_stride::AtomicBitmap;
 use std::path::Path;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use inetnum::addr::Prefix;
-use log::{debug, info, log_enabled, trace};
+use log::{debug, error, info, log_enabled, trace};
 
 use crossbeam_epoch as epoch;
-use crossbeam_utils::Backoff;
 use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
 
-use crate::local_array::in_memory::tree::{
-    SizedStrideNode, Stride, Stride3, Stride4, Stride5, StrideNodeId,
-    TreeBitMap,
-};
+use crate::local_array::in_memory::tree::{StrideNodeId, TreeBitMap};
 use crate::local_array::types::PrefixId;
 use crate::stats::CreatedNodes;
 use crate::{
-    local_array::{bit_span::BitSpan, errors::PrefixStoreError},
-    prefix_record::PublicRecord,
+    local_array::errors::PrefixStoreError, prefix_record::PublicRecord,
 };
 
+use crate::local_array::in_memory::atomic_types::PrefixBuckets;
 use crate::local_array::in_memory::atomic_types::{
     NodeBuckets, StoredPrefix,
 };
-use crate::local_array::in_memory::atomic_types::{NodeSet, StoredNode};
-use crate::local_array::in_memory::atomic_types::{PrefixBuckets, PrefixSet};
-use crate::local_array::in_memory::node::TreeBitMapNode;
-use crate::local_array::in_memory::tree::SizedStrideRef;
+use crate::local_array::in_memory::tree::{NewNodeOrIndex, SizedStrideRef};
 
 // Make sure to also import the other methods for the Rib, so the proc macro
 // create_store can use them.
 pub use crate::local_array::iterators;
 pub use crate::local_array::query;
 
-use crate::{
-    impl_search_level, impl_search_level_for_mui, retrieve_node_mut_closure,
-    store_node_closure, MatchType, Meta, QueryResult,
-};
+use crate::{insert_match, IPv4, IPv6, MatchType, Meta, QueryResult};
 
 use crate::AddressFamily;
 
@@ -353,7 +157,6 @@ pub struct Rib<
     pub(crate) in_memory_tree: TreeBitMap<AF, M, NB, PB>,
     #[cfg(feature = "persist")]
     persist_tree: Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
-    // Global Roaring BitMap INdex that stores MUIs.
     pub counters: Counters,
 }
 
@@ -367,10 +170,16 @@ impl<
         const KEY_SIZE: usize,
     > Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
-    pub(crate) fn init(
-        root_node: SizedStrideNode<AF>,
+    pub fn new(
         config: StoreConfig,
-    ) -> Result<Self, Box<dyn std::error::Error>> {
+    ) -> Result<
+        Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
+        Box<dyn std::error::Error>,
+    > {
+        Rib::<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>::init(config)
+    }
+
+    fn init(config: StoreConfig) -> Result<Self, Box<dyn std::error::Error>> {
         info!("store: initialize store {}", AF::BITS);
 
         let persist_tree = match config.persist_strategy {
@@ -383,255 +192,397 @@ impl<
 
         let store = Rib {
             config,
-            in_memory_tree: TreeBitMap::<AF, M, NB, PB>::init(),
+            in_memory_tree: TreeBitMap::<AF, M, NB, PB>::new()?,
             persist_tree,
             counters: Counters::default(),
         };
 
-        let _retry_count = store.store_node(
-            StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0),
-            0_u32,
-            root_node,
-        )?;
-
         Ok(store)
     }
 
-    pub(crate) fn acquire_new_node_id(
-        &self,
-        (prefix_net, sub_prefix_len): (AF, u8),
-    ) -> StrideNodeId<AF> {
-        StrideNodeId::new_with_cleaned_id(prefix_net, sub_prefix_len)
-    }
-
-    // Create a new node in the store with payload `next_node`.
-    //
-    // Next node will be ignored if a node with the same `id` already exists,
-    // but the multi_uniq_id will be added to the rbm_index of the NodeSet.
-    //
-    // Returns: a tuple with the node_id of the created node and the number of
-    // retry_count
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn store_node(
+    pub fn insert(
         &self,
-        id: StrideNodeId<AF>,
-        multi_uniq_id: u32,
-        next_node: SizedStrideNode<AF>,
-    ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn Fn(
-                &SearchLevel<AF, S>,
-                &NodeSet<AF, S>,
-                TreeBitMapNode<AF, S>,
-                u32, // multi_uniq_id
-                u8,  // the store level
-                u32, // retry_count
-            ) -> Result<
-                (StrideNodeId<AF>, u32),
-                PrefixStoreError,
-            >,
-        }
+        pfx: PrefixId<AF>,
+        record: PublicRecord<M>,
+        update_path_selections: Option<M::TBI>,
+    ) -> Result<UpsertReport, PrefixStoreError> {
+        let guard = &epoch::pin();
 
-        let search_level_3 =
-            store_node_closure![Stride3; id; guard; back_off;];
-        let search_level_4 =
-            store_node_closure![Stride4; id; guard; back_off;];
-        let search_level_5 =
-            store_node_closure![Stride5; id; guard; back_off;];
-
-        if log_enabled!(log::Level::Trace) {
-            debug!(
-                "{} store: Store node {}: {:?} mui {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                next_node,
-                multi_uniq_id
-            );
-        }
-        self.counters.inc_nodes_count();
-
-        match next_node {
-            SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
-                &search_level_3,
-                self.in_memory_tree.node_buckets.get_store3(id),
-                new_node,
-                multi_uniq_id,
-                0,
-                0,
-            ),
-            SizedStrideNode::Stride4(new_node) => (search_level_4.f)(
-                &search_level_4,
-                self.in_memory_tree.node_buckets.get_store4(id),
-                new_node,
-                multi_uniq_id,
-                0,
-                0,
-            ),
-            SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
-                &search_level_5,
-                self.in_memory_tree.node_buckets.get_store5(id),
-                new_node,
-                multi_uniq_id,
-                0,
-                0,
-            ),
+        if pfx.get_len() == 0 {
+            let res = self.update_default_route_prefix_meta(record, guard)?;
+            return Ok(res);
         }
-    }
 
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_node(
-        &'a self,
-        id: StrideNodeId<AF>,
-    ) -> Option<SizedStrideRef<'a, AF>> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, S>,
-                &'a NodeSet<AF, S>,
-                u8,
-            )
-                -> Option<SizedStrideRef<'a, AF>>,
-        }
+        let mut stride_end: u8 = 0;
+        let mut cur_i = self.in_memory_tree.get_root_node_id();
+        let mut level: u8 = 0;
+        let mut acc_retry_count = 0;
 
-        let search_level_3 = impl_search_level![Stride3; id;];
-        let search_level_4 = impl_search_level![Stride4; id;];
-        let search_level_5 = impl_search_level![Stride5; id;];
+        loop {
+            let stride =
+                self.in_memory_tree.get_stride_sizes()[level as usize];
+            stride_end += stride;
+            let nibble_len = if pfx.get_len() < stride_end {
+                stride + pfx.get_len() - stride_end
+            } else {
+                stride
+            };
 
-        if log_enabled!(log::Level::Trace) {
-            trace!(
-                "{} store: Retrieve node {} from l{}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                id.get_id().1
+            let nibble = AF::get_nibble(
+                pfx.get_net(),
+                stride_end - stride,
+                nibble_len,
             );
-        }
-
-        match self.get_stride_for_id(id) {
-            3 => (search_level_3.f)(
-                &search_level_3,
-                self.in_memory_tree.node_buckets.get_store3(id),
-                0,
-            ),
-            4 => (search_level_4.f)(
-                &search_level_4,
-                self.in_memory_tree.node_buckets.get_store4(id),
-                0,
-            ),
-            _ => (search_level_5.f)(
-                &search_level_5,
-                self.in_memory_tree.node_buckets.get_store5(id),
-                0,
-            ),
+            let is_last_stride = pfx.get_len() <= stride_end;
+            let stride_start = stride_end - stride;
+            // let back_off = crossbeam_utils::Backoff::new();
+
+            // insert_match! returns the node_id of the next node to be
+            // traversed. It was created if it did not exist.
+            let node_result = insert_match![
+                // applicable to the whole outer match in the macro
+                self;
+                guard;
+                nibble_len;
+                nibble;
+                is_last_stride;
+                pfx;
+                record;
+                // perform an update for the paths in this record
+                update_path_selections;
+                // the length at the start of the stride a.k.a. start_bit
+                stride_start;
+                stride;
+                cur_i;
+                level;
+                acc_retry_count;
+                // Strides to create match arm for; stats level
+                Stride3; 0,
+                Stride4; 1,
+                Stride5; 2
+            ];
+
+            match node_result {
+                Ok((next_id, retry_count)) => {
+                    cur_i = next_id;
+                    level += 1;
+                    acc_retry_count += retry_count;
+                }
+                Err(err) => {
+                    if log_enabled!(log::Level::Error) {
+                        error!(
+                            "{} failing to store (intermediate) node {}.
+                             Giving up this node. This shouldn't happen!",
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread"),
+                            cur_i,
+                        );
+                        error!(
+                            "{} {}",
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread"),
+                            err
+                        );
+                    }
+                }
+            }
         }
     }
 
-    // retrieve a node, but only its bitmap index contains the specified mui.
-    // Used for iterators per mui.
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_node_for_mui(
-        &'a self,
-        id: StrideNodeId<AF>,
-        // The mui that is tested to be present in the nodes bitmap index
-        mui: u32,
-    ) -> Option<SizedStrideRef<'a, AF>> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, S>,
-                &'a NodeSet<AF, S>,
-                u8,
-            )
-                -> Option<SizedStrideRef<'a, AF>>,
-        }
-
-        let search_level_3 = impl_search_level_for_mui![Stride3; id; mui;];
-        let search_level_4 = impl_search_level_for_mui![Stride4; id; mui;];
-        let search_level_5 = impl_search_level_for_mui![Stride5; id; mui;];
-
-        if log_enabled!(log::Level::Trace) {
-            trace!(
-                "{} store: Retrieve node {} from l{} for mui {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                id.get_id().1,
-                mui
-            );
-        }
+    // Yes, we're hating this. But, the root node has no room for a serial of
+    // the prefix 0/0 (the default route), which doesn't even matter, unless,
+    // UNLESS, somebody wants to store a default route. So we have to store a
+    // serial for this prefix. The normal place for a serial of any prefix is
+    // on the pfxvec of its paren. But, hey, guess what, the
+    // default-route-prefix lives *on* the root node, and, you know, the root
+    // node doesn't have a parent. We can:
+    // - Create a type RootTreeBitmapNode with a ptrbitarr with a size one
+    //   bigger than a "normal" TreeBitMapNod for the first stride size. no we
+    //   have to iterate over the root-node type in all matches on
+    //   stride_size, just because we have exactly one instance of the
+    //   RootTreeBitmapNode. So no.
+    // - Make the `get_pfx_index` method on the implementations of the
+    //   `Stride` trait check for a length of zero and branch if it is and
+    //   return the serial of the root node. Now each and every call to this
+    //   method will have to check a condition for exactly one instance of
+    //   RootTreeBitmapNode. So again, no.
+    // - The root node only gets used at the beginning of a search query or an
+    //   insert. So if we provide two specialised methods that will now how to
+    //   search for the default-route prefix and now how to set serial for
+    //  that prefix and make sure we start searching/inserting with one of
+    //   those specialized methods we're good to go.
+    fn update_default_route_prefix_meta(
+        &self,
+        record: PublicRecord<M>,
+        guard: &epoch::Guard,
+    ) -> Result<UpsertReport, PrefixStoreError> {
+        trace!("Updating the default route...");
+
+        if let Some(root_node) = self.in_memory_tree.retrieve_node_mut(
+            self.in_memory_tree.get_root_node_id(),
+            record.multi_uniq_id,
+        ) {
+            match root_node {
+                SizedStrideRef::Stride3(_) => {
+                    self.in_memory_tree
+                        .node_buckets
+                        .get_store3(self.in_memory_tree.get_root_node_id())
+                        .update_rbm_index(record.multi_uniq_id)?;
+                }
+                SizedStrideRef::Stride4(_) => {
+                    self.in_memory_tree
+                        .node_buckets
+                        .get_store4(self.in_memory_tree.get_root_node_id())
+                        .update_rbm_index(record.multi_uniq_id)?;
+                }
+                SizedStrideRef::Stride5(_) => {
+                    self.in_memory_tree
+                        .node_buckets
+                        .get_store5(self.in_memory_tree.get_root_node_id())
+                        .update_rbm_index(record.multi_uniq_id)?;
+                }
+            };
+        };
 
-        match self.get_stride_for_id(id) {
-            3 => (search_level_3.f)(
-                &search_level_3,
-                self.in_memory_tree.node_buckets.get_store3(id),
-                0,
-            ),
-            4 => (search_level_4.f)(
-                &search_level_4,
-                self.in_memory_tree.node_buckets.get_store4(id),
-                0,
-            ),
-            _ => (search_level_5.f)(
-                &search_level_5,
-                self.in_memory_tree.node_buckets.get_store5(id),
-                0,
-            ),
-        }
+        self.upsert_prefix(
+            PrefixId::new(AF::zero(), 0),
+            record,
+            // Do not update the path selection for the default route.
+            None,
+            guard,
+        )
     }
 
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_node_mut(
-        &'a self,
-        id: StrideNodeId<AF>,
-        multi_uniq_id: u32,
-    ) -> Option<SizedStrideRef<'a, AF>> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, S>,
-                &'a NodeSet<AF, S>,
-                u8,
-            )
-                -> Option<SizedStrideRef<'a, AF>>,
-        }
+    // // Create a new node in the store with payload `next_node`.
+    // //
+    // // Next node will be ignored if a node with the same `id` already exists,
+    // // but the multi_uniq_id will be added to the rbm_index of the NodeSet.
+    // //
+    // // Returns: a tuple with the node_id of the created node and the number of
+    // // retry_count
+    // #[allow(clippy::type_complexity)]
+    // pub(crate) fn store_node(
+    //     &self,
+    //     id: StrideNodeId<AF>,
+    //     multi_uniq_id: u32,
+    //     next_node: SizedStrideNode<AF>,
+    // ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
+    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+    //         f: &'s dyn Fn(
+    //             &SearchLevel<AF, S>,
+    //             &NodeSet<AF, S>,
+    //             TreeBitMapNode<AF, S>,
+    //             u32, // multi_uniq_id
+    //             u8,  // the store level
+    //             u32, // retry_count
+    //         ) -> Result<
+    //             (StrideNodeId<AF>, u32),
+    //             PrefixStoreError,
+    //         >,
+    //     }
+
+    //     let search_level_3 =
+    //         store_node_closure![Stride3; id; guard; back_off;];
+    //     let search_level_4 =
+    //         store_node_closure![Stride4; id; guard; back_off;];
+    //     let search_level_5 =
+    //         store_node_closure![Stride5; id; guard; back_off;];
+
+    //     if log_enabled!(log::Level::Trace) {
+    //         debug!(
+    //             "{} store: Store node {}: {:?} mui {}",
+    //             std::thread::current().name().unwrap_or("unnamed-thread"),
+    //             id,
+    //             next_node,
+    //             multi_uniq_id
+    //         );
+    //     }
+    //     self.counters.inc_nodes_count();
+
+    //     match next_node {
+    //         SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
+    //             &search_level_3,
+    //             self.in_memory_tree.node_buckets.get_store3(id),
+    //             new_node,
+    //             multi_uniq_id,
+    //             0,
+    //             0,
+    //         ),
+    //         SizedStrideNode::Stride4(new_node) => (search_level_4.f)(
+    //             &search_level_4,
+    //             self.in_memory_tree.node_buckets.get_store4(id),
+    //             new_node,
+    //             multi_uniq_id,
+    //             0,
+    //             0,
+    //         ),
+    //         SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
+    //             &search_level_5,
+    //             self.in_memory_tree.node_buckets.get_store5(id),
+    //             new_node,
+    //             multi_uniq_id,
+    //             0,
+    //             0,
+    //         ),
+    //     }
+    // }
 
-        let search_level_3 =
-            retrieve_node_mut_closure![Stride3; id; multi_uniq_id;];
-        let search_level_4 =
-            retrieve_node_mut_closure![Stride4; id; multi_uniq_id;];
-        let search_level_5 =
-            retrieve_node_mut_closure![Stride5; id; multi_uniq_id;];
-
-        if log_enabled!(log::Level::Trace) {
-            trace!(
-                "{} store: Retrieve node mut {} from l{}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                id.get_id().1
-            );
-        }
+    // #[allow(clippy::type_complexity)]
+    // pub(crate) fn retrieve_node(
+    //     &'a self,
+    //     id: StrideNodeId<AF>,
+    // ) -> Option<SizedStrideRef<'a, AF>> {
+    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+    //         f: &'s dyn for<'a> Fn(
+    //             &SearchLevel<AF, S>,
+    //             &'a NodeSet<AF, S>,
+    //             u8,
+    //         )
+    //             -> Option<SizedStrideRef<'a, AF>>,
+    //     }
+
+    //     let search_level_3 = impl_search_level![Stride3; id;];
+    //     let search_level_4 = impl_search_level![Stride4; id;];
+    //     let search_level_5 = impl_search_level![Stride5; id;];
+
+    //     if log_enabled!(log::Level::Trace) {
+    //         trace!(
+    //             "{} store: Retrieve node {} from l{}",
+    //             std::thread::current().name().unwrap_or("unnamed-thread"),
+    //             id,
+    //             id.get_id().1
+    //         );
+    //     }
+
+    //     match self.get_stride_for_id(id) {
+    //         3 => (search_level_3.f)(
+    //             &search_level_3,
+    //             self.in_memory_tree.node_buckets.get_store3(id),
+    //             0,
+    //         ),
+    //         4 => (search_level_4.f)(
+    //             &search_level_4,
+    //             self.in_memory_tree.node_buckets.get_store4(id),
+    //             0,
+    //         ),
+    //         _ => (search_level_5.f)(
+    //             &search_level_5,
+    //             self.in_memory_tree.node_buckets.get_store5(id),
+    //             0,
+    //         ),
+    //     }
+    // }
 
-        match self.in_memory_tree.node_buckets.get_stride_for_id(id) {
-            3 => (search_level_3.f)(
-                &search_level_3,
-                self.in_memory_tree.node_buckets.get_store3(id),
-                0,
-            ),
-
-            4 => (search_level_4.f)(
-                &search_level_4,
-                self.in_memory_tree.node_buckets.get_store4(id),
-                0,
-            ),
-            _ => (search_level_5.f)(
-                &search_level_5,
-                self.in_memory_tree.node_buckets.get_store5(id),
-                0,
-            ),
-        }
-    }
+    // retrieve a node, but only its bitmap index contains the specified mui.
+    // Used for iterators per mui.
+    // #[allow(clippy::type_complexity)]
+    // pub(crate) fn retrieve_node_for_mui(
+    //     &'a self,
+    //     id: StrideNodeId<AF>,
+    //     // The mui that is tested to be present in the nodes bitmap index
+    //     mui: u32,
+    // ) -> Option<SizedStrideRef<'a, AF>> {
+    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+    //         f: &'s dyn for<'a> Fn(
+    //             &SearchLevel<AF, S>,
+    //             &'a NodeSet<AF, S>,
+    //             u8,
+    //         )
+    //             -> Option<SizedStrideRef<'a, AF>>,
+    //     }
+
+    //     let search_level_3 = impl_search_level_for_mui![Stride3; id; mui;];
+    //     let search_level_4 = impl_search_level_for_mui![Stride4; id; mui;];
+    //     let search_level_5 = impl_search_level_for_mui![Stride5; id; mui;];
+
+    //     if log_enabled!(log::Level::Trace) {
+    //         trace!(
+    //             "{} store: Retrieve node {} from l{} for mui {}",
+    //             std::thread::current().name().unwrap_or("unnamed-thread"),
+    //             id,
+    //             id.get_id().1,
+    //             mui
+    //         );
+    //     }
+
+    //     match self.get_stride_for_id(id) {
+    //         3 => (search_level_3.f)(
+    //             &search_level_3,
+    //             self.in_memory_tree.node_buckets.get_store3(id),
+    //             0,
+    //         ),
+    //         4 => (search_level_4.f)(
+    //             &search_level_4,
+    //             self.in_memory_tree.node_buckets.get_store4(id),
+    //             0,
+    //         ),
+    //         _ => (search_level_5.f)(
+    //             &search_level_5,
+    //             self.in_memory_tree.node_buckets.get_store5(id),
+    //             0,
+    //         ),
+    //     }
+    // }
 
-    pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
-        StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0)
-    }
+    // #[allow(clippy::type_complexity)]
+    // pub(crate) fn retrieve_node_mut(
+    //     &'a self,
+    //     id: StrideNodeId<AF>,
+    //     multi_uniq_id: u32,
+    // ) -> Option<SizedStrideRef<'a, AF>> {
+    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+    //         f: &'s dyn for<'a> Fn(
+    //             &SearchLevel<AF, S>,
+    //             &'a NodeSet<AF, S>,
+    //             u8,
+    //         )
+    //             -> Option<SizedStrideRef<'a, AF>>,
+    //     }
+
+    //     let search_level_3 =
+    //         retrieve_node_mut_closure![Stride3; id; multi_uniq_id;];
+    //     let search_level_4 =
+    //         retrieve_node_mut_closure![Stride4; id; multi_uniq_id;];
+    //     let search_level_5 =
+    //         retrieve_node_mut_closure![Stride5; id; multi_uniq_id;];
+
+    //     if log_enabled!(log::Level::Trace) {
+    //         trace!(
+    //             "{} store: Retrieve node mut {} from l{}",
+    //             std::thread::current().name().unwrap_or("unnamed-thread"),
+    //             id,
+    //             id.get_id().1
+    //         );
+    //     }
+
+    //     match self.in_memory_tree.node_buckets.get_stride_for_id(id) {
+    //         3 => (search_level_3.f)(
+    //             &search_level_3,
+    //             self.in_memory_tree.node_buckets.get_store3(id),
+    //             0,
+    //         ),
+
+    //         4 => (search_level_4.f)(
+    //             &search_level_4,
+    //             self.in_memory_tree.node_buckets.get_store4(id),
+    //             0,
+    //         ),
+    //         _ => (search_level_5.f)(
+    //             &search_level_5,
+    //             self.in_memory_tree.node_buckets.get_store5(id),
+    //             0,
+    //         ),
+    //     }
+    // }
+
+    // pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
+    //     StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0)
+    // }
 
     pub fn get_nodes_count(&self) -> usize {
-        self.counters.get_nodes_count()
+        self.in_memory_tree.get_nodes_count()
     }
 
     // Prefixes related methods
@@ -673,44 +624,46 @@ impl<
         let mut prefix_is_new = true;
         let mut mui_is_new = true;
 
-        let (mui_count, cas_count) =
-            match self.non_recursive_retrieve_prefix_mut(prefix) {
-                // There's no StoredPrefix at this location yet. Create a new
-                // PrefixRecord and try to store it in the empty slot.
-                (locked_prefix, false) => {
-                    if log_enabled!(log::Level::Debug) {
-                        debug!(
-                            "{} store: Create new prefix record",
-                            std::thread::current()
-                                .name()
-                                .unwrap_or("unnamed-thread")
-                        );
-                    }
-
-                    let (mui_count, retry_count) =
-                        locked_prefix.record_map.upsert_record(
-                            prefix,
-                            record,
-                            &self.persist_tree,
-                            self.config.persist_strategy,
-                        )?;
-
-                    // See if someone beat us to creating the record.
-                    if mui_count.is_some() {
-                        mui_is_new = false;
-                        prefix_is_new = false;
-                    } else {
-                        // No, we were the first, we created a new prefix
-                        self.counters.inc_prefixes_count(prefix.get_len());
-                    }
+        let (mui_count, cas_count) = match self
+            .in_memory_tree
+            .non_recursive_retrieve_prefix_mut(prefix)
+        {
+            // There's no StoredPrefix at this location yet. Create a new
+            // PrefixRecord and try to store it in the empty slot.
+            (locked_prefix, false) => {
+                if log_enabled!(log::Level::Debug) {
+                    debug!(
+                        "{} store: Create new prefix record",
+                        std::thread::current()
+                            .name()
+                            .unwrap_or("unnamed-thread")
+                    );
+                }
 
-                    (mui_count, retry_count)
+                let (mui_count, retry_count) =
+                    locked_prefix.record_map.upsert_record(
+                        prefix,
+                        record,
+                        &self.persist_tree,
+                        self.config.persist_strategy,
+                    )?;
+
+                // See if someone beat us to creating the record.
+                if mui_count.is_some() {
+                    mui_is_new = false;
+                    prefix_is_new = false;
+                } else {
+                    // No, we were the first, we created a new prefix
+                    self.counters.inc_prefixes_count(prefix.get_len());
                 }
-                // There already is a StoredPrefix with a record at this
-                // location.
-                (stored_prefix, true) => {
-                    if log_enabled!(log::Level::Debug) {
-                        debug!(
+
+                (mui_count, retry_count)
+            }
+            // There already is a StoredPrefix with a record at this
+            // location.
+            (stored_prefix, true) => {
+                if log_enabled!(log::Level::Debug) {
+                    debug!(
                         "{} store: Found existing prefix record for {}/{}",
                         std::thread::current()
                             .name()
@@ -718,30 +671,30 @@ impl<
                         prefix.get_net(),
                         prefix.get_len()
                     );
-                    }
-                    prefix_is_new = false;
-
-                    // Update the already existing record_map with our
-                    // caller's record.
-                    stored_prefix.set_ps_outdated(guard)?;
-
-                    let (mui_count, retry_count) =
-                        stored_prefix.record_map.upsert_record(
-                            prefix,
-                            record,
-                            &self.persist_tree,
-                            self.config.persist_strategy,
-                        )?;
-                    mui_is_new = mui_count.is_none();
-
-                    if let Some(tbi) = update_path_selections {
-                        stored_prefix
-                            .calculate_and_store_best_backup(&tbi, guard)?;
-                    }
-
-                    (mui_count, retry_count)
                 }
-            };
+                prefix_is_new = false;
+
+                // Update the already existing record_map with our
+                // caller's record.
+                stored_prefix.set_ps_outdated(guard)?;
+
+                let (mui_count, retry_count) =
+                    stored_prefix.record_map.upsert_record(
+                        prefix,
+                        record,
+                        &self.persist_tree,
+                        self.config.persist_strategy,
+                    )?;
+                mui_is_new = mui_count.is_none();
+
+                if let Some(tbi) = update_path_selections {
+                    stored_prefix
+                        .calculate_and_store_best_backup(&tbi, guard)?;
+                }
+
+                (mui_count, retry_count)
+            }
+        };
 
         Ok(UpsertReport {
             prefix_new: prefix_is_new,
@@ -770,8 +723,9 @@ impl<
         prefix: PrefixId<AF>,
         mui: u32,
     ) -> Result<(), PrefixStoreError> {
-        let (stored_prefix, exists) =
-            self.non_recursive_retrieve_prefix_mut(prefix);
+        let (stored_prefix, exists) = self
+            .in_memory_tree
+            .non_recursive_retrieve_prefix_mut(prefix);
 
         if !exists {
             return Err(PrefixStoreError::StoreNotReadyError);
@@ -789,8 +743,9 @@ impl<
         prefix: PrefixId<AF>,
         mui: u32,
     ) -> Result<(), PrefixStoreError> {
-        let (stored_prefix, exists) =
-            self.non_recursive_retrieve_prefix_mut(prefix);
+        let (stored_prefix, exists) = self
+            .in_memory_tree
+            .non_recursive_retrieve_prefix_mut(prefix);
 
         if !exists {
             return Err(PrefixStoreError::StoreNotReadyError);
@@ -897,7 +852,7 @@ impl<
 
     // Helper to filter out records that are not-active (Inactive or
     // Withdrawn), or whose mui appears in the global withdrawn index.
-    pub fn get_filtered_records(
+    pub(crate) fn get_filtered_records(
         &self,
         pfx: &StoredPrefix<AF, M>,
         mui: Option<u32>,
@@ -917,190 +872,145 @@ impl<
     //
     // The error condition really shouldn't happen, because that basically
     // means the root node for that particular prefix length doesn't exist.
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn non_recursive_retrieve_prefix_mut(
-        &'a self,
-        search_prefix_id: PrefixId<AF>,
-    ) -> (&'a StoredPrefix<AF, M>, bool) {
-        trace!("non_recursive_retrieve_prefix_mut_with_guard");
-        let mut prefix_set = self
-            .in_memory_tree
-            .prefix_buckets
-            .get_root_prefix_set(search_prefix_id.get_len());
-        let mut level: u8 = 0;
-
-        trace!("root prefix_set {:?}", prefix_set);
-        loop {
-            // HASHING FUNCTION
-            let index = Self::hash_prefix_id(search_prefix_id, level);
-
-            // probe the slot with the index that's the result of the hashing.
-            // let locked_prefix = prefix_set.0.get(index);
-            let stored_prefix = match prefix_set.0.get(index) {
-                Some(p) => {
-                    trace!("prefix set found.");
-                    (p, true)
-                }
-                None => {
-                    // We're at the end of the chain and haven't found our
-                    // search_prefix_id anywhere. Return the end-of-the-chain
-                    // StoredPrefix, so the caller can attach a new one.
-                    trace!(
-                        "no record. returning last found record in level
-                        {}, with index {}.",
-                        level,
-                        index
-                    );
-                    let index = Self::hash_prefix_id(search_prefix_id, level);
-                    trace!("calculate next index {}", index);
-                    let var_name = (
-                        prefix_set
-                            .0
-                            .get_or_init(index, || {
-                                StoredPrefix::new::<PB>(
-                                    PrefixId::new(
-                                        search_prefix_id.get_net(),
-                                        search_prefix_id.get_len(),
-                                    ),
-                                    level,
-                                )
-                            })
-                            .0,
-                        false,
-                    );
-                    var_name
-                }
-            };
-
-            if search_prefix_id == stored_prefix.0.prefix {
-                // GOTCHA!
-                // Our search-prefix is stored here, so we're returning
-                // it, so its PrefixRecord can be updated by the caller.
-                trace!("found requested prefix {:?}", search_prefix_id);
-                return stored_prefix;
-            } else {
-                // A Collision. Follow the chain.
-                level += 1;
-                prefix_set = &stored_prefix.0.next_bucket;
-                continue;
-            }
-        }
-    }
-
-    // This function is used by the match_prefix, and [more|less]_specifics
-    // public methods on the TreeBitMap (indirectly).
-    #[allow(clippy::type_complexity)]
-    pub fn non_recursive_retrieve_prefix(
-        &'a self,
-        id: PrefixId<AF>,
-    ) -> (
-        Option<&'a StoredPrefix<AF, M>>,
-        Option<(
-            PrefixId<AF>,
-            u8,
-            &'a PrefixSet<AF, M>,
-            [Option<(&'a PrefixSet<AF, M>, usize)>; 32],
-            usize,
-        )>,
-    ) {
-        let mut prefix_set = self
-            .in_memory_tree
-            .prefix_buckets
-            .get_root_prefix_set(id.get_len());
-        let mut parents = [None; 32];
-        let mut level: u8 = 0;
-        let backoff = Backoff::new();
-
-        loop {
-            // The index of the prefix in this array (at this len and
-            // level) is calculated by performing the hash function
-            // over the prefix.
-
-            // HASHING FUNCTION
-            let index = Self::hash_prefix_id(id, level);
-
-            if let Some(stored_prefix) = prefix_set.0.get(index) {
-                if id == stored_prefix.get_prefix_id() {
-                    trace!("found requested prefix {:?}", id);
-                    parents[level as usize] = Some((prefix_set, index));
-                    return (
-                        Some(stored_prefix),
-                        Some((id, level, prefix_set, parents, index)),
-                    );
-                };
-                // Advance to the next level.
-
-                prefix_set = &stored_prefix.next_bucket;
-                level += 1;
-                backoff.spin();
-                continue;
-            }
-
-            trace!("no prefix found for {:?}", id);
-            parents[level as usize] = Some((prefix_set, index));
-            return (None, Some((id, level, prefix_set, parents, index)));
-        }
-    }
-
-    #[deprecated]
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_prefix(
-        &'a self,
-        prefix_id: PrefixId<AF>,
-    ) -> Option<(&'a StoredPrefix<AF, M>, usize)> {
-        struct SearchLevel<
-            's,
-            AF: AddressFamily,
-            M: crate::prefix_record::Meta,
-        > {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, M>,
-                &'a PrefixSet<AF, M>,
-                u8,
-            )
-                -> Option<(&'a StoredPrefix<AF, M>, usize)>,
-        }
-
-        let search_level = SearchLevel {
-            f: &|search_level: &SearchLevel<AF, M>,
-                 prefix_set: &PrefixSet<AF, M>,
-                 mut level: u8| {
-                // HASHING FUNCTION
-                let index = Self::hash_prefix_id(prefix_id, level);
-
-                if let Some(stored_prefix) = prefix_set.0.get(index) {
-                    if prefix_id == stored_prefix.prefix {
-                        trace!("found requested prefix {:?}", prefix_id,);
-                        return Some((stored_prefix, 0));
-                    };
-                    level += 1;
-
-                    (search_level.f)(
-                        search_level,
-                        &stored_prefix.next_bucket,
-                        level,
-                    );
-                }
-                None
-            },
-        };
+    // #[allow(clippy::type_complexity)]
+    // pub(crate) fn non_recursive_retrieve_prefix_mut(
+    //     &'a self,
+    //     search_prefix_id: PrefixId<AF>,
+    // ) -> (&'a StoredPrefix<AF, M>, bool) {
+    //     trace!("non_recursive_retrieve_prefix_mut_with_guard");
+    //     let mut prefix_set = self
+    //         .in_memory_tree
+    //         .prefix_buckets
+    //         .get_root_prefix_set(search_prefix_id.get_len());
+    //     let mut level: u8 = 0;
+
+    //     trace!("root prefix_set {:?}", prefix_set);
+    //     loop {
+    //         // HASHING FUNCTION
+    //         let index = TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(
+    //             search_prefix_id,
+    //             level,
+    //         );
+
+    //         // probe the slot with the index that's the result of the hashing.
+    //         // let locked_prefix = prefix_set.0.get(index);
+    //         let stored_prefix = match prefix_set.0.get(index) {
+    //             Some(p) => {
+    //                 trace!("prefix set found.");
+    //                 (p, true)
+    //             }
+    //             None => {
+    //                 // We're at the end of the chain and haven't found our
+    //                 // search_prefix_id anywhere. Return the end-of-the-chain
+    //                 // StoredPrefix, so the caller can attach a new one.
+    //                 trace!(
+    //                     "no record. returning last found record in level
+    //                     {}, with index {}.",
+    //                     level,
+    //                     index
+    //                 );
+    //                 let index = TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(
+    //                     search_prefix_id,
+    //                     level,
+    //                 );
+    //                 trace!("calculate next index {}", index);
+    //                 let var_name = (
+    //                     prefix_set
+    //                         .0
+    //                         .get_or_init(index, || {
+    //                             StoredPrefix::new::<PB>(
+    //                                 PrefixId::new(
+    //                                     search_prefix_id.get_net(),
+    //                                     search_prefix_id.get_len(),
+    //                                 ),
+    //                                 level,
+    //                             )
+    //                         })
+    //                         .0,
+    //                     false,
+    //                 );
+    //                 var_name
+    //             }
+    //         };
+
+    //         if search_prefix_id == stored_prefix.0.prefix {
+    //             // GOTCHA!
+    //             // Our search-prefix is stored here, so we're returning
+    //             // it, so its PrefixRecord can be updated by the caller.
+    //             trace!("found requested prefix {:?}", search_prefix_id);
+    //             return stored_prefix;
+    //         } else {
+    //             // A Collision. Follow the chain.
+    //             level += 1;
+    //             prefix_set = &stored_prefix.0.next_bucket;
+    //             continue;
+    //         }
+    //     }
+    // }
 
-        (search_level.f)(
-            &search_level,
-            self.in_memory_tree
-                .prefix_buckets
-                .get_root_prefix_set(prefix_id.get_len()),
-            0,
-        )
-    }
+    // // This function is used by the match_prefix, and [more|less]_specifics
+    // // public methods on the TreeBitMap (indirectly).
+    // #[allow(clippy::type_complexity)]
+    // pub fn non_recursive_retrieve_prefix(
+    //     &'a self,
+    //     id: PrefixId<AF>,
+    // ) -> (
+    //     Option<&'a StoredPrefix<AF, M>>,
+    //     Option<(
+    //         PrefixId<AF>,
+    //         u8,
+    //         &'a PrefixSet<AF, M>,
+    //         [Option<(&'a PrefixSet<AF, M>, usize)>; 32],
+    //         usize,
+    //     )>,
+    // ) {
+    //     let mut prefix_set = self
+    //         .in_memory_tree
+    //         .prefix_buckets
+    //         .get_root_prefix_set(id.get_len());
+    //     let mut parents = [None; 32];
+    //     let mut level: u8 = 0;
+    //     let backoff = Backoff::new();
+
+    //     loop {
+    //         // The index of the prefix in this array (at this len and
+    //         // level) is calculated by performing the hash function
+    //         // over the prefix.
+
+    //         // HASHING FUNCTION
+    //         let index =
+    //             TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(id, level);
+
+    //         if let Some(stored_prefix) = prefix_set.0.get(index) {
+    //             if id == stored_prefix.get_prefix_id() {
+    //                 trace!("found requested prefix {:?}", id);
+    //                 parents[level as usize] = Some((prefix_set, index));
+    //                 return (
+    //                     Some(stored_prefix),
+    //                     Some((id, level, prefix_set, parents, index)),
+    //                 );
+    //             };
+    //             // Advance to the next level.
+
+    //             prefix_set = &stored_prefix.next_bucket;
+    //             level += 1;
+    //             backoff.spin();
+    //             continue;
+    //         }
+
+    //         trace!("no prefix found for {:?}", id);
+    //         parents[level as usize] = Some((prefix_set, index));
+    //         return (None, Some((id, level, prefix_set, parents, index)));
+    //     }
+    // }
 
-    #[allow(dead_code)]
-    fn remove_prefix(&mut self, index: PrefixId<AF>) -> Option<M> {
-        match index.is_empty() {
-            false => self.in_memory_tree.prefix_buckets.remove(index),
-            true => None,
-        }
-    }
+    // #[allow(dead_code)]
+    // fn remove_prefix(&mut self, index: PrefixId<AF>) -> Option<M> {
+    //     match index.is_empty() {
+    //         false => self.in_memory_tree.prefix_buckets.remove(index),
+    //         true => None,
+    //     }
+    // }
 
     pub fn get_prefixes_count(&self) -> usize {
         self.counters.get_prefixes_count().iter().sum()
@@ -1112,10 +1022,11 @@ impl<
 
     // Stride related methods
 
-    pub(crate) fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8 {
-        self.in_memory_tree.node_buckets.get_stride_for_id(id)
-    }
+    // pub(crate) fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8 {
+    //     self.in_memory_tree.node_buckets.get_stride_for_id(id)
+    // }
 
+    // Pass through the in_memory stride sizes, for printing purposes
     pub fn get_stride_sizes(&self) -> &[u8] {
         self.in_memory_tree.node_buckets.get_stride_sizes()
     }
@@ -1124,43 +1035,43 @@ impl<
     //     NB::get_strides_len()
     // }
 
-    pub(crate) fn get_first_stride_size() -> u8 {
-        NB::get_first_stride_size()
-    }
+    // pub(crate) fn get_first_stride_size() -> u8 {
+    //     NB::get_first_stride_size()
+    // }
 
     // Calculates the id of the node that COULD host a prefix in its
     // ptrbitarr.
-    pub(crate) fn get_node_id_for_prefix(
-        &self,
-        prefix: &PrefixId<AF>,
-    ) -> (StrideNodeId<AF>, BitSpan) {
-        let mut acc = 0;
-        for i in self.get_stride_sizes() {
-            acc += *i;
-            if acc >= prefix.get_len() {
-                let node_len = acc - i;
-                return (
-                    StrideNodeId::new_with_cleaned_id(
-                        prefix.get_net(),
-                        node_len,
-                    ),
-                    // NOT THE HASHING FUNCTION!
-                    // Do the right shift in a checked manner, for the sake
-                    // of 0/0. A search for 0/0 will perform a 0 << MAX_LEN,
-                    // which will panic in debug mode (undefined behaviour
-                    // in prod).
-                    BitSpan::new(
-                        ((prefix.get_net() << node_len).checked_shr_or_zero(
-                            (AF::BITS - (prefix.get_len() - node_len)).into(),
-                        ))
-                        .dangerously_truncate_to_u32(),
-                        prefix.get_len() - node_len,
-                    ),
-                );
-            }
-        }
-        panic!("prefix length for {:?} is too long", prefix);
-    }
+    // pub(crate) fn get_node_id_for_prefix(
+    //     &self,
+    //     prefix: &PrefixId<AF>,
+    // ) -> (StrideNodeId<AF>, BitSpan) {
+    //     let mut acc = 0;
+    //     for i in self.get_stride_sizes() {
+    //         acc += *i;
+    //         if acc >= prefix.get_len() {
+    //             let node_len = acc - i;
+    //             return (
+    //                 StrideNodeId::new_with_cleaned_id(
+    //                     prefix.get_net(),
+    //                     node_len,
+    //                 ),
+    //                 // NOT THE HASHING FUNCTION!
+    //                 // Do the right shift in a checked manner, for the sake
+    //                 // of 0/0. A search for 0/0 will perform a 0 << MAX_LEN,
+    //                 // which will panic in debug mode (undefined behaviour
+    //                 // in prod).
+    //                 BitSpan::new(
+    //                     ((prefix.get_net() << node_len).checked_shr_or_zero(
+    //                         (AF::BITS - (prefix.get_len() - node_len)).into(),
+    //                     ))
+    //                     .dangerously_truncate_to_u32(),
+    //                     prefix.get_len() - node_len,
+    //                 ),
+    //             );
+    //         }
+    //     }
+    //     panic!("prefix length for {:?} is too long", prefix);
+    // }
 
     //-------- Persistence ---------------------------------------------------
 
@@ -1289,49 +1200,75 @@ impl<
     // element, where each element in the list has an array of its own that
     // uses the hash function with the level incremented.
 
-    pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
-        // And, this is all of our hashing function.
-        let last_level = if level > 0 {
-            <NB>::len_to_store_bits(id.get_id().1, level - 1)
-        } else {
-            0
-        };
-        let this_level = <NB>::len_to_store_bits(id.get_id().1, level);
-        trace!("bits division {}", this_level);
-        trace!(
-            "calculated index ({} << {}) >> {}",
-            id.get_id().0,
-            last_level,
-            ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-        );
-        // HASHING FUNCTION
-        ((id.get_id().0 << last_level)
-            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
-            .dangerously_truncate_to_u32() as usize
+    // pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
+    //     // And, this is all of our hashing function.
+    //     let last_level = if level > 0 {
+    //         <NB>::len_to_store_bits(id.get_id().1, level - 1)
+    //     } else {
+    //         0
+    //     };
+    //     let this_level = <NB>::len_to_store_bits(id.get_id().1, level);
+    //     trace!("bits division {}", this_level);
+    //     trace!(
+    //         "calculated index ({} << {}) >> {}",
+    //         id.get_id().0,
+    //         last_level,
+    //         ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
+    //     );
+    //     // HASHING FUNCTION
+    //     ((id.get_id().0 << last_level)
+    //         >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
+    //         .dangerously_truncate_to_u32() as usize
+    // }
+
+    // pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
+    //     // And, this is all of our hashing function.
+    //     let last_level = if level > 0 {
+    //         <PB>::get_bits_for_len(id.get_len(), level - 1)
+    //     } else {
+    //         0
+    //     };
+    //     let this_level = <PB>::get_bits_for_len(id.get_len(), level);
+    //     trace!(
+    //         "bits division {}; no of bits {}",
+    //         this_level,
+    //         this_level - last_level
+    //     );
+    //     trace!(
+    //         "calculated index ({} << {}) >> {}",
+    //         id.get_net(),
+    //         last_level,
+    //         ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
+    //     );
+    //     // HASHING FUNCTION
+    //     ((id.get_net() << last_level)
+    //         >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
+    //         .dangerously_truncate_to_u32() as usize
+    // }
+}
+
+impl<
+        M: Meta,
+        NB: NodeBuckets<IPv4>,
+        PB: PrefixBuckets<IPv4, M>,
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > std::fmt::Display for Rib<IPv4, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "Rib<IPv4, {}>", std::any::type_name::<M>())
     }
+}
 
-    pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
-        // And, this is all of our hashing function.
-        let last_level = if level > 0 {
-            <PB>::get_bits_for_len(id.get_len(), level - 1)
-        } else {
-            0
-        };
-        let this_level = <PB>::get_bits_for_len(id.get_len(), level);
-        trace!(
-            "bits division {}; no of bits {}",
-            this_level,
-            this_level - last_level
-        );
-        trace!(
-            "calculated index ({} << {}) >> {}",
-            id.get_net(),
-            last_level,
-            ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-        );
-        // HASHING FUNCTION
-        ((id.get_net() << last_level)
-            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
-            .dangerously_truncate_to_u32() as usize
+impl<
+        M: Meta,
+        NB: NodeBuckets<IPv6>,
+        PB: PrefixBuckets<IPv6, M>,
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > std::fmt::Display for Rib<IPv6, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "Rib<IPv6, {}>", std::any::type_name::<M>())
     }
 }
diff --git a/src/local_array/store/atomic_types.rs b/src/local_array/store/atomic_types.rs
deleted file mode 100644
index e0e1ff40..00000000
--- a/src/local_array/store/atomic_types.rs
+++ /dev/null
@@ -1,760 +0,0 @@
-use std::collections::HashMap;
-use std::sync::{Arc, Mutex, MutexGuard, RwLock};
-use std::{
-    fmt::{Debug, Display},
-    sync::atomic::Ordering,
-};
-
-use crossbeam_epoch::{self as epoch, Atomic};
-
-use crossbeam_utils::Backoff;
-use log::{debug, log_enabled, trace};
-
-use epoch::{Guard, Owned};
-use roaring::RoaringBitmap;
-
-use crate::custom_alloc::{PersistStrategy, PersistTree};
-use crate::local_array::tree::*;
-use crate::prefix_record::PublicRecord;
-use crate::prelude::Meta;
-use crate::AddressFamily;
-
-use super::errors::PrefixStoreError;
-use super::oncebox::OnceBoxSlice;
-
-// ----------- Node related structs -----------------------------------------
-
-#[derive(Debug)]
-pub struct StoredNode<AF, S>
-where
-    Self: Sized,
-    S: Stride,
-    AF: AddressFamily,
-{
-    pub(crate) node_id: StrideNodeId<AF>,
-    // The ptrbitarr and pfxbitarr for this node
-    pub(crate) node: TreeBitMapNode<AF, S>,
-    // Child nodes linked from this node
-    pub(crate) node_set: NodeSet<AF, S>,
-}
-
-#[allow(clippy::type_complexity)]
-#[derive(Debug)]
-pub struct NodeSet<AF: AddressFamily, S: Stride>(
-    pub OnceBoxSlice<StoredNode<AF, S>>,
-    // A Bitmap index that keeps track of the `multi_uniq_id`s (mui) that are
-    // present in value collections in the meta-data tree in the child nodes
-    pub RwLock<RoaringBitmap>,
-);
-
-impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
-<<<<<<< Updated upstream
-    pub fn init(p2_size: u8) -> Self {
-        if log_enabled!(log::Level::Debug) {
-            debug!(
-                "{} store: creating space for {} nodes",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                1 << p2_size
-            );
-        }
-
-        NodeSet(OnceBoxSlice::new(p2_size), RoaringBitmap::new().into())
-=======
-    pub fn init(pow2_size: u8) -> Self {
-        if log_enabled!(log::Level::Debug) {
-            debug!(
-                "{} store: creating space for {} nodes",
-                std::thread::current().name().unwrap(),
-                1 << pow2_size
-            );
-        }
-
-        // let mut l = vec![];
-        // for _i in 0..size {
-        //     l.push(OnceBox::new());
-        // }
-        NodeSet(OnceBoxSlice::new(pow2_size), RoaringBitmap::new().into())
->>>>>>> Stashed changes
-    }
-
-    pub fn update_rbm_index(
-        &self,
-        multi_uniq_id: u32,
-    ) -> Result<u32, crate::prelude::multi::PrefixStoreError>
-    where
-        S: crate::local_array::atomic_stride::Stride,
-        AF: crate::AddressFamily,
-    {
-        let try_count = 0;
-        let mut rbm = self.1.write().unwrap();
-        rbm.insert(multi_uniq_id);
-
-        Ok(try_count)
-    }
-
-    pub fn remove_from_rbm_index(
-        &self,
-        multi_uniq_id: u32,
-        _guard: &crate::epoch::Guard,
-    ) -> Result<u32, crate::prelude::multi::PrefixStoreError>
-    where
-        S: crate::local_array::atomic_stride::Stride,
-        AF: crate::AddressFamily,
-    {
-        let try_count = 0;
-
-        let mut rbm = self.1.write().unwrap();
-        rbm.remove(multi_uniq_id);
-
-        Ok(try_count)
-    }
-}
-
-// ----------- Prefix related structs ---------------------------------------
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub struct PathSelections {
-    pub(crate) path_selection_muis: (Option<u32>, Option<u32>),
-}
-
-impl PathSelections {
-    pub fn best(&self) -> Option<u32> {
-        self.path_selection_muis.0
-    }
-
-    pub fn backup(&self) -> Option<u32> {
-        self.path_selection_muis.1
-    }
-}
-
-// ----------- StoredPrefix -------------------------------------------------
-// This is the top-level struct that's linked from the slots in the buckets.
-// It contains a super_agg_record that is supposed to hold counters for the
-// records that are stored inside it, so that iterators over its linked lists
-// don't have to go into them if there's nothing there and could stop early.
-#[derive(Debug)]
-pub struct StoredPrefix<AF: AddressFamily, M: crate::prefix_record::Meta> {
-    // the serial number
-    // pub serial: usize,
-    // the prefix itself,
-    pub prefix: PrefixId<AF>,
-    // the aggregated data for this prefix
-    pub record_map: MultiMap<M>,
-    // (mui of best path entry, mui of backup path entry) from the record_map
-    path_selections: Atomic<PathSelections>,
-    // the reference to the next set of records for this prefix, if any.
-    pub next_bucket: PrefixSet<AF, M>,
-}
-
-impl<AF: AddressFamily, M: crate::prefix_record::Meta> StoredPrefix<AF, M> {
-    pub(crate) fn new<PB: PrefixBuckets<AF, M>>(
-        pfx_id: PrefixId<AF>,
-        level: u8,
-    ) -> Self {
-        // start calculation size of next set, it's dependent on the level
-        // we're in.
-        // let pfx_id = PrefixId::new(record.net, record.len);
-        let this_level = PB::get_bits_for_len(pfx_id.get_len(), level);
-        let next_level = PB::get_bits_for_len(pfx_id.get_len(), level + 1);
-
-        trace!("this level {} next level {}", this_level, next_level);
-        let next_bucket: PrefixSet<AF, M> = if next_level > 0 {
-            debug!(
-                "{} store: INSERT with new bucket of size {} at prefix len {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                1 << (next_level - this_level),
-                pfx_id.get_len()
-            );
-            PrefixSet::init(next_level - this_level)
-        } else {
-            debug!(
-                "{} store: INSERT at LAST LEVEL with empty bucket at prefix len {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                pfx_id.get_len()
-            );
-<<<<<<< Updated upstream
-            PrefixSet::init(next_level - this_level)
-=======
-            PrefixSet::empty(next_level - this_level)
->>>>>>> Stashed changes
-        };
-        // End of calculation
-
-        let rec_map = HashMap::new();
-
-        StoredPrefix {
-            // serial: 1,
-            prefix: pfx_id,
-            path_selections: Atomic::init(PathSelections {
-                path_selection_muis: (None, None),
-            }),
-            record_map: MultiMap::new(rec_map),
-            next_bucket,
-        }
-    }
-
-    pub(crate) fn get_prefix_id(&self) -> PrefixId<AF> {
-        self.prefix
-    }
-
-    pub fn get_path_selections(&self, guard: &Guard) -> PathSelections {
-        let path_selections =
-            self.path_selections.load(Ordering::Acquire, guard);
-
-        unsafe { path_selections.as_ref() }.map_or(
-            PathSelections {
-                path_selection_muis: (None, None),
-            },
-            |ps| *ps,
-        )
-    }
-
-    pub(crate) fn set_path_selections(
-        &self,
-        path_selections: PathSelections,
-        guard: &Guard,
-    ) -> Result<(), PrefixStoreError> {
-        let current = self.path_selections.load(Ordering::SeqCst, guard);
-
-        if unsafe { current.as_ref() } == Some(&path_selections) {
-            debug!("unchanged path_selections");
-            return Ok(());
-        }
-
-        self.path_selections
-            .compare_exchange(
-                current,
-                // Set the tag to indicate we're updated
-                Owned::new(path_selections).with_tag(0),
-                Ordering::AcqRel,
-                Ordering::Acquire,
-                guard,
-            )
-            .map_err(|_| PrefixStoreError::PathSelectionOutdated)?;
-        Ok(())
-    }
-
-    pub fn set_ps_outdated(
-        &self,
-        guard: &Guard,
-    ) -> Result<(), PrefixStoreError> {
-        self.path_selections
-            .fetch_update(Ordering::Acquire, Ordering::Acquire, guard, |p| {
-                Some(p.with_tag(1))
-            })
-            .map(|_| ())
-            .map_err(|_| PrefixStoreError::StoreNotReadyError)
-    }
-
-    pub fn is_ps_outdated(&self, guard: &Guard) -> bool {
-        self.path_selections.load(Ordering::Acquire, guard).tag() == 1
-    }
-
-    pub fn calculate_and_store_best_backup<'a>(
-        &'a self,
-        tbi: &M::TBI,
-        guard: &'a Guard,
-    ) -> Result<(Option<u32>, Option<u32>), super::errors::PrefixStoreError>
-    {
-        let path_selection_muis = self.record_map.best_backup(*tbi);
-
-        self.set_path_selections(
-            PathSelections {
-                path_selection_muis,
-            },
-            guard,
-        )?;
-
-        Ok(path_selection_muis)
-    }
-
-    pub(crate) fn get_next_bucket(&self) -> Option<&PrefixSet<AF, M>> {
-        if self.next_bucket.is_empty() {
-            None
-        } else {
-            Some(&self.next_bucket)
-        }
-    }
-}
-
-#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
-pub enum RouteStatus {
-    Active,
-    InActive,
-    Withdrawn,
-}
-
-impl std::fmt::Display for RouteStatus {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            RouteStatus::Active => write!(f, "active"),
-            RouteStatus::InActive => write!(f, "inactive"),
-            RouteStatus::Withdrawn => write!(f, "withdrawn"),
-        }
-    }
-}
-
-impl From<RouteStatus> for u8 {
-    fn from(value: RouteStatus) -> Self {
-        match value {
-            RouteStatus::Active => 1,
-            RouteStatus::InActive => 2,
-            RouteStatus::Withdrawn => 3,
-        }
-    }
-}
-
-impl TryFrom<u8> for RouteStatus {
-    type Error = PrefixStoreError;
-
-    fn try_from(value: u8) -> Result<Self, Self::Error> {
-        match value {
-            1 => Ok(RouteStatus::Active),
-            2 => Ok(RouteStatus::InActive),
-            3 => Ok(RouteStatus::Withdrawn),
-            _ => Err(PrefixStoreError::StoreNotReadyError),
-        }
-    }
-}
-
-#[derive(Copy, Clone, Debug)]
-pub(crate) struct PersistStatus(bool);
-
-impl PersistStatus {
-    pub(crate) fn persisted() -> Self {
-        Self(true)
-    }
-
-    pub(crate) fn not_persisted() -> Self {
-        Self(false)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub(crate) struct MultiMapValue<M> {
-    meta: M,
-    ltime: u64,
-    route_status: RouteStatus,
-    persist_status: PersistStatus,
-}
-
-impl<M: Meta> MultiMapValue<M> {
-    pub(crate) fn logical_time(&self) -> u64 {
-        self.ltime
-    }
-
-    pub(crate) fn meta(&self) -> &M {
-        &self.meta
-    }
-
-    pub(crate) fn route_status(&self) -> RouteStatus {
-        self.route_status
-    }
-
-    pub(crate) fn set_route_status(&mut self, status: RouteStatus) {
-        self.route_status = status;
-    }
-
-    pub(crate) fn has_persisted_data(&self) -> bool {
-        self.persist_status.0
-    }
-}
-
-impl<M: Meta> std::fmt::Display for MultiMapValue<M> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{} {}",
-            // self.meta(),
-            self.logical_time(),
-            self.route_status()
-        )
-    }
-}
-
-impl<M: Meta> From<(PublicRecord<M>, PersistStatus)> for MultiMapValue<M> {
-    fn from(value: (PublicRecord<M>, PersistStatus)) -> Self {
-        Self {
-            ltime: value.0.ltime,
-            route_status: value.0.status,
-            meta: value.0.meta,
-            persist_status: value.1,
-        }
-    }
-}
-
-impl<M: Meta> From<(u32, &MultiMapValue<M>)> for PublicRecord<M> {
-    fn from(value: (u32, &MultiMapValue<M>)) -> Self {
-        Self {
-            multi_uniq_id: value.0,
-            meta: value.1.meta().clone(),
-            ltime: value.1.ltime,
-            status: value.1.route_status,
-        }
-    }
-}
-
-// ----------- MultiMap ------------------------------------------------------
-// This is the record that holds the aggregates at the top-level for a given
-// prefix.
-
-#[derive(Debug)]
-pub struct MultiMap<M: Meta>(
-    Arc<Mutex<std::collections::HashMap<u32, MultiMapValue<M>>>>,
-);
-
-impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
-    pub(crate) fn new(record_map: HashMap<u32, MultiMapValue<M>>) -> Self {
-        Self(Arc::new(Mutex::new(record_map)))
-    }
-
-    fn guard_with_retry(
-        &self,
-        mut retry_count: usize,
-    ) -> (MutexGuard<HashMap<u32, MultiMapValue<M>>>, usize) {
-        let backoff = Backoff::new();
-
-        loop {
-            if let Ok(guard) = self.0.try_lock() {
-                return (guard, retry_count);
-            }
-
-            backoff.spin();
-            retry_count += 1;
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        record_map.len()
-    }
-
-    pub fn get_record_for_active_mui(
-        &self,
-        mui: u32,
-    ) -> Option<PublicRecord<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-
-        record_map.get(&mui).and_then(|r| {
-            if r.route_status() == RouteStatus::Active {
-                Some(PublicRecord::from((mui, r)))
-            } else {
-                None
-            }
-        })
-    }
-
-    pub fn best_backup(&self, tbi: M::TBI) -> (Option<u32>, Option<u32>) {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        let ord_routes = record_map
-            .iter()
-            .map(|r| (r.1.meta().as_orderable(tbi), *r.0));
-        let (best, bckup) =
-            routecore::bgp::path_selection::best_backup_generic(ord_routes);
-        (best.map(|b| b.1), bckup.map(|b| b.1))
-    }
-
-    pub(crate) fn get_record_for_mui_with_rewritten_status(
-        &self,
-        mui: u32,
-        bmin: &RoaringBitmap,
-        rewrite_status: RouteStatus,
-    ) -> Option<PublicRecord<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        record_map.get(&mui).map(|r| {
-            // We'll return a cloned record: the record in the store remains
-            // untouched.
-            let mut r = r.clone();
-            if bmin.contains(mui) {
-                r.set_route_status(rewrite_status);
-            }
-            PublicRecord::from((mui, &r))
-        })
-    }
-
-    // Helper to filter out records that are not-active (Inactive or
-    // Withdrawn), or whose mui appears in the global withdrawn index.
-    pub fn get_filtered_records(
-        &self,
-        mui: Option<u32>,
-        bmin: &RoaringBitmap,
-    ) -> Vec<PublicRecord<M>> {
-        if let Some(mui) = mui {
-            self.get_record_for_active_mui(mui).into_iter().collect()
-        } else {
-            self.as_active_records_not_in_bmin(bmin)
-        }
-    }
-
-    // return all records regardless of their local status, or any globally
-    // set status for the mui of the record. However, the local status for a
-    // record whose mui appears in the specified bitmap index, will be
-    // rewritten with the specified RouteStatus.
-    pub fn as_records_with_rewritten_status(
-        &self,
-        bmin: &RoaringBitmap,
-        rewrite_status: RouteStatus,
-    ) -> Vec<PublicRecord<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        record_map
-            .iter()
-            .map(move |r| {
-                let mut rec = r.1.clone();
-                if bmin.contains(*r.0) {
-                    rec.set_route_status(rewrite_status);
-                }
-                PublicRecord::from((*r.0, &rec))
-            })
-            .collect::<Vec<_>>()
-    }
-
-    pub fn as_records(&self) -> Vec<PublicRecord<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        record_map
-            .iter()
-            .map(|r| PublicRecord::from((*r.0, r.1)))
-            .collect::<Vec<_>>()
-    }
-
-    // Returns a vec of records whose keys are not in the supplied bitmap
-    // index, and whose local Status is set to Active. Used to filter out
-    // withdrawn routes.
-    pub fn as_active_records_not_in_bmin(
-        &self,
-        bmin: &RoaringBitmap,
-    ) -> Vec<PublicRecord<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        record_map
-            .iter()
-            .filter_map(|r| {
-                if r.1.route_status() == RouteStatus::Active
-                    && !bmin.contains(*r.0)
-                {
-                    Some(PublicRecord::from((*r.0, r.1)))
-                } else {
-                    None
-                }
-            })
-            .collect::<Vec<_>>()
-    }
-
-    // Change the local status of the record for this mui to Withdrawn.
-    pub fn mark_as_withdrawn_for_mui(&self, mui: u32) {
-        let c_map = Arc::clone(&self.0);
-        let mut record_map = c_map.lock().unwrap();
-        if let Some(rec) = record_map.get_mut(&mui) {
-            rec.set_route_status(RouteStatus::Withdrawn);
-        }
-    }
-
-    // Change the local status of the record for this mui to Active.
-    pub fn mark_as_active_for_mui(&self, mui: u32) {
-        let record_map = Arc::clone(&self.0);
-        let mut r_map = record_map.lock().unwrap();
-        if let Some(rec) = r_map.get_mut(&mui) {
-            rec.set_route_status(RouteStatus::Active);
-        }
-    }
-
-    // Insert or replace the PublicRecord in the HashMap for the key of
-    // record.multi_uniq_id. Returns the number of entries in the HashMap
-    // after updating it, if it's more than 1. Returns None if this is the
-    // first entry.
-    pub(crate) fn upsert_record<
-        AF: AddressFamily,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    >(
-        &self,
-        prefix: PrefixId<AF>,
-        new_rec: PublicRecord<M>,
-        persistence: &Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
-        strategy: PersistStrategy,
-    ) -> Result<(Option<usize>, usize), PrefixStoreError> {
-        let (mut record_map, retry_count) = self.guard_with_retry(0);
-        let key = new_rec.multi_uniq_id;
-
-        match (strategy, record_map.get_mut(&key)) {
-            // New record for (prefix, mui) in memory.
-
-            // We store the record in memory only.
-            (
-                PersistStrategy::PersistHistory | PersistStrategy::MemoryOnly,
-                None,
-            ) => {
-                record_map.insert(
-                    key,
-                    MultiMapValue::from((
-                        new_rec,
-                        PersistStatus::not_persisted(),
-                    )),
-                );
-
-                Ok((None, retry_count))
-            }
-            // We only persist the record.
-            (PersistStrategy::PersistOnly, None) => {
-                if let Some(persistence) = persistence {
-                    persistence.persist_record(prefix, key, &new_rec);
-                    Ok((None, retry_count))
-                } else {
-                    Err(PrefixStoreError::PersistFailed)
-                }
-            }
-            // We store both in memory and persist it.
-            (PersistStrategy::WriteAhead, None) => {
-                if let Some(persistence) = persistence {
-                    persistence.persist_record(prefix, key, &new_rec);
-                    let mmv = MultiMapValue::from((
-                        new_rec,
-                        PersistStatus::persisted(),
-                    ));
-                    record_map.insert(key, mmv);
-
-                    Ok((None, retry_count))
-                } else {
-                    Err(PrefixStoreError::PersistFailed)
-                }
-            }
-
-            // Existing record for (prefix, mui) in memory.
-
-            // We store the record in memory only, and discard the old record.
-            (PersistStrategy::MemoryOnly, Some(exist_rec)) => {
-                *exist_rec = MultiMapValue::from((
-                    new_rec,
-                    PersistStatus::not_persisted(),
-                ));
-
-                Ok((Some(record_map.len()), retry_count))
-            }
-            // We only persist record, so how come there's one in memory?
-            // Should not happen.
-            (PersistStrategy::PersistOnly, Some(_)) => {
-                panic!("Encountered illegally stored record");
-            }
-            // We store the new record in memory and persist the old record.
-            (
-                PersistStrategy::PersistHistory | PersistStrategy::WriteAhead,
-                Some(exist_rec),
-            ) => {
-                if let Some(persistence) = persistence {
-                    persistence.persist_record(prefix, key, &new_rec);
-                    *exist_rec = MultiMapValue::from((
-                        new_rec,
-                        PersistStatus::persisted(),
-                    ));
-
-                    Ok((Some(record_map.len()), retry_count))
-                } else {
-                    Err(PrefixStoreError::PersistFailed)
-                }
-            }
-        }
-    }
-}
-
-impl<M: Meta> Clone for MultiMap<M> {
-    fn clone(&self) -> Self {
-        Self(Arc::clone(&self.0))
-    }
-}
-
-// ----------- FamilyBuckets Trait ------------------------------------------
-//
-// Implementations of this trait are done by a proc-macro called
-// `stride_sizes`from the `rotonda-macros` crate.
-
-pub trait NodeBuckets<AF: AddressFamily> {
-    fn init() -> Self;
-    fn len_to_store_bits(len: u8, level: u8) -> u8;
-    fn get_stride_sizes(&self) -> &[u8];
-    fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8;
-    fn get_store3(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride3>;
-    fn get_store4(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride4>;
-    fn get_store5(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride5>;
-    fn get_strides_len() -> u8;
-    fn get_first_stride_size() -> u8;
-}
-
-pub trait PrefixBuckets<AF: AddressFamily, M: Meta>
-where
-    Self: Sized,
-{
-    fn init() -> Self;
-    fn remove(&mut self, id: PrefixId<AF>) -> Option<M>;
-    fn get_root_prefix_set(&self, len: u8) -> &'_ PrefixSet<AF, M>;
-    fn get_bits_for_len(len: u8, level: u8) -> u8;
-}
-
-//------------ PrefixSet ----------------------------------------------------
-
-// The PrefixSet is the ARRAY that holds all the child prefixes in a node.
-// Since we are storing these prefixes in the global store in a HashMap that
-// is keyed on the tuple (addr_bits, len, serial number) we can get away with
-// storing ONLY THE SERIAL NUMBER in the pfx_vec: The addr_bits and len are
-// implied in the position in the array a serial number has. A PrefixSet
-// doesn't know anything about the node it is contained in, so it needs a base
-// address to be able to calculate the complete prefix of a child prefix.
-
-#[derive(Debug)]
-#[repr(align(8))]
-pub struct PrefixSet<AF: AddressFamily, M: Meta>(
-    pub OnceBoxSlice<StoredPrefix<AF, M>>,
-);
-
-impl<AF: AddressFamily, M: Meta> PrefixSet<AF, M> {
-<<<<<<< Updated upstream
-    pub fn init(p2_size: u8) -> Self {
-        PrefixSet(OnceBoxSlice::new(p2_size))
-=======
-    pub fn init(pow2_size: u8) -> Self {
-        // let mut l = Vec::with_capacity(size);
-
-        // let l = AtomicStoredPrefix<AF, M>>>::new(vec![
-        // AtomicStoredPrefix::empty(),
-        // ]);
-        trace!("creating space for {} prefixes in prefix_set", pow2_size);
-        // for _i in 0..size {
-        //     l.push(OnceBox::new());
-        // }
-        PrefixSet(OnceBoxSlice::new(pow2_size))
->>>>>>> Stashed changes
-    }
-
-    pub(crate) fn is_empty(&self) -> bool {
-<<<<<<< Updated upstream
-=======
-        // if self.0.len() == 1 {
-        //     true
-        // } else {
-        //     false
-        // }
->>>>>>> Stashed changes
-        self.0.is_null()
-    }
-
-    pub(crate) fn get_by_index(
-        &self,
-        index: usize,
-<<<<<<< Updated upstream
-    ) -> Option<&StoredPrefix<AF, M>> {
-        self.0.get(index)
-=======
-        _guard: &'a Guard,
-    ) -> Option<&'a StoredPrefix<AF, M>> {
-        self.0.get(index)
-    }
-
-    pub(crate) fn empty(pow2_size: u8) -> Self {
-        PrefixSet(OnceBoxSlice::new(pow2_size))
->>>>>>> Stashed changes
-    }
-}
diff --git a/src/local_array/store/custom_alloc.rs b/src/local_array/store/custom_alloc.rs
deleted file mode 100644
index 74375400..00000000
--- a/src/local_array/store/custom_alloc.rs
+++ /dev/null
@@ -1,1553 +0,0 @@
-// ----------- THE STORE ----------------------------------------------------
-//
-// The CustomAllocStore provides in-memory storage for the BitTreeMapNodes
-// and for prefixes and their meta-data. The storage for node is on the
-// `buckets` field, and the prefixes are stored in, well, the `prefixes`
-// field. They are both organised in the same way, as chained hash tables,
-// one per (prefix|node)-length. The hashing function (that is detailed
-// lower down in this file), basically takes the address part of the
-// node|prefix and uses `(node|prefix)-address part % bucket size`
-// as its index.
-//
-// Both the prefixes and the buckets field have one bucket per (prefix|node)
-// -length that start out with a fixed-size array. The size of the arrays is
-// set in the rotonda_macros/maps.rs file.
-//
-// For lower (prefix|node)-lengths the number of elements in the array is
-// equal to the number of prefixes in that length, so there's exactly one
-// element per (prefix|node). For greater lengths there will be collisions,
-// in that case the stored (prefix|node) will have a reference to another
-// bucket (also of a fixed size), that holds a (prefix|node) that collided
-// with the one that was already stored. A (node|prefix) lookup will have to
-// go over all (node|prefix) buckets until it matches the requested (node|
-// prefix) or it reaches the end of the chain.
-//
-// The chained (node|prefixes) are occupied at a first-come, first-serve
-// basis, and are not re-ordered on new insertions of (node|prefixes). This
-// may change in the future, since it prevents iterators from being ordered.
-//
-// One of the nice things of having one table per (node|prefix)-length is that
-// a search can start directly at the prefix-length table it wishes, and go
-// go up and down into other tables if it needs to (e.g., because more- or
-// less-specifics were asked for). In contrast if you do a lookup by
-// traversing the tree of nodes, we would always have to go through the root-
-// node first and then go up the tree to the requested node. The lower nodes
-// of the tree (close to the root) would be a formidable bottle-neck then.
-//
-// Currently, the meta-data is an atomically stored value, that is required to
-// implement the `Meta` and the `Clone` trait. New meta-data
-// instances are stored atomically without further ado, but updates to a
-// piece of meta-data are done by merging the previous meta-data with the new
-// meta-data, through use of the `MergeUpdate` trait.
-//
-// The `upsert_prefix` methods retrieve only the most recent insert
-// for a prefix (for now).
-//
-// Future work could have a user-configurable retention strategy that allows
-// the meta-data to be stored as a linked-list of references, where each
-// meta-data object has a reference to its predecessor.
-//
-// Prefix example
-//
-//         (level 0 arrays)         prefixes  bucket
-//                                    /len     size
-//         ┌──┐
-// len /0  │ 0│                        1        1     ■
-//         └──┘                                       │
-//         ┌──┬──┐                                    │
-// len /1  │00│01│                     2        2     │
-//         └──┴──┘                                 perfect
-//         ┌──┬──┬──┬──┐                             hash
-// len /2  │  │  │  │  │               4        4     │
-//         └──┴──┴──┴──┘                              │
-//         ┌──┬──┬──┬──┬──┬──┬──┬──┐                  │
-// len /3  │  │  │  │  │  │  │  │  │   8        8     ■
-//         └──┴──┴──┴──┴──┴──┴──┴──┘
-//         ┌──┬──┬──┬──┬──┬──┬──┬──┐                        ┌────────────┐
-// len /4  │  │  │  │  │  │  │  │  │   8        16 ◀────────│ collision  │
-//         └──┴──┴──┴┬─┴──┴──┴──┴──┘                        └────────────┘
-//                   └───┐
-//                       │              ┌─collision─────────┐
-//                   ┌───▼───┐          │                   │
-//                   │       │ ◀────────│ 0x0100 and 0x0101 │
-//                   │ 0x010 │          └───────────────────┘
-//                   │       │
-//                   ├───────┴──────────────┬──┬──┐
-//                   │ StoredPrefix 0x0101  │  │  │
-//                   └──────────────────────┴─┬┴─┬┘
-//                                            │  │
-//                       ┌────────────────────┘  └──┐
-//            ┌──────────▼──────────┬──┐          ┌─▼┬──┐
-//         ┌─▶│ metadata (current)  │  │          │ 0│ 1│ (level 1 array)
-//         │  └─────────────────────┴──┘          └──┴──┘
-//    merge└─┐                        │             │
-//    update │           ┌────────────┘             │
-//           │┌──────────▼──────────┬──┐        ┌───▼───┐
-//         ┌─▶│ metadata (previous) │  │        │       │
-//         │  └─────────────────────┴──┘        │  0x0  │
-//    merge└─┐                        │         │       │
-//    update │           ┌────────────┘         ├───────┴──────────────┬──┐
-//           │┌──────────▼──────────┬──┐        │ StoredPrefix 0x0110  │  │
-//            │ metadata (oldest)   │  │        └──────────────────────┴──┘
-//            └─────────────────────┴──┘                                 │
-//                                                         ┌─────────────┘
-//                                              ┌──────────▼──────────────┐
-//                                              │ metadata (current)      │
-//                                              └─────────────────────────┘
-
-// Note about the memory usage of the data-structures of the Buckets
-//
-// As said, the prefixes and nodes are stored in buckets. A bucket right now
-// is of type `[MaybeUnit<Atomic<StoredPrefix>>]`, this has the advantage
-// that the length can be variable, based on the stride size for that level.
-// It saves us to have to implement a generic something.
-// Another advantage is the fixed place in which an atomic StoredPrefix
-// lives: this makes compare-and-swapping it relatively straight forward.
-// Each accessing thread would try to read/write the exact same entry in the
-// array, so shouldn't be any 'rug pulling' on the whole array.
-//
-// A disadvantage is that this is a fixed size, sparse array the moment it
-// is created. Theoretically, an `Atomic<Vec<StoredPrefix>`
-// would not have this disadvantage. Manipulating the whole vec atomically
-// though is very tricky (we would have to atomically compare-and-swap the
-// whole vec each time the prefix meta-data is changed) and inefficient,
-// since we would have to either keep the vec sorted on `PrefixId` at all
-// times, or, we would have to inspect each value in the vec on *every* read
-// or write. the StoredPrefix (this is a challenge in itself, since the
-// StoredPrefix needs to be read atomically to retrieve the PrefixId).
-// Compare-and-swapping a whole vec most probably would need a hash over the
-// vec to determine whether it was changed. I gave up on this approach,
-//
-// Another approach to try to limit the memory use is to try to use other
-// indexes in the same array on collision (the array mentioned above), before
-// heading off and following the reference to the next bucket. This would
-// limit the amount of (sparse) arrays being created for a typical prefix
-// treebitmap, at the cost of longer average search times. Two
-// implementations of this approach are Cuckoo hashing[^1], and Skip Lists.
-// Skip lists[^2] are a probabilistic data-structure, famously used by Redis,
-// (and by TiKv). I haven't tries either of these. Crossbeam has a SkipList
-// implementation, that wasn't ready at the time I wrote this. Cuckoo
-// hashing has the advantage of being easier to understand/implement. Maybe
-// Cuckoo hashing can also be combined with Fibonacci hashing[^3]. Note that
-// Robin Hood hashing maybe faster than Cuckoo hashing for reads, but it
-// requires shifting around existing entries, which is rather costly to do
-// atomically (and complex).
-
-// [^1]: [https://en.wikipedia.org/wiki/Cuckoo_hashing]
-// [^3]: [https://docs.rs/crossbeam-skiplist/0.1.1/crossbeam_skiplist/]
-// [^3]: [https://probablydance.com/2018/06/16/fibonacci-hashing-
-//  the-optimization-that-the-world-forgot-or-a-better-alternative-
-//  to-integer-modulo/]
-
-// Notes on memory leaks in Rotonda-store
-//
-// Both valgrind and miri report memory leaks on the multi-threaded prefix
-// store. Valgrind only reports it when it a binary stops using the tree,
-// while still keeping it around. An interrupted use of the mt-prefix-store
-// does not report any memory leaks. Miri is persistent in reporting memory
-// leaks in the mt-prefix-store. They both report the memory leaks in the same
-// location: the init method of the node- and prefix-buckets.
-//
-// I have reasons to believe these reported memory leaks aren't real, or that
-// crossbeam-epoch leaks a bit of memory when creating a new `Atomic`
-// instance. Since neither prefix nor node buckets can or should be dropped
-// this is not a big issue anyway, it just means that an `Atomic` occupies
-// more memory than it could in an optimal situation. Since we're not storing
-// the actual meta-data in an `Atomic` (it is stored in an `flurry Map`), this
-// means memory usage won't grow on updating the meta-data on a prefix,
-// (unless the meta-data itself grows of course, but that's up to the user).
-//
-// To get a better understanding on the nature of the reported memory leaks I
-// have created a branch (`vec_set`) that replaces the dynamically sized array
-// with a (equally sparse) Vec, that is not filled with `Atomic:::null()`, but
-// with `Option<StoredPrefix` instead, in order to see if this would eliminate
-// the memory leaks reporting. It did not. Valgrind still reports the memory
-// leaks at the same location, although they're now reported as `indirectly
-// leaked`, instead of `directly`. Miri is unchanged. The attentive reader may
-// now suspect that the `Atomic` inside the `AtomicStoredPrefix` may be the
-// culprit, but: a. Both miri and valgrind report the leaks still in the same
-// place: the creation of the buckets, not the creation of the stored
-//    prefixes. b. Tests that look at the memory usage of the prefix stores
-// under heavy modification of existing prefixes do not exhibit memory leaking
-//    behavior. c. Tests that look at memory usage under addition of new
-//    prefixes exhibit linear incrementation of memory usage (which is
-// expected). d. Tests that look at memory usage under contention do not
-//    exhibit increased memory usage either.
-//
-// My strong suspicion is that both Miri and Valgrind report the sparse slots
-// in the buckets as leaks, no matter whether they're `Atomic::null()`, or
-// `None` values, probably as a result of the way `crossbeam-epoch` indexes
-// into these, with pointer arithmetic (unsafe as hell).
-//
-// I would be super grateful if somebody would prove me wrong and can point to
-// an actual memory leak in the mt-prefix-store (and even more if they can
-// produce a fix for it).
-
-use std::path::Path;
-use std::sync::atomic::{AtomicUsize, Ordering};
-
-use log::{debug, info, log_enabled, trace};
-
-use crossbeam_epoch::{self as epoch, Atomic};
-use crossbeam_utils::Backoff;
-use epoch::{Guard, Owned};
-use lsm_tree::AbstractTree;
-use roaring::RoaringBitmap;
-
-use std::marker::PhantomData;
-
-use crate::{local_array::tree::*, stats::CreatedNodes};
-use crate::{
-<<<<<<< Updated upstream
-=======
-    local_array::store::oncebox::{OnceBox, OnceBoxSlice},
->>>>>>> Stashed changes
-    local_array::{bit_span::BitSpan, store::errors::PrefixStoreError},
-    prefix_record::PublicRecord,
-};
-
-use crate::{
-    impl_search_level, impl_search_level_for_mui, retrieve_node_mut_closure,
-    store_node_closure, Meta,
-};
-
-use super::atomic_types::*;
-use crate::AddressFamily;
-
-//------------ Counters -----------------------------------------------------
-
-#[derive(Debug)]
-pub struct Counters {
-    nodes: AtomicUsize,
-    prefixes: [AtomicUsize; 129],
-}
-
-impl Counters {
-    pub fn get_nodes_count(&self) -> usize {
-        self.nodes.load(Ordering::Relaxed)
-    }
-
-    pub fn inc_nodes_count(&self) {
-        self.nodes.fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn get_prefixes_count(&self) -> Vec<usize> {
-        self.prefixes
-            .iter()
-            .map(|pc| pc.load(Ordering::Relaxed))
-            .collect::<Vec<_>>()
-    }
-
-    pub fn inc_prefixes_count(&self, len: u8) {
-        self.prefixes[len as usize].fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn get_prefix_stats(&self) -> Vec<CreatedNodes> {
-        self.prefixes
-            .iter()
-            .enumerate()
-            .filter_map(|(len, count)| {
-                let count = count.load(Ordering::Relaxed);
-                if count != 0 {
-                    Some(CreatedNodes {
-                        depth_level: len as u8,
-                        count,
-                    })
-                } else {
-                    None
-                }
-            })
-            .collect()
-    }
-}
-
-impl Default for Counters {
-    fn default() -> Self {
-        let mut prefixes: Vec<AtomicUsize> = Vec::with_capacity(129);
-        for _ in 0..=128 {
-            prefixes.push(AtomicUsize::new(0));
-        }
-
-        Self {
-            nodes: AtomicUsize::new(0),
-            prefixes: prefixes.try_into().unwrap(),
-        }
-    }
-}
-
-//------------ StoreStats ----------------------------------------------
-
-#[derive(Debug)]
-pub struct StoreStats {
-    pub v4: Vec<CreatedNodes>,
-    pub v6: Vec<CreatedNodes>,
-}
-
-//------------ UpsertReport --------------------------------------------------
-
-#[derive(Debug)]
-pub struct UpsertReport {
-    // Indicates the number of Atomic Compare-and-Swap operations were
-    // necessary to create/update the Record entry. High numbers indicate
-    // contention.
-    pub cas_count: usize,
-    // Indicates whether this was the first mui record for this prefix was
-    // created. So, the prefix did not exist before hand.
-    pub prefix_new: bool,
-    // Indicates whether this mui was new for this prefix. False means an old
-    // value was overwritten.
-    pub mui_new: bool,
-    // The number of mui records for this prefix after the upsert operation.
-    pub mui_count: usize,
-}
-
-//------------ PersistTree ---------------------------------------------------
-
-pub struct PersistTree<
-    AF: AddressFamily,
-    // The size in bytes of the prefix in the peristed storage (disk), this
-    // amounnts to the bytes for the addres (4 for IPv4, 16 for IPv6) and 1
-    // bytefor the prefix length.
-    const PREFIX_SIZE: usize,
-    // The size in bytes of the complete key in the persisted storage, this
-    // is PREFIX_SIZE bytes (4; 16) + mui size (4) + ltime (8)
-    const KEY_SIZE: usize,
->(lsm_tree::Tree, PhantomData<AF>);
-
-impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
-    PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
-{
-    pub fn insert(&self, key: [u8; KEY_SIZE], value: &[u8]) -> (u32, u32) {
-        self.0.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
-    }
-
-    pub fn get_records_for_prefix<M: Meta>(
-        &self,
-        prefix: PrefixId<AF>,
-    ) -> Vec<PublicRecord<M>> {
-        let prefix_b = &prefix.as_bytes::<PREFIX_SIZE>();
-        (*self.0.prefix(prefix_b))
-            .into_iter()
-            .map(|kv| {
-                let kv = kv.unwrap();
-                let (_, mui, ltime, status) = Self::parse_key(kv.0.as_ref());
-                PublicRecord::new(
-                    mui,
-                    ltime,
-                    status.try_into().unwrap(),
-                    kv.1.as_ref().to_vec().into(),
-                )
-            })
-            .collect::<Vec<_>>()
-    }
-
-    pub fn get_records_for_key<M: Meta + From<Vec<u8>>>(
-        &self,
-        key: &[u8],
-    ) -> Vec<(inetnum::addr::Prefix, PublicRecord<M>)> {
-        (*self.0.prefix(key))
-            .into_iter()
-            .map(|kv| {
-                let kv = kv.unwrap();
-                let (pfx, mui, ltime, status) =
-                    Self::parse_key(kv.0.as_ref());
-
-                (
-                    PrefixId::<AF>::from(pfx).into_pub(),
-                    PublicRecord::new(
-                        mui,
-                        ltime,
-                        status.try_into().unwrap(),
-                        kv.1.as_ref().to_vec().into(),
-                    ),
-                )
-            })
-            .collect::<Vec<_>>()
-    }
-
-    pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
-        let segment = self.0.flush_active_memtable(0);
-
-        if let Ok(Some(segment)) = segment {
-            self.0.register_segments(&[segment])?;
-            self.0.compact(
-                std::sync::Arc::new(lsm_tree::compaction::Leveled::default()),
-                0,
-            )?;
-        };
-
-        Ok(())
-    }
-
-    pub fn approximate_len(&self) -> usize {
-        self.0.approximate_len()
-    }
-
-    pub fn disk_space(&self) -> u64 {
-        self.0.disk_space()
-    }
-
-    #[cfg(feature = "persist")]
-    pub fn persistence_key(
-        // PREFIX_SIZE bytes
-        prefix_id: PrefixId<AF>,
-        // 4 bytes
-        mui: u32,
-        // 8 bytes
-        ltime: u64,
-        // 1 byte
-        status: RouteStatus,
-    ) -> [u8; KEY_SIZE] {
-        assert!(KEY_SIZE > PREFIX_SIZE);
-        let key = &mut [0_u8; KEY_SIZE];
-
-        // prefix 5 or 17 bytes
-        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
-
-        // mui 4 bytes
-        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
-            .first_chunk_mut::<4>()
-            .unwrap() = mui.to_le_bytes();
-
-        // ltime 8 bytes
-        *key[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
-            .first_chunk_mut::<8>()
-            .unwrap() = ltime.to_le_bytes();
-
-        // status 1 byte
-        key[PREFIX_SIZE + 12] = status.into();
-
-        *key
-    }
-
-    #[cfg(feature = "persist")]
-    pub fn prefix_mui_persistence_key(
-        prefix_id: PrefixId<AF>,
-        mui: u32,
-    ) -> Vec<u8> {
-        let mut key = vec![];
-        // prefix 5 or 17 bytes
-        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
-
-        // mui 4 bytes
-        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
-            .first_chunk_mut::<4>()
-            .unwrap() = mui.to_le_bytes();
-
-        key
-    }
-
-    #[cfg(feature = "persist")]
-    pub fn parse_key(bytes: &[u8]) -> ([u8; PREFIX_SIZE], u32, u64, u8) {
-        (
-            // prefix 5 or 17 bytes
-            *bytes.first_chunk::<PREFIX_SIZE>().unwrap(),
-            // mui 4 bytes
-            u32::from_le_bytes(
-                *bytes[PREFIX_SIZE..PREFIX_SIZE + 4]
-                    .first_chunk::<4>()
-                    .unwrap(),
-            ),
-            // ltime 8 bytes
-            u64::from_le_bytes(
-                *bytes[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
-                    .first_chunk::<8>()
-                    .unwrap(),
-            ),
-            // status 1 byte
-            bytes[PREFIX_SIZE + 12],
-        )
-    }
-
-    #[cfg(feature = "persist")]
-    pub fn parse_prefix(bytes: &[u8]) -> [u8; PREFIX_SIZE] {
-        *bytes.first_chunk::<PREFIX_SIZE>().unwrap()
-    }
-
-    #[cfg(feature = "persist")]
-    pub(crate) fn persist_record<M: Meta>(
-        &self,
-        prefix: PrefixId<AF>,
-        mui: u32,
-        record: &PublicRecord<M>,
-    ) {
-        self.insert(
-            PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
-                prefix,
-                mui,
-                record.ltime,
-                record.status,
-            ),
-            record.meta.as_ref(),
-        );
-    }
-}
-
-impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
-    std::fmt::Debug for PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
-{
-    fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        todo!()
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub enum PersistStrategy {
-    WriteAhead,
-    PersistHistory,
-    MemoryOnly,
-    PersistOnly,
-}
-
-#[derive(Debug, Clone)]
-pub struct StoreConfig {
-    pub persist_strategy: PersistStrategy,
-    pub persist_path: String,
-}
-
-impl StoreConfig {
-    pub(crate) fn persist_strategy(&self) -> PersistStrategy {
-        self.persist_strategy
-    }
-}
-
-// ----------- CustomAllocStorage -------------------------------------------
-//
-// CustomAllocStorage is a storage backend that uses a custom allocator, that
-// consists of arrays that point to other arrays on collision.
-#[derive(Debug)]
-pub struct CustomAllocStorage<
-    AF: AddressFamily,
-    M: Meta,
-    NB: NodeBuckets<AF>,
-    PB: PrefixBuckets<AF, M>,
-    const PREFIX_SIZE: usize,
-    const KEY_SIZE: usize,
-> {
-    pub(crate) config: StoreConfig,
-    pub(crate) buckets: NB,
-    pub prefixes: PB,
-    #[cfg(feature = "persist")]
-    pub persistence: Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
-    pub default_route_prefix_serial: AtomicUsize,
-    // Global Roaring BitMap INdex that stores MUIs.
-    pub withdrawn_muis_bmin: Atomic<RoaringBitmap>,
-    pub counters: Counters,
-    _m: PhantomData<M>,
-}
-
-impl<
-        'a,
-        AF: AddressFamily,
-        M: crate::prefix_record::Meta,
-        NB: NodeBuckets<AF>,
-        PB: PrefixBuckets<AF, M>,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > CustomAllocStorage<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
-{
-    pub(crate) fn init(
-        root_node: SizedStrideNode<AF>,
-        config: StoreConfig,
-    ) -> Result<Self, Box<dyn std::error::Error>> {
-        info!("store: initialize store {}", AF::BITS);
-
-        let persistence = match config.persist_strategy {
-            PersistStrategy::MemoryOnly => None,
-            _ => {
-                let persist_path = Path::new(&config.persist_path);
-                Some(PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>(
-                    lsm_tree::Config::new(persist_path).open()?,
-                    PhantomData,
-                ))
-            }
-        };
-
-        let store = CustomAllocStorage {
-            config,
-            buckets: NodeBuckets::<AF>::init(),
-            prefixes: PrefixBuckets::<AF, M>::init(),
-            persistence,
-            default_route_prefix_serial: AtomicUsize::new(0),
-            withdrawn_muis_bmin: RoaringBitmap::new().into(),
-            counters: Counters::default(),
-            _m: PhantomData,
-        };
-
-        let _retry_count = store.store_node(
-            StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0),
-            0_u32,
-            root_node,
-        )?;
-
-        Ok(store)
-    }
-
-    pub(crate) fn acquire_new_node_id(
-        &self,
-        (prefix_net, sub_prefix_len): (AF, u8),
-    ) -> StrideNodeId<AF> {
-        StrideNodeId::new_with_cleaned_id(prefix_net, sub_prefix_len)
-    }
-
-    // Create a new node in the store with payload `next_node`.
-    //
-    // Next node will be ignored if a node with the same `id` already exists,
-    // but the multi_uniq_id will be added to the rbm_index of the NodeSet.
-    //
-    // Returns: a tuple with the node_id of the created node and the number of
-    // retry_count
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn store_node(
-        &self,
-        id: StrideNodeId<AF>,
-        multi_uniq_id: u32,
-        next_node: SizedStrideNode<AF>,
-    ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn Fn(
-                &SearchLevel<AF, S>,
-                &NodeSet<AF, S>,
-                TreeBitMapNode<AF, S>,
-                u32, // multi_uniq_id
-                u8,  // the store level
-                u32, // retry_count
-            ) -> Result<
-                (StrideNodeId<AF>, u32),
-                PrefixStoreError,
-            >,
-        }
-
-        let search_level_3 =
-            store_node_closure![Stride3; id; guard; back_off;];
-        let search_level_4 =
-            store_node_closure![Stride4; id; guard; back_off;];
-        let search_level_5 =
-            store_node_closure![Stride5; id; guard; back_off;];
-
-        if log_enabled!(log::Level::Trace) {
-            debug!(
-                "{} store: Store node {}: {:?} mui {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                next_node,
-                multi_uniq_id
-            );
-        }
-        self.counters.inc_nodes_count();
-
-        match next_node {
-            SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
-                &search_level_3,
-                self.buckets.get_store3(id),
-                new_node,
-                multi_uniq_id,
-                0,
-                0,
-            ),
-            SizedStrideNode::Stride4(new_node) => (search_level_4.f)(
-                &search_level_4,
-                self.buckets.get_store4(id),
-                new_node,
-                multi_uniq_id,
-                0,
-                0,
-            ),
-            SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
-                &search_level_5,
-                self.buckets.get_store5(id),
-                new_node,
-                multi_uniq_id,
-                0,
-                0,
-            ),
-        }
-    }
-
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_node(
-        &'a self,
-        id: StrideNodeId<AF>,
-    ) -> Option<SizedStrideRef<'a, AF>> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, S>,
-                &'a NodeSet<AF, S>,
-                u8,
-            )
-                -> Option<SizedStrideRef<'a, AF>>,
-        }
-
-        let search_level_3 = impl_search_level![Stride3; id;];
-        let search_level_4 = impl_search_level![Stride4; id;];
-        let search_level_5 = impl_search_level![Stride5; id;];
-
-        if log_enabled!(log::Level::Trace) {
-            trace!(
-                "{} store: Retrieve node {} from l{}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                id.get_id().1
-            );
-        }
-
-        match self.get_stride_for_id(id) {
-            3 => (search_level_3.f)(
-                &search_level_3,
-                self.buckets.get_store3(id),
-                0,
-            ),
-            4 => (search_level_4.f)(
-                &search_level_4,
-                self.buckets.get_store4(id),
-                0,
-            ),
-            _ => (search_level_5.f)(
-                &search_level_5,
-                self.buckets.get_store5(id),
-                0,
-            ),
-        }
-    }
-
-    // retrieve a node, but only its bitmap index contains the specified mui.
-    // Used for iterators per mui.
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_node_for_mui(
-        &'a self,
-        id: StrideNodeId<AF>,
-        // The mui that is tested to be present in the nodes bitmap index
-        mui: u32,
-    ) -> Option<SizedStrideRef<'a, AF>> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, S>,
-                &'a NodeSet<AF, S>,
-                u8,
-            )
-                -> Option<SizedStrideRef<'a, AF>>,
-        }
-
-        let search_level_3 = impl_search_level_for_mui![Stride3; id; mui;];
-        let search_level_4 = impl_search_level_for_mui![Stride4; id; mui;];
-        let search_level_5 = impl_search_level_for_mui![Stride5; id; mui;];
-
-        if log_enabled!(log::Level::Trace) {
-            trace!(
-                "{} store: Retrieve node {} from l{} for mui {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                id.get_id().1,
-                mui
-            );
-        }
-
-        match self.get_stride_for_id(id) {
-            3 => (search_level_3.f)(
-                &search_level_3,
-                self.buckets.get_store3(id),
-                0,
-            ),
-            4 => (search_level_4.f)(
-                &search_level_4,
-                self.buckets.get_store4(id),
-                0,
-            ),
-            _ => (search_level_5.f)(
-                &search_level_5,
-                self.buckets.get_store5(id),
-                0,
-            ),
-        }
-    }
-
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_node_mut(
-        &'a self,
-        id: StrideNodeId<AF>,
-        multi_uniq_id: u32,
-    ) -> Option<SizedStrideRef<'a, AF>> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, S>,
-                &'a NodeSet<AF, S>,
-                u8,
-            )
-                -> Option<SizedStrideRef<'a, AF>>,
-        }
-
-        let search_level_3 =
-            retrieve_node_mut_closure![Stride3; id; multi_uniq_id;];
-        let search_level_4 =
-            retrieve_node_mut_closure![Stride4; id; multi_uniq_id;];
-        let search_level_5 =
-            retrieve_node_mut_closure![Stride5; id; multi_uniq_id;];
-
-        if log_enabled!(log::Level::Trace) {
-            trace!(
-                "{} store: Retrieve node mut {} from l{}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                id.get_id().1
-            );
-        }
-
-        match self.buckets.get_stride_for_id(id) {
-            3 => (search_level_3.f)(
-                &search_level_3,
-                self.buckets.get_store3(id),
-                0,
-            ),
-
-            4 => (search_level_4.f)(
-                &search_level_4,
-                self.buckets.get_store4(id),
-                0,
-            ),
-            _ => (search_level_5.f)(
-                &search_level_5,
-                self.buckets.get_store5(id),
-                0,
-            ),
-        }
-    }
-
-    pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
-        StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0)
-    }
-
-    pub fn get_nodes_count(&self) -> usize {
-        self.counters.get_nodes_count()
-    }
-
-    // Prefixes related methods
-
-    pub(crate) fn load_default_route_prefix_serial(&self) -> usize {
-        self.default_route_prefix_serial.load(Ordering::SeqCst)
-    }
-
-    #[allow(dead_code)]
-    fn increment_default_route_prefix_serial(&self) -> usize {
-        self.default_route_prefix_serial
-            .fetch_add(1, Ordering::SeqCst)
-    }
-
-    // THE CRITICAL SECTION
-    //
-    // CREATING OR UPDATING A PREFIX IN THE STORE
-    //
-    // YES, THE MAGIC HAPPENS HERE!
-    //
-    // This uses the TAG feature of crossbeam_utils::epoch to ensure that we
-    // are not overwriting a prefix meta-data that already has been created
-    // or was updated by another thread.
-    //
-    // Our plan:
-    //
-    // 1. LOAD
-    //    Load the current prefix and meta-data from the store if any.
-    // 2. INSERT
-    //    If there is no current meta-data, create it.
-    // 3. UPDATE
-    //    If there is a prefix, meta-data combo, then load it and merge
-    //    the existing meta-dat with our meta-data using the `MergeUpdate`
-    //    trait (a so-called 'Read-Copy-Update').
-    // 4. SUCCESS
-    //    See if we can successfully store the updated meta-data in the store.
-    // 5. DONE
-    //    If Step 4 succeeded we're done!
-    // 6. FAILURE - REPEAT
-    //    If Step 4 failed we're going to do the whole thing again.
-
-    pub(crate) fn upsert_prefix(
-        &self,
-        prefix: PrefixId<AF>,
-        record: PublicRecord<M>,
-        update_path_selections: Option<M::TBI>,
-        guard: &Guard,
-    ) -> Result<UpsertReport, PrefixStoreError> {
-        let mut prefix_is_new = true;
-        let mut mui_is_new = true;
-
-        let (mui_count, cas_count) =
-            match self.non_recursive_retrieve_prefix_mut(prefix) {
-                // There's no StoredPrefix at this location yet. Create a new
-                // PrefixRecord and try to store it in the empty slot.
-                (locked_prefix, false) => {
-                    if log_enabled!(log::Level::Debug) {
-                        debug!(
-                            "{} store: Create new prefix record",
-                            std::thread::current()
-                                .name()
-                                .unwrap_or("unnamed-thread")
-                        );
-                    }
-
-<<<<<<< Updated upstream
-                    let (mui_count, retry_count) =
-                        locked_prefix.record_map.upsert_record(
-                            prefix,
-                            record,
-                            &self.persistence,
-                            self.config.persist_strategy,
-                        )?;
-=======
-                    // We're creating a StoredPrefix without our record first,
-                    // to avoid having to clone it on retry.
-                    let this_level =
-                        PB::get_bits_for_len(prefix.get_len(), level);
-                    let next_level =
-                        PB::get_bits_for_len(prefix.get_len(), level + 1);
->>>>>>> Stashed changes
-
-                    // See if someone beat us to creating the record.
-                    if mui_count.is_some() {
-                        mui_is_new = false;
-                        prefix_is_new = false;
-                    } else {
-                        // No, we were the first, we created a new prefix
-                        self.counters.inc_prefixes_count(prefix.get_len());
-                    }
-
-<<<<<<< Updated upstream
-                    (mui_count, retry_count)
-=======
-                    let p = locked_prefix
-                        .get_or_init(1 << (next_level - this_level), || {
-                            new_stored_prefix
-                        });
-                    // let back_off = Backoff::new();
-
-                    let res = p.record_map.upsert_record(record);
-
-                    // loop {
-                    //     if let Ok(()) = set_res {
-                    //         break;
-                    //     } else {
-                    //         if let Some(stored_record) = locked_prefix.get() {
-                    //             res = stored_record
-                    //                 .record_map
-                    //                 .upsert_record(record.clone());
-                    //             // locked_prefix.set(*stored_record);
-                    //         };
-                    //     }
-                    //     back_off.snooze();
-                    // }
-
-                    // We're expecting an empty slot.
-                    // match atomic_stored_prefix.0.compare_exchange(
-                    //     Shared::null(),
-                    //     // tag with value 1, means the path selection is set to
-                    //     // outdated.
-                    //     Owned::new(new_stored_prefix).with_tag(1),
-                    //     Ordering::AcqRel,
-                    //     Ordering::Acquire,
-                    //     guard,
-                    // ) {
-                    //     // ...and we got an empty slot, the newly created
-                    //     // StoredPrefix is stored into it.
-                    //     Ok(spfx) => {
-                    //         if log_enabled!(log::Level::Info) {
-                    //             let StoredPrefix {
-                    //                 prefix,
-                    //                 record_map: stored_record,
-                    //                 ..
-                    //             } = unsafe { spfx.deref() };
-                    //             if log_enabled!(log::Level::Info) {
-                    //                 info!(
-                    //                         "{} store: Inserted new prefix record {}/{} with {:?}",
-                    //                         std::thread::current().name().unwrap(),
-                    //                         prefix.get_net().into_ipaddr(), prefix.get_len(),
-                    //                         stored_record
-                    //                     );
-                    //             }
-                    //         }
-
-                    self.counters.inc_prefixes_count(prefix.get_len());
-
-                    //         // ..and update the record_map with the actual record
-                    //         // we got from the user.
-                    //         unsafe { spfx.deref() }
-                    //             .record_map
-                    //             .upsert_record(record)
-                    //     }
-                    //     // ...somebody beat us to it, the slot's not empty
-                    //     // anymore, we'll have to do it again.
-                    //     Err(CompareExchangeError { current, new: _ }) => {
-                    //         if log_enabled!(log::Level::Debug) {
-                    //             debug!(
-                    //                     "{} store: Prefix can't be inserted as new {:?}",
-                    //                     std::thread::current().name().unwrap(),
-                    //                     current
-                    //                 );
-                    //         }
-                    //         retry_count += 1;
-                    //         let stored_prefix = unsafe { current.deref() };
-
-                    //         // update the record_map from the winning thread
-                    //         // with our caller's record.
-                    //         stored_prefix.set_ps_outdated(guard)?;
-                    //         stored_prefix.record_map.upsert_record(record)
-                    //     }
-                    // }
-                    res
->>>>>>> Stashed changes
-                }
-                // There already is a StoredPrefix with a record at this
-                // location.
-                (stored_prefix, true) => {
-                    if log_enabled!(log::Level::Debug) {
-                        debug!(
-                        "{} store: Found existing prefix record for {}/{}",
-                        std::thread::current()
-                            .name()
-                            .unwrap_or("unnamed-thread"),
-                        prefix.get_net(),
-                        prefix.get_len()
-                    );
-                    }
-                    prefix_is_new = false;
-
-                    // Update the already existing record_map with our
-                    // caller's record.
-                    stored_prefix.set_ps_outdated(guard)?;
-
-                    let (mui_count, retry_count) =
-                        stored_prefix.record_map.upsert_record(
-                            prefix,
-                            record,
-                            &self.persistence,
-                            self.config.persist_strategy,
-                        )?;
-                    mui_is_new = mui_count.is_none();
-
-                    if let Some(tbi) = update_path_selections {
-                        stored_prefix
-                            .calculate_and_store_best_backup(&tbi, guard)?;
-                    }
-
-                    (mui_count, retry_count)
-                }
-            };
-
-        Ok(UpsertReport {
-            prefix_new: prefix_is_new,
-            cas_count,
-            mui_new: mui_is_new,
-            mui_count: mui_count.unwrap_or(1),
-        })
-    }
-
-    // Change the status of the record for the specified (prefix, mui)
-    // combination  to Withdrawn.
-    pub fn mark_mui_as_withdrawn_for_prefix(
-        &self,
-        prefix: PrefixId<AF>,
-        mui: u32,
-    ) -> Result<(), PrefixStoreError> {
-        let (stored_prefix, exists) =
-            self.non_recursive_retrieve_prefix_mut(prefix);
-
-        if !exists {
-            return Err(PrefixStoreError::StoreNotReadyError);
-        }
-
-        stored_prefix.record_map.mark_as_withdrawn_for_mui(mui);
-
-        Ok(())
-    }
-
-    // Change the status of the record for the specified (prefix, mui)
-    // combination  to Active.
-    pub fn mark_mui_as_active_for_prefix(
-        &self,
-        prefix: PrefixId<AF>,
-        mui: u32,
-    ) -> Result<(), PrefixStoreError> {
-        let (stored_prefix, exists) =
-            self.non_recursive_retrieve_prefix_mut(prefix);
-
-        if !exists {
-            return Err(PrefixStoreError::StoreNotReadyError);
-        }
-
-        stored_prefix.record_map.mark_as_active_for_mui(mui);
-
-        Ok(())
-    }
-
-    // Change the status of the mui globally to Withdrawn. Iterators and match
-    // functions will by default not return any records for this mui.
-    pub fn mark_mui_as_withdrawn(
-        &self,
-        mui: u32,
-        guard: &Guard,
-    ) -> Result<(), PrefixStoreError> {
-        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
-
-        let mut new = unsafe { current.as_ref() }.unwrap().clone();
-        new.insert(mui);
-
-        #[allow(clippy::assigning_clones)]
-        loop {
-            match self.withdrawn_muis_bmin.compare_exchange(
-                current,
-                Owned::new(new),
-                Ordering::AcqRel,
-                Ordering::Acquire,
-                guard,
-            ) {
-                Ok(_) => return Ok(()),
-                Err(updated) => {
-                    new =
-                        unsafe { updated.current.as_ref() }.unwrap().clone();
-                }
-            }
-        }
-    }
-
-    // Change the status of the mui globally to Active. Iterators and match
-    // functions will default to the status on the record itself.
-    pub fn mark_mui_as_active(
-        &self,
-        mui: u32,
-        guard: &Guard,
-    ) -> Result<(), PrefixStoreError> {
-        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
-
-        let mut new = unsafe { current.as_ref() }.unwrap().clone();
-        new.remove(mui);
-
-        #[allow(clippy::assigning_clones)]
-        loop {
-            match self.withdrawn_muis_bmin.compare_exchange(
-                current,
-                Owned::new(new),
-                Ordering::AcqRel,
-                Ordering::Acquire,
-                guard,
-            ) {
-                Ok(_) => return Ok(()),
-                Err(updated) => {
-                    new =
-                        unsafe { updated.current.as_ref() }.unwrap().clone();
-                }
-            }
-        }
-    }
-
-    // Whether this mui is globally withdrawn. Note that this overrules (by
-    // default) any (prefix, mui) combination in iterators and match functions.
-    pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
-        unsafe {
-            self.withdrawn_muis_bmin
-                .load(Ordering::Acquire, guard)
-                .as_ref()
-        }
-        .unwrap()
-        .contains(mui)
-    }
-
-    // Whether this mui is globally active. Note that the local statuses of
-    // records (prefix, mui) may be set to withdrawn in iterators and match
-    // functions.
-    pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
-        !unsafe {
-            self.withdrawn_muis_bmin
-                .load(Ordering::Acquire, guard)
-                .as_ref()
-        }
-        .unwrap()
-        .contains(mui)
-    }
-
-    // This function is used by the upsert_prefix function above.
-    //
-    // We're using a Chained Hash Table and this function returns one of:
-    // - a StoredPrefix that already exists for this search_prefix_id
-    // - the Last StoredPrefix in the chain.
-    // - an error, if no StoredPrefix whatsoever can be found in the store.
-    //
-    // The error condition really shouldn't happen, because that basically
-    // means the root node for that particular prefix length doesn't exist.
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn non_recursive_retrieve_prefix_mut(
-        &'a self,
-        search_prefix_id: PrefixId<AF>,
-<<<<<<< Updated upstream
-    ) -> (&'a StoredPrefix<AF, M>, bool) {
-=======
-    ) -> Result<
-        (&'a StoredPrefix<AF, M>, u8),
-        (&'a OnceBoxSlice<StoredPrefix<AF, M>>, u8),
-    > {
->>>>>>> Stashed changes
-        trace!("non_recursive_retrieve_prefix_mut_with_guard");
-        let mut prefix_set = self
-            .prefixes
-            .get_root_prefix_set(search_prefix_id.get_len());
-        let mut level: u8 = 0;
-
-        trace!("root prefix_set {:?}", prefix_set);
-        loop {
-            // HASHING FUNCTION
-            let index = Self::hash_prefix_id(search_prefix_id, level);
-
-            // probe the slot with the index that's the result of the hashing.
-<<<<<<< Updated upstream
-            // let locked_prefix = prefix_set.0.get(index);
-            let stored_prefix = match prefix_set.0.get(index) {
-                Some(p) => {
-                    trace!("prefix set found.");
-                    (p, true)
-                }
-=======
-            // stored_prefix = Some(prefix_set.0[index]);
-            // let prefix_probe = if !prefixes.is_null() {
-            trace!("prefix set found.");
-            let locked_prefix = prefix_set.0.get(index);
-            let stored_prefix = match locked_prefix {
-                // None => {
-                // panic!("index for PrefixSet out of bounds. search_prefix_id {:?}, level {}", search_prefix_id, level);
-                // }
-                Some(p) => p,
->>>>>>> Stashed changes
-                None => {
-                    // We're at the end of the chain and haven't found our
-                    // search_prefix_id anywhere. Return the end-of-the-chain
-                    // StoredPrefix, so the caller can attach a new one.
-<<<<<<< Updated upstream
-                    trace!(
-                        "no record. returning last found record in level
-                        {}, with index {}.",
-                        level,
-                        index
-                    );
-                    let index = Self::hash_prefix_id(search_prefix_id, level);
-                    trace!("calculate next index {}", index);
-                    (
-                        prefix_set
-                            .0
-                            .get_or_init(index, || {
-                                StoredPrefix::new::<PB>(
-                                    PrefixId::new(
-                                        search_prefix_id.get_net(),
-                                        search_prefix_id.get_len(),
-                                    ),
-                                    level,
-                                )
-                            })
-                            .0,
-                        false,
-                    )
-=======
-                    // trace!("no prefix set.");
-                    trace!("no record. returning last found record.");
-                    return Err((&prefix_set.0, level));
-                    // .map(|sp| (sp, level)
-                    // .ok_or(locked_prefix)
->>>>>>> Stashed changes
-                }
-            };
-
-            if search_prefix_id == stored_prefix.0.prefix {
-                // GOTCHA!
-                // Our search-prefix is stored here, so we're returning
-                // it, so its PrefixRecord can be updated by the caller.
-                trace!("found requested prefix {:?}", search_prefix_id);
-                return stored_prefix;
-            } else {
-                // A Collision. Follow the chain.
-                level += 1;
-                prefix_set = &stored_prefix.0.next_bucket;
-                continue;
-            }
-        }
-    }
-
-    // This function is used by the match_prefix, and [more|less]_specifics
-    // public methods on the TreeBitMap (indirectly).
-    #[allow(clippy::type_complexity)]
-    pub fn non_recursive_retrieve_prefix(
-        &'a self,
-        id: PrefixId<AF>,
-    ) -> (
-        Option<&'a StoredPrefix<AF, M>>,
-        Option<(
-            PrefixId<AF>,
-            u8,
-            &'a PrefixSet<AF, M>,
-            [Option<(&'a PrefixSet<AF, M>, usize)>; 32],
-            usize,
-        )>,
-    ) {
-        let mut prefix_set = self.prefixes.get_root_prefix_set(id.get_len());
-        let mut parents = [None; 32];
-        let mut level: u8 = 0;
-        let backoff = Backoff::new();
-
-        loop {
-            // The index of the prefix in this array (at this len and
-            // level) is calculated by performing the hash function
-            // over the prefix.
-
-            // HASHING FUNCTION
-            let index = Self::hash_prefix_id(id, level);
-
-<<<<<<< Updated upstream
-            if let Some(stored_prefix) = prefix_set.0.get(index) {
-=======
-            // let mut prefixes = prefix_set; //.0.load(Ordering::Acquire, guard);
-
-            // if !prefixes.is_none() {
-            // let prefix_ref = prefixes.0[index];
-            if let Some(stored_prefix) = prefix_set.0.get(index)
-            // unsafe { prefix_ref }.get_stored_prefix(guard)
-            {
->>>>>>> Stashed changes
-                if id == stored_prefix.get_prefix_id() {
-                    trace!("found requested prefix {:?}", id);
-                    parents[level as usize] = Some((prefix_set, index));
-                    return (
-                        Some(stored_prefix),
-                        Some((id, level, prefix_set, parents, index)),
-                    );
-                };
-                // Advance to the next level.
-
-                prefix_set = &stored_prefix.next_bucket;
-                level += 1;
-                backoff.spin();
-                continue;
-            }
-
-            trace!("no prefix found for {:?}", id);
-            parents[level as usize] = Some((prefix_set, index));
-            return (None, Some((id, level, prefix_set, parents, index)));
-        }
-    }
-
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_prefix(
-        &'a self,
-        prefix_id: PrefixId<AF>,
-    ) -> Option<(&'a StoredPrefix<AF, M>, usize)> {
-        struct SearchLevel<
-            's,
-            AF: AddressFamily,
-            M: crate::prefix_record::Meta,
-        > {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, M>,
-                &'a PrefixSet<AF, M>,
-                u8,
-            )
-                -> Option<(&'a StoredPrefix<AF, M>, usize)>,
-        }
-
-        let search_level = SearchLevel {
-            f: &|search_level: &SearchLevel<AF, M>,
-                 prefix_set: &PrefixSet<AF, M>,
-                 mut level: u8| {
-                // HASHING FUNCTION
-                let index = Self::hash_prefix_id(prefix_id, level);
-
-<<<<<<< Updated upstream
-                if let Some(stored_prefix) = prefix_set.0.get(index) {
-=======
-                // let prefixes = prefix_set.0.load(Ordering::SeqCst, guard);
-                // let tag = prefixes.tag();
-                // let prefix_ref = unsafe { &prefixes.deref()[index] };
-
-                if let Some(stored_prefix) =
-                    // unsafe { prefix_ref }.get_stored_prefix(guard)
-                    prefix_set.0.get(index)
-                {
->>>>>>> Stashed changes
-                    if prefix_id == stored_prefix.prefix {
-                        trace!("found requested prefix {:?}", prefix_id,);
-                        return Some((stored_prefix, 0));
-                    };
-                    level += 1;
-
-                    (search_level.f)(
-                        search_level,
-                        &stored_prefix.next_bucket,
-                        level,
-                    );
-                }
-                None
-            },
-        };
-
-        (search_level.f)(
-            &search_level,
-            self.prefixes.get_root_prefix_set(prefix_id.get_len()),
-            0,
-        )
-    }
-
-    #[allow(dead_code)]
-    fn remove_prefix(&mut self, index: PrefixId<AF>) -> Option<M> {
-        match index.is_empty() {
-            false => self.prefixes.remove(index),
-            true => None,
-        }
-    }
-
-    pub fn get_prefixes_count(&self) -> usize {
-        self.counters.get_prefixes_count().iter().sum()
-    }
-
-    pub fn get_prefixes_count_for_len(&self, len: u8) -> usize {
-        self.counters.get_prefixes_count()[len as usize]
-    }
-
-    // Stride related methods
-
-    pub(crate) fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8 {
-        self.buckets.get_stride_for_id(id)
-    }
-
-    pub fn get_stride_sizes(&self) -> &[u8] {
-        self.buckets.get_stride_sizes()
-    }
-
-    // pub(crate) fn get_strides_len() -> u8 {
-    //     NB::get_strides_len()
-    // }
-
-    pub(crate) fn get_first_stride_size() -> u8 {
-        NB::get_first_stride_size()
-    }
-
-    // Calculates the id of the node that COULD host a prefix in its
-    // ptrbitarr.
-    pub(crate) fn get_node_id_for_prefix(
-        &self,
-        prefix: &PrefixId<AF>,
-    ) -> (StrideNodeId<AF>, BitSpan) {
-        let mut acc = 0;
-        for i in self.get_stride_sizes() {
-            acc += *i;
-            if acc >= prefix.get_len() {
-                let node_len = acc - i;
-                return (
-                    StrideNodeId::new_with_cleaned_id(
-                        prefix.get_net(),
-                        node_len,
-                    ),
-                    // NOT THE HASHING FUNCTION!
-                    // Do the right shift in a checked manner, for the sake
-                    // of 0/0. A search for 0/0 will perform a 0 << MAX_LEN,
-                    // which will panic in debug mode (undefined behaviour
-                    // in prod).
-                    BitSpan::new(
-                        ((prefix.get_net() << node_len).checked_shr_or_zero(
-                            (AF::BITS - (prefix.get_len() - node_len)).into(),
-                        ))
-                        .dangerously_truncate_to_u32(),
-                        prefix.get_len() - node_len,
-                    ),
-                );
-            }
-        }
-        panic!("prefix length for {:?} is too long", prefix);
-    }
-
-    // ------- THE HASHING FUNCTION -----------------------------------------
-
-    // Ok, so hashing is really hard, but we're keeping it simple, and
-    // because we're keeping it simple we're having lots of collisions, but
-    // we don't care!
-    //
-    // We're using a part of bitarray representation of the address part of
-    // a prefix the as the hash. Sounds complicated, but isn't.
-    // Suppose we have an IPv4 prefix, say 130.24.55.0/24.
-    // The address part is 130.24.55.0 or as a bitarray that would be:
-    //
-    // pos  0    4    8    12   16   20   24   28
-    // bit  1000 0010 0001 1000 0011 0111 0000 0000
-    //
-    // First, we're discarding the bits after the length of the prefix, so
-    // we'll have:
-    //
-    // pos  0    4    8    12   16   20
-    // bit  1000 0010 0001 1000 0011 0111
-    //
-    // Now we're dividing this bitarray into one or more levels. A level can
-    // be an arbitrary number of bits between 1 and the length of the prefix,
-    // but the number of bits summed over all levels should be exactly the
-    // prefix length. So in our case they should add up to 24. A possible
-    // division could be: 4, 4, 4, 4, 4, 4. Another one would be: 12, 12. The
-    // actual division being used is described in the function
-    // `<NB>::get_bits_for_len` in the `rotonda-macros` crate. Each level has
-    // its own hash, so for our example prefix this would be:
-    //
-    // pos   0    4    8    12   16   20
-    // level 0              1
-    // hash  1000 0010 0001 1000 0011 0111
-    //
-    // level 1 hash: 1000 0010 0001
-    // level 2 hash: 1000 0011 0011
-    //
-    // The hash is now converted to a usize integer, by shifting it all the
-    // way to the right in a u32 and then converting to a usize. Why a usize
-    // you ask? Because the hash is used by the CustomAllocStorage as the
-    // index to the array for that specific prefix length and level.
-    // So for our example this means that the hash on level 1 is now 0x821
-    // (decimal 2081) and the hash on level 2 is 0x833 (decimal 2099).
-    // Now, if we only consider the hash on level 1 and that we're going to
-    // use that as the index to the array that stores all prefixes, you'll
-    // notice very quickly that all prefixes starting with 130.[16..31] will
-    // cause a collision: they'll all point to the same array element. These
-    // collisions are resolved by creating a linked list from each array
-    // element, where each element in the list has an array of its own that
-    // uses the hash function with the level incremented.
-
-    pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
-        // And, this is all of our hashing function.
-        let last_level = if level > 0 {
-            <NB>::len_to_store_bits(id.get_id().1, level - 1)
-        } else {
-            0
-        };
-        let this_level = <NB>::len_to_store_bits(id.get_id().1, level);
-        trace!("bits division {}", this_level);
-        trace!(
-            "calculated index ({} << {}) >> {}",
-            id.get_id().0,
-            last_level,
-            ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-        );
-        // HASHING FUNCTION
-        ((id.get_id().0 << last_level)
-            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
-            .dangerously_truncate_to_u32() as usize
-    }
-
-    pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
-        // And, this is all of our hashing function.
-        let last_level = if level > 0 {
-            <PB>::get_bits_for_len(id.get_len(), level - 1)
-        } else {
-            0
-        };
-        let this_level = <PB>::get_bits_for_len(id.get_len(), level);
-        trace!(
-            "bits division {}; no of bits {}",
-            this_level,
-            this_level - last_level
-        );
-        trace!(
-            "calculated index ({} << {}) >> {}",
-            id.get_net(),
-            last_level,
-            ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-        );
-        // HASHING FUNCTION
-        ((id.get_net() << last_level)
-            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
-            .dangerously_truncate_to_u32() as usize
-    }
-}
diff --git a/src/local_array/store/macros.rs b/src/local_array/store/macros.rs
deleted file mode 100644
index 8b263bc8..00000000
--- a/src/local_array/store/macros.rs
+++ /dev/null
@@ -1,456 +0,0 @@
-#[macro_export]
-#[doc(hidden)]
-macro_rules! impl_search_level {
-    (
-        $(
-            $stride: ident;
-            $id: ident;
-        ),
-    * ) => {
-        $(
-        SearchLevel {
-            f: &|search_level: &SearchLevel<AF, $stride>,
-                nodes,
-                mut level: u8,
-            | {
-                    // HASHING FUNCTION
-                    let index = Self::hash_node_id($id, level);
-
-                    // Read the node from the block pointed to by the Atomic
-                    // pointer.
-                    // let stored_node = unsafe {
-                    //     &mut nodes.0[index]
-                    // };
-                    // let this_node = stored_node.load(Ordering::Acquire, guard);
-
-                    match nodes.0.get(index) {
-                        None => None,
-                        Some(stored_node) => {
-                            let StoredNode { node_id, node, node_set, .. } = stored_node;
-                            if $id == *node_id {
-                                // YES, It's the one we're looking for!
-                                return Some(SizedStrideRef::$stride(&node));
-                            };
-                            // Meh, it's not, but we can a go to the next
-                            // level and see if it lives there.
-                            level += 1;
-                            match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
-                                // on to the next level!
-                                next_bit_shift if next_bit_shift > 0 => {
-                                    (search_level.f)(
-                                        search_level,
-                                        &node_set,
-                                        level,
-                                        // guard,
-                                    )
-                                }
-                                // There's no next level, we found nothing.
-                                _ => None,
-                            }
-                        }
-                    }
-                }
-        }
-        )*
-    };
-}
-
-#[macro_export]
-#[doc(hidden)]
-macro_rules! impl_search_level_for_mui {
-    (
-        $(
-            $stride: ident;
-            $id: ident;
-            $mui: ident;
-        ),
-    * ) => {
-        $(
-        SearchLevel {
-            f: &|search_level: &SearchLevel<AF, $stride>,
-                nodes,
-                mut level: u8| {
-                    // HASHING FUNCTION
-                    let index = Self::hash_node_id($id, level);
-
-                    // Read the node from the block pointed to by the Atomic
-                    // pointer.
-                    // let stored_node = unsafe {
-                    //     &mut nodes.0[index].assume_init_ref()
-                    // };
-                    // let this_node = stored_node.load(Ordering::Acquire, guard);
-
-                    match nodes.0.get(index) {
-                        None => None,
-                        Some(this_node) => {
-                            let StoredNode { node_id, node, node_set, .. } = this_node;
-
-                            // early return if the mui is not in the index
-                            // stored in this node, meaning the mui does not
-                            // appear anywhere in the sub-tree formed from
-                            // this node.
-                            let bmin = node_set.1.read().unwrap(); // load(Ordering::Acquire, guard).deref()
-                            if !bmin.contains($mui) {
-                                return None;
-                            }
-
-                            if $id == *node_id {
-                                // YES, It's the one we're looking for!
-                                return Some(SizedStrideRef::$stride(&node));
-                            };
-                            // Meh, it's not, but we can a go to the next
-                            // level and see if it lives there.
-                            level += 1;
-                            match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
-                                // on to the next level!
-                                next_bit_shift if next_bit_shift > 0 => {
-                                    (search_level.f)(
-                                        search_level,
-                                        &node_set,
-                                        level,
-                                        // guard,
-                                    )
-                                }
-                                // There's no next level, we found nothing.
-                                _ => None,
-                            }
-                        }
-                    }
-                }
-        }
-        )*
-    };
-}
-
-// This macro creates a closure that is used in turn in the macro
-// 'eBox', that is used in the public `insert` method on a TreeBitMap.
-//
-// It retrieves the node specified by $id recursively, creates it if it does
-// not exist. It is responsible for setting/updating the RBMIN, but is does
-// *not* set/update the pfxbitarr or ptrbitarr of the TreeBitMapNode. The
-// `insert_match` takes care of the latter.
-//
-// This closure should not be called repeatedly to create the same node, if it
-// returns `None` that is basically a data race in the store and therefore an
-// error. Also the caller should make sure to stay within the limit of the
-// defined number of levels, although the closure will return at the end of
-// the maximum depth.
-#[macro_export]
-#[doc(hidden)]
-macro_rules! retrieve_node_mut_closure {
-    (
-        $(
-            $stride: ident;
-            $id: ident;
-            $multi_uniq_id: ident;
-        ),
-    * ) => {$(
-        SearchLevel {
-            f: &|
-                search_level: &SearchLevel<AF, $stride>,
-                nodes,
-                mut level: u8,
-            | {
-                // HASHING FUNCTION
-                let index = Self::hash_node_id($id, level);
-                let node;
-
-<<<<<<< Updated upstream
-                match nodes.0.get(index) {
-                    // This arm only ever gets called in multi-threaded code
-                    // where our thread (running this code *now*), andgot ahead
-                    // of another thread: After the other thread created the
-                    // TreeBitMapNode first, it was overtaken by our thread
-                    // running this method, so our thread enounters an empty node
-                    // in the store.
-                    None => {
-                        let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            $id.get_id().1, level
-                        );
-                        let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            $id.get_id().1, level + 1
-                        );
-                        let node_set = NodeSet::init(next_level - this_level);
-
-                        // See if we can create the node
-                        (node, _) = nodes.0.get_or_init(index, || StoredNode {
-                            node_id: $id,
-                            node: TreeBitMapNode::new(),
-                            node_set
-                        });
-
-                        // We may have lost, and a different node than we
-                        // intended could live here, if so go a level deeper
-                        if $id == node.node_id {
-                            // Nope, its ours or at least the node we need.
-                            let _retry_count = node.node_set.update_rbm_index(
-                                $multi_uniq_id
-                            ).ok();
-
-                            return Some(SizedStrideRef::$stride(&node.node));
-                        };
-                    },
-=======
-                // Read the node from the block pointed to by the Atomic
-                // pointer.
-                // assert!(nodes.0.get(index).is_some());
-                // let stored_node = unsafe {
-                //     &mut nodes.0[index].assume_init_ref()
-                // };
-                // let this_node = stored_node.load(Ordering::Acquire, guard);
-
-                match nodes.0.get(index) {
-                    None => None,
->>>>>>> Stashed changes
-                    Some(this_node) => {
-                        node = this_node;
-                        if $id == this_node.node_id {
-                            // YES, It's the one we're looking for!
-
-                            // Update the rbm_index in this node with the
-                            // multi_uniq_id that the caller specified. This
-                            // is the only atomic operation we need to do
-                            // here. The NodeSet that the index is attached
-                            // to, does not need to be written to, it's part
-                            // of a trie, so it just needs to "exist" (and it
-                            // already does).
-                            let retry_count = this_node.node_set.update_rbm_index(
-                                $multi_uniq_id
-                            ).ok();
-
-                            trace!("Retry_count rbm index {:?}", retry_count);
-                            trace!("add multi uniq id to bitmap index {} for node {}",
-                                $multi_uniq_id, this_node.node
-                            );
-                            return Some(SizedStrideRef::$stride(&this_node.node));
-                        };
-                    }
-                }
-                // It isn't ours. Move one level deeper.
-                level += 1;
-                match <NB as NodeBuckets<AF>>::len_to_store_bits(
-                    $id.get_id().1, level
-                ) {
-                    // on to the next level!
-                    next_bit_shift if next_bit_shift > 0 => {
-                        (search_level.f)(
-                            search_level,
-                            &node.node_set,
-                            level,
-                        )
-                    }
-                    // There's no next level, we found nothing.
-                    _ => None,
-                }
-            }
-        }
-    )*};
-}
-
-#[macro_export]
-#[doc(hidden)]
-macro_rules! store_node_closure {
-    (
-        $(
-            $stride: ident;
-            $id: ident;
-            // $multi_uniq_id: ident;
-            $guard: ident;
-            $back_off: ident;
-        ),
-    *) => {
-        $(
-        SearchLevel {
-            f: &|
-            search_level: &SearchLevel<AF, $stride>,
-            nodes,
-            new_node: TreeBitMapNode<AF, $stride>,
-            multi_uniq_id: u32,
-            mut level: u8,
-            retry_count: u32| {
-<<<<<<< Updated upstream
-=======
-                // println!("-");
->>>>>>> Stashed changes
-                let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level);
-                trace!("{:032b}", $id.get_id().0);
-                trace!("id {:?}", $id.get_id());
-                trace!("multi_uniq_id {}", multi_uniq_id);
-
-                // HASHING FUNCTION
-                let index = Self::hash_node_id($id, level);
-
-<<<<<<< Updated upstream
-                match nodes.0.get(index) {
-                    None => {
-                        // No node exists, so we create one here.
-                        let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level + 1);
-=======
-                // No node exists, so we create one here.
-                let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level + 1);
-
-                let node_set_len = next_level - this_level;
->>>>>>> Stashed changes
-
-                let ptrbitarr = new_node.ptrbitarr.load();
-                let pfxbitarr = new_node.pfxbitarr.load();
-
-                let stored_node = nodes.0.get_or_init(index, || {
-                        if log_enabled!(log::Level::Trace) {
-                            trace!("Empty node found, creating new node {} len{} lvl{}",
-                                $id, $id.get_id().1, level + 1
-                            );
-                            trace!("Next level {}",
-                                next_level
-                            );
-                            trace!("Creating space for {} nodes",
-                                if next_level >= this_level { 1 << (next_level - this_level) } else { 1 }
-                            );
-                        }
-
-                        trace!("multi uniq id {}", multi_uniq_id);
-
-<<<<<<< Updated upstream
-                        let node_set = NodeSet::init(next_level - this_level);
-
-                        let ptrbitarr = new_node.ptrbitarr.load();
-                        let pfxbitarr = new_node.pfxbitarr.load();
-
-                        let (stored_node, its_us) = nodes.0.get_or_init(
-                            index,
-                            || StoredNode {
-                                node_id: $id,
-                                node: new_node,
-                                node_set
-                            }
-                        );
-
-                        if stored_node.node_id == $id {
-                            stored_node.node_set.update_rbm_index(
-                                multi_uniq_id
-                            )?;
-
-                            if !its_us && ptrbitarr != 0 {
-                                stored_node.node.ptrbitarr.merge_with(ptrbitarr);
-                            }
-
-                            if !its_us && pfxbitarr != 0 {
-                                stored_node.node.pfxbitarr.merge_with(pfxbitarr);
-                            }
-=======
-                        // let node_set = if next_level > 0 {
-                        //     NodeSet::init((1 << (next_level - this_level)) as usize )
-                        // } else {
-                        //     NodeSet(
-                        //         Box::new([]),
-                        //         std::sync::RwLock::new(RoaringBitmap::new())
-                        //     )
-                        // }
-                        StoredNode {
-                            node_id: $id,
-                            node: TreeBitMapNode::<AF, $stride>::empty(),
-                            node_set: NodeSet::init(node_set_len)
->>>>>>> Stashed changes
-                        }
-                    });
-
-<<<<<<< Updated upstream
-                        return Ok(($id, retry_count));
-                    }
-                    Some(stored_node) => {
-                        // A node exists, might be ours, might be
-                        // another one.
-
-                        if log_enabled!(log::Level::Trace) {
-                            trace!("
-                                {} store: Node here exists {:?}",
-                                    std::thread::current().name().unwrap_or("unnamed-thread"),
-                                    stored_node.node_id
-                            );
-                            trace!("node_id {:?}", stored_node.node_id.get_id());
-                            trace!("node_id {:032b}", stored_node.node_id.get_id().0);
-                            trace!("id {}", $id);
-                            trace!("     id {:032b}", $id.get_id().0);
-                        }
-
-                        // See if somebody beat us to creating our
-                        // node already, if so, we still need to do
-                        // work: we have to update the bitmap index
-                        // with the multi_uniq_id we've got from the
-                        // caller.
-                        if $id == stored_node.node_id {
-                            stored_node.node_set.update_rbm_index(
-                                multi_uniq_id
-                            )?;
-
-                            if new_node.ptrbitarr.load() != 0 {
-                                stored_node.node.ptrbitarr.merge_with(new_node.ptrbitarr.load());
-                            }
-                            if new_node.pfxbitarr.load() != 0 {
-                                stored_node.node.pfxbitarr.merge_with(new_node.pfxbitarr.load());
-                            }
-
-                            return Ok(($id, retry_count));
-                        } else {
-                            // it's not "our" node, make a (recursive)
-                            // call to create it.
-                            level += 1;
-                            trace!("Collision with node_id {}, move to next level: {} len{} next_lvl{} index {}",
-                                stored_node.node_id, $id, $id.get_id().1, level, index
-                            );
-
-                            return match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
-                                // on to the next level!
-                                next_bit_shift if next_bit_shift > 0 => {
-                                    (search_level.f)(
-                                        search_level,
-                                        &stored_node.node_set,
-                                        new_node,
-                                        multi_uniq_id,
-                                        level,
-                                        retry_count
-                                    )
-                                }
-                                // There's no next level!
-                                _ => panic!("out of storage levels, current level is {}", level),
-                            }
-=======
-                if stored_node.node_id == $id {
-                    stored_node.node_set.update_rbm_index(
-                        multi_uniq_id, $guard
-                    )?;
-
-                    stored_node.node.ptrbitarr.merge_with(ptrbitarr);
-                    stored_node.node.pfxbitarr.merge_with(pfxbitarr);
-
-                    Ok(($id, retry_count))
-                } else {
-                    // it's not "our" node, make a (recursive)
-                    // call to create it.
-                    level += 1;
-                    trace!("Collision with node_id {}, move to next level: {} len{} next_lvl{} index {}",
-                        stored_node.node_id, $id, $id.get_id().1, level, index
-                    );
-
-                    return match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
-                        // on to the next level!
-                        next_bit_shift if next_bit_shift > 0 => {
-                            (search_level.f)(
-                                search_level,
-                                &stored_node.node_set,
-                                new_node,
-                                multi_uniq_id,
-                                level,
-                                retry_count
-                            )
->>>>>>> Stashed changes
-                        }
-                        // There's no next level!
-                        _ => panic!("out of storage levels, current level is {}", level),
-                    }
-                }
-            }
-        }
-    )*
-    };
-}
diff --git a/src/local_array/store/oncebox.rs b/src/local_array/store/oncebox.rs
deleted file mode 100644
index 228c8b60..00000000
--- a/src/local_array/store/oncebox.rs
+++ /dev/null
@@ -1,210 +0,0 @@
-use std::ptr::null_mut;
-use std::slice;
-use std::sync::atomic::{AtomicPtr, Ordering};
-
-#[derive(Debug, Default)]
-pub struct OnceBox<T> {
-    ptr: AtomicPtr<T>,
-}
-
-impl<T> OnceBox<T> {
-    pub fn new() -> Self {
-        Self {
-            ptr: AtomicPtr::new(null_mut()),
-        }
-    }
-
-    pub fn get(&self) -> Option<&T> {
-        let ptr = self.ptr.load(Ordering::Relaxed);
-        if ptr.is_null() {
-            None
-        } else {
-            Some(unsafe { &*ptr })
-        }
-    }
-
-    pub fn get_or_init(&self, create: impl FnOnce() -> T) -> (&T, bool) {
-        let mut its_us = false;
-        if let Some(res) = self.get() {
-            return (res, its_us);
-        }
-        let ptr = Box::leak(Box::new(create()));
-        let res = match self.ptr.compare_exchange(
-            null_mut(),
-            ptr,
-            Ordering::Acquire,
-            Ordering::Relaxed,
-        ) {
-            Ok(current) => {
-                // We set the new value, return it.
-                assert!(current.is_null());
-                its_us = true;
-                ptr as *const _
-            }
-            Err(current) => {
-                // `current` is the real value we need to drop our value.
-                assert!(!current.is_null());
-                let _ = unsafe { Box::from_raw(ptr) };
-                current as *const _
-            }
-        };
-        (unsafe { &*res }, its_us)
-    }
-}
-
-impl<T> Drop for OnceBox<T> {
-    fn drop(&mut self) {
-        let ptr = self.ptr.swap(null_mut(), Ordering::Relaxed);
-        if !ptr.is_null() {
-            let _ = unsafe { Box::from_raw(ptr) };
-        }
-    }
-}
-
-<<<<<<< Updated upstream
-#[derive(Debug, Default)]
-pub struct OnceBoxSlice<T> {
-    ptr: AtomicPtr<OnceBox<T>>,
-    p2_size: u8,
-}
-
-impl<T> OnceBoxSlice<T> {
-    pub fn new(p2_size: u8) -> Self {
-        Self {
-            ptr: AtomicPtr::new(null_mut()),
-            p2_size,
-=======
-#[derive(Debug)]
-pub struct OnceBoxSlice<T> {
-    ptr: AtomicPtr<OnceBox<T>>,
-    pow2_size: u8,
-}
-
-impl<T> OnceBoxSlice<T> {
-    pub fn new(size: u8) -> Self {
-        Self {
-            ptr: AtomicPtr::new(null_mut()),
-            pow2_size: 1 << size as usize,
->>>>>>> Stashed changes
-        }
-    }
-
-    pub fn is_null(&self) -> bool {
-        self.ptr.load(Ordering::Relaxed).is_null()
-    }
-
-    pub fn get(&self, idx: usize) -> Option<&T> {
-        let ptr = self.ptr.load(Ordering::Relaxed);
-        if ptr.is_null() {
-            None
-        } else {
-<<<<<<< Updated upstream
-            let slice =
-                unsafe { slice::from_raw_parts(ptr, 1 << self.p2_size) };
-=======
-            let slice = unsafe {
-                slice::from_raw_parts(ptr, 1 << self.pow2_size as usize)
-            };
->>>>>>> Stashed changes
-            slice.get(idx).and_then(|inner| inner.get())
-        }
-    }
-
-<<<<<<< Updated upstream
-    pub fn get_or_init(
-        &self,
-        idx: usize,
-        create: impl FnOnce() -> T,
-    ) -> (&T, bool) {
-        // assert!(idx < (1 << self.p2_size));
-=======
-    pub fn is_null(&self) -> bool {
-        self.ptr.load(Ordering::Relaxed).is_null()
-    }
-
-    pub fn get_or_init(&self, idx: usize, create: impl FnOnce() -> T) -> &T {
->>>>>>> Stashed changes
-        let slice = self.get_or_make_slice();
-        slice[idx].get_or_init(create)
-    }
-
-    fn get_or_make_slice(&self) -> &[OnceBox<T>] {
-        let ptr = self.ptr.load(Ordering::Relaxed);
-<<<<<<< Updated upstream
-        if !ptr.is_null() {
-            return unsafe { slice::from_raw_parts(ptr, 1 << self.p2_size) };
-        }
-
-        // Create a slice, set it, get again.
-        let mut vec = Vec::with_capacity(1 << self.p2_size);
-        for _ in 0..(1 << self.p2_size) {
-=======
-        if ptr != null_mut() {
-            return unsafe {
-                slice::from_raw_parts(ptr, 1 << self.pow2_size as usize)
-            };
-        }
-
-        // Create a slice, set it, get again.
-        let mut vec = Vec::with_capacity(1 << self.pow2_size as usize);
-        for _ in 0..(1 << self.pow2_size) {
->>>>>>> Stashed changes
-            vec.push(OnceBox::new())
-        }
-        // Convert Vec<[OnceBox<T>] -> Box<[OnceBox<T>] -> &mut [OnceBox<T>]
-        //  -> *mut OnceBox<T>
-        let ptr = Box::leak(vec.into_boxed_slice()).as_mut_ptr();
-        let res = match self.ptr.compare_exchange(
-            null_mut(),
-            ptr,
-            Ordering::Acquire,
-            Ordering::Relaxed,
-        ) {
-            Ok(current) => {
-                // We set the new value, return it.
-                assert!(current.is_null());
-                ptr
-            }
-            Err(current) => {
-                // There was a value already: current. Drop our new thing and
-                // return current.
-                assert!(!current.is_null());
-                let _ = unsafe {
-                    Box::from_raw(slice::from_raw_parts_mut(
-                        ptr,
-<<<<<<< Updated upstream
-                        1 << self.p2_size,
-=======
-                        self.pow2_size as usize,
->>>>>>> Stashed changes
-                    ))
-                };
-                current
-            }
-        };
-
-<<<<<<< Updated upstream
-        unsafe { slice::from_raw_parts(res, 1 << self.p2_size) }
-=======
-        unsafe { slice::from_raw_parts(res, self.pow2_size as usize) }
->>>>>>> Stashed changes
-    }
-}
-
-impl<T> Drop for OnceBoxSlice<T> {
-    fn drop(&mut self) {
-        let ptr = self.ptr.swap(null_mut(), Ordering::Relaxed);
-        if !ptr.is_null() {
-            let _ = unsafe {
-                Box::from_raw(slice::from_raw_parts_mut(
-                    ptr,
-<<<<<<< Updated upstream
-                    1 << self.p2_size,
-=======
-                    self.pow2_size as usize,
->>>>>>> Stashed changes
-                ))
-            };
-        }
-    }
-}

From dea2c6a7e6428bf399e8100f98f65e54724bf12c Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 18 Dec 2024 15:43:24 +0100
Subject: [PATCH 027/147] clean

---
 src/local_array/in_memory/macros.rs |  11 +-
 src/local_array/in_memory/tree.rs   |   2 +-
 src/local_array/rib/rib.rs          | 698 ++++------------------------
 3 files changed, 85 insertions(+), 626 deletions(-)

diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index b911c124..c953f05a 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -66,7 +66,6 @@ macro_rules! insert_match {
                                 $stride_len,
                                 // the length of the next stride
                                 $self
-                                    .in_memory_tree
                                     .get_stride_sizes()
                                     .get(($level + 1) as usize),
                                 $is_last_stride,
@@ -78,11 +77,11 @@ macro_rules! insert_match {
 
                                     // get a new identifier for the node we're
                                     // going to create.
-                                        let new_id =
-                                            StrideNodeId::new_with_cleaned_id(
-                                                $pfx.get_net(),
-                                                $truncate_len + $nibble_len
-                                           );
+                                    let new_id =
+                                        StrideNodeId::new_with_cleaned_id(
+                                            $pfx.get_net(),
+                                            $truncate_len + $nibble_len
+                                       );
 
                                     // store the new node in the global
                                     // store. It returns the created id
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index ed3351be..3519e3ec 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -1,4 +1,3 @@
-use crate::local_array::bit_span::BitSpan;
 // ----------- THE STORE ----------------------------------------------------
 //
 // The CustomAllocStore provides in-memory storage for the BitTreeMapNodes
@@ -184,6 +183,7 @@ use crate::local_array::bit_span::BitSpan;
 // an actual memory leak in the mt-prefix-store (and even more if they can
 // produce a fix for it).
 
+use crate::local_array::bit_span::BitSpan;
 use crate::local_array::in_memory::atomic_types::StoredNode;
 use crate::prefix_record::Meta;
 use crate::prelude::multi::{NodeSet, PrefixId};
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index fe849321..bc9dd501 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -291,330 +291,7 @@ impl<
         }
     }
 
-    // Yes, we're hating this. But, the root node has no room for a serial of
-    // the prefix 0/0 (the default route), which doesn't even matter, unless,
-    // UNLESS, somebody wants to store a default route. So we have to store a
-    // serial for this prefix. The normal place for a serial of any prefix is
-    // on the pfxvec of its paren. But, hey, guess what, the
-    // default-route-prefix lives *on* the root node, and, you know, the root
-    // node doesn't have a parent. We can:
-    // - Create a type RootTreeBitmapNode with a ptrbitarr with a size one
-    //   bigger than a "normal" TreeBitMapNod for the first stride size. no we
-    //   have to iterate over the root-node type in all matches on
-    //   stride_size, just because we have exactly one instance of the
-    //   RootTreeBitmapNode. So no.
-    // - Make the `get_pfx_index` method on the implementations of the
-    //   `Stride` trait check for a length of zero and branch if it is and
-    //   return the serial of the root node. Now each and every call to this
-    //   method will have to check a condition for exactly one instance of
-    //   RootTreeBitmapNode. So again, no.
-    // - The root node only gets used at the beginning of a search query or an
-    //   insert. So if we provide two specialised methods that will now how to
-    //   search for the default-route prefix and now how to set serial for
-    //  that prefix and make sure we start searching/inserting with one of
-    //   those specialized methods we're good to go.
-    fn update_default_route_prefix_meta(
-        &self,
-        record: PublicRecord<M>,
-        guard: &epoch::Guard,
-    ) -> Result<UpsertReport, PrefixStoreError> {
-        trace!("Updating the default route...");
-
-        if let Some(root_node) = self.in_memory_tree.retrieve_node_mut(
-            self.in_memory_tree.get_root_node_id(),
-            record.multi_uniq_id,
-        ) {
-            match root_node {
-                SizedStrideRef::Stride3(_) => {
-                    self.in_memory_tree
-                        .node_buckets
-                        .get_store3(self.in_memory_tree.get_root_node_id())
-                        .update_rbm_index(record.multi_uniq_id)?;
-                }
-                SizedStrideRef::Stride4(_) => {
-                    self.in_memory_tree
-                        .node_buckets
-                        .get_store4(self.in_memory_tree.get_root_node_id())
-                        .update_rbm_index(record.multi_uniq_id)?;
-                }
-                SizedStrideRef::Stride5(_) => {
-                    self.in_memory_tree
-                        .node_buckets
-                        .get_store5(self.in_memory_tree.get_root_node_id())
-                        .update_rbm_index(record.multi_uniq_id)?;
-                }
-            };
-        };
-
-        self.upsert_prefix(
-            PrefixId::new(AF::zero(), 0),
-            record,
-            // Do not update the path selection for the default route.
-            None,
-            guard,
-        )
-    }
-
-    // // Create a new node in the store with payload `next_node`.
-    // //
-    // // Next node will be ignored if a node with the same `id` already exists,
-    // // but the multi_uniq_id will be added to the rbm_index of the NodeSet.
-    // //
-    // // Returns: a tuple with the node_id of the created node and the number of
-    // // retry_count
-    // #[allow(clippy::type_complexity)]
-    // pub(crate) fn store_node(
-    //     &self,
-    //     id: StrideNodeId<AF>,
-    //     multi_uniq_id: u32,
-    //     next_node: SizedStrideNode<AF>,
-    // ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
-    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-    //         f: &'s dyn Fn(
-    //             &SearchLevel<AF, S>,
-    //             &NodeSet<AF, S>,
-    //             TreeBitMapNode<AF, S>,
-    //             u32, // multi_uniq_id
-    //             u8,  // the store level
-    //             u32, // retry_count
-    //         ) -> Result<
-    //             (StrideNodeId<AF>, u32),
-    //             PrefixStoreError,
-    //         >,
-    //     }
-
-    //     let search_level_3 =
-    //         store_node_closure![Stride3; id; guard; back_off;];
-    //     let search_level_4 =
-    //         store_node_closure![Stride4; id; guard; back_off;];
-    //     let search_level_5 =
-    //         store_node_closure![Stride5; id; guard; back_off;];
-
-    //     if log_enabled!(log::Level::Trace) {
-    //         debug!(
-    //             "{} store: Store node {}: {:?} mui {}",
-    //             std::thread::current().name().unwrap_or("unnamed-thread"),
-    //             id,
-    //             next_node,
-    //             multi_uniq_id
-    //         );
-    //     }
-    //     self.counters.inc_nodes_count();
-
-    //     match next_node {
-    //         SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
-    //             &search_level_3,
-    //             self.in_memory_tree.node_buckets.get_store3(id),
-    //             new_node,
-    //             multi_uniq_id,
-    //             0,
-    //             0,
-    //         ),
-    //         SizedStrideNode::Stride4(new_node) => (search_level_4.f)(
-    //             &search_level_4,
-    //             self.in_memory_tree.node_buckets.get_store4(id),
-    //             new_node,
-    //             multi_uniq_id,
-    //             0,
-    //             0,
-    //         ),
-    //         SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
-    //             &search_level_5,
-    //             self.in_memory_tree.node_buckets.get_store5(id),
-    //             new_node,
-    //             multi_uniq_id,
-    //             0,
-    //             0,
-    //         ),
-    //     }
-    // }
-
-    // #[allow(clippy::type_complexity)]
-    // pub(crate) fn retrieve_node(
-    //     &'a self,
-    //     id: StrideNodeId<AF>,
-    // ) -> Option<SizedStrideRef<'a, AF>> {
-    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-    //         f: &'s dyn for<'a> Fn(
-    //             &SearchLevel<AF, S>,
-    //             &'a NodeSet<AF, S>,
-    //             u8,
-    //         )
-    //             -> Option<SizedStrideRef<'a, AF>>,
-    //     }
-
-    //     let search_level_3 = impl_search_level![Stride3; id;];
-    //     let search_level_4 = impl_search_level![Stride4; id;];
-    //     let search_level_5 = impl_search_level![Stride5; id;];
-
-    //     if log_enabled!(log::Level::Trace) {
-    //         trace!(
-    //             "{} store: Retrieve node {} from l{}",
-    //             std::thread::current().name().unwrap_or("unnamed-thread"),
-    //             id,
-    //             id.get_id().1
-    //         );
-    //     }
-
-    //     match self.get_stride_for_id(id) {
-    //         3 => (search_level_3.f)(
-    //             &search_level_3,
-    //             self.in_memory_tree.node_buckets.get_store3(id),
-    //             0,
-    //         ),
-    //         4 => (search_level_4.f)(
-    //             &search_level_4,
-    //             self.in_memory_tree.node_buckets.get_store4(id),
-    //             0,
-    //         ),
-    //         _ => (search_level_5.f)(
-    //             &search_level_5,
-    //             self.in_memory_tree.node_buckets.get_store5(id),
-    //             0,
-    //         ),
-    //     }
-    // }
-
-    // retrieve a node, but only its bitmap index contains the specified mui.
-    // Used for iterators per mui.
-    // #[allow(clippy::type_complexity)]
-    // pub(crate) fn retrieve_node_for_mui(
-    //     &'a self,
-    //     id: StrideNodeId<AF>,
-    //     // The mui that is tested to be present in the nodes bitmap index
-    //     mui: u32,
-    // ) -> Option<SizedStrideRef<'a, AF>> {
-    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-    //         f: &'s dyn for<'a> Fn(
-    //             &SearchLevel<AF, S>,
-    //             &'a NodeSet<AF, S>,
-    //             u8,
-    //         )
-    //             -> Option<SizedStrideRef<'a, AF>>,
-    //     }
-
-    //     let search_level_3 = impl_search_level_for_mui![Stride3; id; mui;];
-    //     let search_level_4 = impl_search_level_for_mui![Stride4; id; mui;];
-    //     let search_level_5 = impl_search_level_for_mui![Stride5; id; mui;];
-
-    //     if log_enabled!(log::Level::Trace) {
-    //         trace!(
-    //             "{} store: Retrieve node {} from l{} for mui {}",
-    //             std::thread::current().name().unwrap_or("unnamed-thread"),
-    //             id,
-    //             id.get_id().1,
-    //             mui
-    //         );
-    //     }
-
-    //     match self.get_stride_for_id(id) {
-    //         3 => (search_level_3.f)(
-    //             &search_level_3,
-    //             self.in_memory_tree.node_buckets.get_store3(id),
-    //             0,
-    //         ),
-    //         4 => (search_level_4.f)(
-    //             &search_level_4,
-    //             self.in_memory_tree.node_buckets.get_store4(id),
-    //             0,
-    //         ),
-    //         _ => (search_level_5.f)(
-    //             &search_level_5,
-    //             self.in_memory_tree.node_buckets.get_store5(id),
-    //             0,
-    //         ),
-    //     }
-    // }
-
-    // #[allow(clippy::type_complexity)]
-    // pub(crate) fn retrieve_node_mut(
-    //     &'a self,
-    //     id: StrideNodeId<AF>,
-    //     multi_uniq_id: u32,
-    // ) -> Option<SizedStrideRef<'a, AF>> {
-    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-    //         f: &'s dyn for<'a> Fn(
-    //             &SearchLevel<AF, S>,
-    //             &'a NodeSet<AF, S>,
-    //             u8,
-    //         )
-    //             -> Option<SizedStrideRef<'a, AF>>,
-    //     }
-
-    //     let search_level_3 =
-    //         retrieve_node_mut_closure![Stride3; id; multi_uniq_id;];
-    //     let search_level_4 =
-    //         retrieve_node_mut_closure![Stride4; id; multi_uniq_id;];
-    //     let search_level_5 =
-    //         retrieve_node_mut_closure![Stride5; id; multi_uniq_id;];
-
-    //     if log_enabled!(log::Level::Trace) {
-    //         trace!(
-    //             "{} store: Retrieve node mut {} from l{}",
-    //             std::thread::current().name().unwrap_or("unnamed-thread"),
-    //             id,
-    //             id.get_id().1
-    //         );
-    //     }
-
-    //     match self.in_memory_tree.node_buckets.get_stride_for_id(id) {
-    //         3 => (search_level_3.f)(
-    //             &search_level_3,
-    //             self.in_memory_tree.node_buckets.get_store3(id),
-    //             0,
-    //         ),
-
-    //         4 => (search_level_4.f)(
-    //             &search_level_4,
-    //             self.in_memory_tree.node_buckets.get_store4(id),
-    //             0,
-    //         ),
-    //         _ => (search_level_5.f)(
-    //             &search_level_5,
-    //             self.in_memory_tree.node_buckets.get_store5(id),
-    //             0,
-    //         ),
-    //     }
-    // }
-
-    // pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
-    //     StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0)
-    // }
-
-    pub fn get_nodes_count(&self) -> usize {
-        self.in_memory_tree.get_nodes_count()
-    }
-
-    // Prefixes related methods
-
-    // THE CRITICAL SECTION
-    //
-    // CREATING OR UPDATING A PREFIX IN THE STORE
-    //
-    // YES, THE MAGIC HAPPENS HERE!
-    //
-    // This uses the TAG feature of crossbeam_utils::epoch to ensure that we
-    // are not overwriting a prefix meta-data that already has been created
-    // or was updated by another thread.
-    //
-    // Our plan:
-    //
-    // 1. LOAD
-    //    Load the current prefix and meta-data from the store if any.
-    // 2. INSERT
-    //    If there is no current meta-data, create it.
-    // 3. UPDATE
-    //    If there is a prefix, meta-data combo, then load it and merge
-    //    the existing meta-dat with our meta-data using the `MergeUpdate`
-    //    trait (a so-called 'Read-Copy-Update').
-    // 4. SUCCESS
-    //    See if we can successfully store the updated meta-data in the store.
-    // 5. DONE
-    //    If Step 4 succeeded we're done!
-    // 6. FAILURE - REPEAT
-    //    If Step 4 failed we're going to do the whole thing again.
-
-    pub(crate) fn upsert_prefix(
+    fn upsert_prefix(
         &self,
         prefix: PrefixId<AF>,
         record: PublicRecord<M>,
@@ -704,6 +381,74 @@ impl<
         })
     }
 
+    // Yes, we're hating this. But, the root node has no room for a serial of
+    // the prefix 0/0 (the default route), which doesn't even matter, unless,
+    // UNLESS, somebody wants to store a default route. So we have to store a
+    // serial for this prefix. The normal place for a serial of any prefix is
+    // on the pfxvec of its paren. But, hey, guess what, the
+    // default-route-prefix lives *on* the root node, and, you know, the root
+    // node doesn't have a parent. We can:
+    // - Create a type RootTreeBitmapNode with a ptrbitarr with a size one
+    //   bigger than a "normal" TreeBitMapNod for the first stride size. no we
+    //   have to iterate over the root-node type in all matches on
+    //   stride_size, just because we have exactly one instance of the
+    //   RootTreeBitmapNode. So no.
+    // - Make the `get_pfx_index` method on the implementations of the
+    //   `Stride` trait check for a length of zero and branch if it is and
+    //   return the serial of the root node. Now each and every call to this
+    //   method will have to check a condition for exactly one instance of
+    //   RootTreeBitmapNode. So again, no.
+    // - The root node only gets used at the beginning of a search query or an
+    //   insert. So if we provide two specialised methods that will now how to
+    //   search for the default-route prefix and now how to set serial for
+    //  that prefix and make sure we start searching/inserting with one of
+    //   those specialized methods we're good to go.
+    fn update_default_route_prefix_meta(
+        &self,
+        record: PublicRecord<M>,
+        guard: &epoch::Guard,
+    ) -> Result<UpsertReport, PrefixStoreError> {
+        trace!("Updating the default route...");
+
+        if let Some(root_node) = self.in_memory_tree.retrieve_node_mut(
+            self.in_memory_tree.get_root_node_id(),
+            record.multi_uniq_id,
+        ) {
+            match root_node {
+                SizedStrideRef::Stride3(_) => {
+                    self.in_memory_tree
+                        .node_buckets
+                        .get_store3(self.in_memory_tree.get_root_node_id())
+                        .update_rbm_index(record.multi_uniq_id)?;
+                }
+                SizedStrideRef::Stride4(_) => {
+                    self.in_memory_tree
+                        .node_buckets
+                        .get_store4(self.in_memory_tree.get_root_node_id())
+                        .update_rbm_index(record.multi_uniq_id)?;
+                }
+                SizedStrideRef::Stride5(_) => {
+                    self.in_memory_tree
+                        .node_buckets
+                        .get_store5(self.in_memory_tree.get_root_node_id())
+                        .update_rbm_index(record.multi_uniq_id)?;
+                }
+            };
+        };
+
+        self.upsert_prefix(
+            PrefixId::new(AF::zero(), 0),
+            record,
+            // Do not update the path selection for the default route.
+            None,
+            guard,
+        )
+    }
+
+    pub fn get_nodes_count(&self) -> usize {
+        self.in_memory_tree.get_nodes_count()
+    }
+
     pub(crate) fn withdrawn_muis_bmin(
         &'a self,
         guard: &'a Guard,
@@ -863,216 +608,27 @@ impl<
         pfx.record_map.get_filtered_records(mui, bmin)
     }
 
-    // This function is used by the upsert_prefix function above.
-    //
-    // We're using a Chained Hash Table and this function returns one of:
-    // - a StoredPrefix that already exists for this search_prefix_id
-    // - the Last StoredPrefix in the chain.
-    // - an error, if no StoredPrefix whatsoever can be found in the store.
-    //
-    // The error condition really shouldn't happen, because that basically
-    // means the root node for that particular prefix length doesn't exist.
-    // #[allow(clippy::type_complexity)]
-    // pub(crate) fn non_recursive_retrieve_prefix_mut(
-    //     &'a self,
-    //     search_prefix_id: PrefixId<AF>,
-    // ) -> (&'a StoredPrefix<AF, M>, bool) {
-    //     trace!("non_recursive_retrieve_prefix_mut_with_guard");
-    //     let mut prefix_set = self
-    //         .in_memory_tree
-    //         .prefix_buckets
-    //         .get_root_prefix_set(search_prefix_id.get_len());
-    //     let mut level: u8 = 0;
-
-    //     trace!("root prefix_set {:?}", prefix_set);
-    //     loop {
-    //         // HASHING FUNCTION
-    //         let index = TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(
-    //             search_prefix_id,
-    //             level,
-    //         );
-
-    //         // probe the slot with the index that's the result of the hashing.
-    //         // let locked_prefix = prefix_set.0.get(index);
-    //         let stored_prefix = match prefix_set.0.get(index) {
-    //             Some(p) => {
-    //                 trace!("prefix set found.");
-    //                 (p, true)
-    //             }
-    //             None => {
-    //                 // We're at the end of the chain and haven't found our
-    //                 // search_prefix_id anywhere. Return the end-of-the-chain
-    //                 // StoredPrefix, so the caller can attach a new one.
-    //                 trace!(
-    //                     "no record. returning last found record in level
-    //                     {}, with index {}.",
-    //                     level,
-    //                     index
-    //                 );
-    //                 let index = TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(
-    //                     search_prefix_id,
-    //                     level,
-    //                 );
-    //                 trace!("calculate next index {}", index);
-    //                 let var_name = (
-    //                     prefix_set
-    //                         .0
-    //                         .get_or_init(index, || {
-    //                             StoredPrefix::new::<PB>(
-    //                                 PrefixId::new(
-    //                                     search_prefix_id.get_net(),
-    //                                     search_prefix_id.get_len(),
-    //                                 ),
-    //                                 level,
-    //                             )
-    //                         })
-    //                         .0,
-    //                     false,
-    //                 );
-    //                 var_name
-    //             }
-    //         };
-
-    //         if search_prefix_id == stored_prefix.0.prefix {
-    //             // GOTCHA!
-    //             // Our search-prefix is stored here, so we're returning
-    //             // it, so its PrefixRecord can be updated by the caller.
-    //             trace!("found requested prefix {:?}", search_prefix_id);
-    //             return stored_prefix;
-    //         } else {
-    //             // A Collision. Follow the chain.
-    //             level += 1;
-    //             prefix_set = &stored_prefix.0.next_bucket;
-    //             continue;
-    //         }
-    //     }
-    // }
-
-    // // This function is used by the match_prefix, and [more|less]_specifics
-    // // public methods on the TreeBitMap (indirectly).
-    // #[allow(clippy::type_complexity)]
-    // pub fn non_recursive_retrieve_prefix(
-    //     &'a self,
-    //     id: PrefixId<AF>,
-    // ) -> (
-    //     Option<&'a StoredPrefix<AF, M>>,
-    //     Option<(
-    //         PrefixId<AF>,
-    //         u8,
-    //         &'a PrefixSet<AF, M>,
-    //         [Option<(&'a PrefixSet<AF, M>, usize)>; 32],
-    //         usize,
-    //     )>,
-    // ) {
-    //     let mut prefix_set = self
-    //         .in_memory_tree
-    //         .prefix_buckets
-    //         .get_root_prefix_set(id.get_len());
-    //     let mut parents = [None; 32];
-    //     let mut level: u8 = 0;
-    //     let backoff = Backoff::new();
-
-    //     loop {
-    //         // The index of the prefix in this array (at this len and
-    //         // level) is calculated by performing the hash function
-    //         // over the prefix.
-
-    //         // HASHING FUNCTION
-    //         let index =
-    //             TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(id, level);
-
-    //         if let Some(stored_prefix) = prefix_set.0.get(index) {
-    //             if id == stored_prefix.get_prefix_id() {
-    //                 trace!("found requested prefix {:?}", id);
-    //                 parents[level as usize] = Some((prefix_set, index));
-    //                 return (
-    //                     Some(stored_prefix),
-    //                     Some((id, level, prefix_set, parents, index)),
-    //                 );
-    //             };
-    //             // Advance to the next level.
-
-    //             prefix_set = &stored_prefix.next_bucket;
-    //             level += 1;
-    //             backoff.spin();
-    //             continue;
-    //         }
-
-    //         trace!("no prefix found for {:?}", id);
-    //         parents[level as usize] = Some((prefix_set, index));
-    //         return (None, Some((id, level, prefix_set, parents, index)));
-    //     }
-    // }
-
-    // #[allow(dead_code)]
-    // fn remove_prefix(&mut self, index: PrefixId<AF>) -> Option<M> {
-    //     match index.is_empty() {
-    //         false => self.in_memory_tree.prefix_buckets.remove(index),
-    //         true => None,
-    //     }
-    // }
-
-    pub fn get_prefixes_count(&self) -> usize {
-        self.counters.get_prefixes_count().iter().sum()
+    pub fn get_prefixes_count(&self) -> (usize, usize) {
+        (
+            self.counters.get_prefixes_count().iter().sum(),
+            self.in_memory_tree.get_prefixes_count(),
+        )
     }
 
-    pub fn get_prefixes_count_for_len(&self, len: u8) -> usize {
-        self.counters.get_prefixes_count()[len as usize]
+    pub fn get_prefixes_count_for_len(&self, len: u8) -> (usize, usize) {
+        (
+            self.counters.get_prefixes_count()[len as usize],
+            self.in_memory_tree.get_prefixes_count(),
+        )
     }
 
     // Stride related methods
 
-    // pub(crate) fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8 {
-    //     self.in_memory_tree.node_buckets.get_stride_for_id(id)
-    // }
-
     // Pass through the in_memory stride sizes, for printing purposes
     pub fn get_stride_sizes(&self) -> &[u8] {
         self.in_memory_tree.node_buckets.get_stride_sizes()
     }
 
-    // pub(crate) fn get_strides_len() -> u8 {
-    //     NB::get_strides_len()
-    // }
-
-    // pub(crate) fn get_first_stride_size() -> u8 {
-    //     NB::get_first_stride_size()
-    // }
-
-    // Calculates the id of the node that COULD host a prefix in its
-    // ptrbitarr.
-    // pub(crate) fn get_node_id_for_prefix(
-    //     &self,
-    //     prefix: &PrefixId<AF>,
-    // ) -> (StrideNodeId<AF>, BitSpan) {
-    //     let mut acc = 0;
-    //     for i in self.get_stride_sizes() {
-    //         acc += *i;
-    //         if acc >= prefix.get_len() {
-    //             let node_len = acc - i;
-    //             return (
-    //                 StrideNodeId::new_with_cleaned_id(
-    //                     prefix.get_net(),
-    //                     node_len,
-    //                 ),
-    //                 // NOT THE HASHING FUNCTION!
-    //                 // Do the right shift in a checked manner, for the sake
-    //                 // of 0/0. A search for 0/0 will perform a 0 << MAX_LEN,
-    //                 // which will panic in debug mode (undefined behaviour
-    //                 // in prod).
-    //                 BitSpan::new(
-    //                     ((prefix.get_net() << node_len).checked_shr_or_zero(
-    //                         (AF::BITS - (prefix.get_len() - node_len)).into(),
-    //                     ))
-    //                     .dangerously_truncate_to_u32(),
-    //                     prefix.get_len() - node_len,
-    //                 ),
-    //             );
-    //         }
-    //     }
-    //     panic!("prefix length for {:?} is too long", prefix);
-    // }
-
     //-------- Persistence ---------------------------------------------------
 
     pub fn persist_strategy(&self) -> PersistStrategy {
@@ -1149,102 +705,6 @@ impl<
             0
         }
     }
-
-    // ------- THE HASHING FUNCTION -----------------------------------------
-
-    // Ok, so hashing is really hard, but we're keeping it simple, and
-    // because we're keeping it simple we're having lots of collisions, but
-    // we don't care!
-    //
-    // We're using a part of bitarray representation of the address part of
-    // a prefix the as the hash. Sounds complicated, but isn't.
-    // Suppose we have an IPv4 prefix, say 130.24.55.0/24.
-    // The address part is 130.24.55.0 or as a bitarray that would be:
-    //
-    // pos  0    4    8    12   16   20   24   28
-    // bit  1000 0010 0001 1000 0011 0111 0000 0000
-    //
-    // First, we're discarding the bits after the length of the prefix, so
-    // we'll have:
-    //
-    // pos  0    4    8    12   16   20
-    // bit  1000 0010 0001 1000 0011 0111
-    //
-    // Now we're dividing this bitarray into one or more levels. A level can
-    // be an arbitrary number of bits between 1 and the length of the prefix,
-    // but the number of bits summed over all levels should be exactly the
-    // prefix length. So in our case they should add up to 24. A possible
-    // division could be: 4, 4, 4, 4, 4, 4. Another one would be: 12, 12. The
-    // actual division being used is described in the function
-    // `<NB>::get_bits_for_len` in the `rotonda-macros` crate. Each level has
-    // its own hash, so for our example prefix this would be:
-    //
-    // pos   0    4    8    12   16   20
-    // level 0              1
-    // hash  1000 0010 0001 1000 0011 0111
-    //
-    // level 1 hash: 1000 0010 0001
-    // level 2 hash: 1000 0011 0011
-    //
-    // The hash is now converted to a usize integer, by shifting it all the
-    // way to the right in a u32 and then converting to a usize. Why a usize
-    // you ask? Because the hash is used by the CustomAllocStorage as the
-    // index to the array for that specific prefix length and level.
-    // So for our example this means that the hash on level 1 is now 0x821
-    // (decimal 2081) and the hash on level 2 is 0x833 (decimal 2099).
-    // Now, if we only consider the hash on level 1 and that we're going to
-    // use that as the index to the array that stores all prefixes, you'll
-    // notice very quickly that all prefixes starting with 130.[16..31] will
-    // cause a collision: they'll all point to the same array element. These
-    // collisions are resolved by creating a linked list from each array
-    // element, where each element in the list has an array of its own that
-    // uses the hash function with the level incremented.
-
-    // pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
-    //     // And, this is all of our hashing function.
-    //     let last_level = if level > 0 {
-    //         <NB>::len_to_store_bits(id.get_id().1, level - 1)
-    //     } else {
-    //         0
-    //     };
-    //     let this_level = <NB>::len_to_store_bits(id.get_id().1, level);
-    //     trace!("bits division {}", this_level);
-    //     trace!(
-    //         "calculated index ({} << {}) >> {}",
-    //         id.get_id().0,
-    //         last_level,
-    //         ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-    //     );
-    //     // HASHING FUNCTION
-    //     ((id.get_id().0 << last_level)
-    //         >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
-    //         .dangerously_truncate_to_u32() as usize
-    // }
-
-    // pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
-    //     // And, this is all of our hashing function.
-    //     let last_level = if level > 0 {
-    //         <PB>::get_bits_for_len(id.get_len(), level - 1)
-    //     } else {
-    //         0
-    //     };
-    //     let this_level = <PB>::get_bits_for_len(id.get_len(), level);
-    //     trace!(
-    //         "bits division {}; no of bits {}",
-    //         this_level,
-    //         this_level - last_level
-    //     );
-    //     trace!(
-    //         "calculated index ({} << {}) >> {}",
-    //         id.get_net(),
-    //         last_level,
-    //         ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-    //     );
-    //     // HASHING FUNCTION
-    //     ((id.get_net() << last_level)
-    //         >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
-    //         .dangerously_truncate_to_u32() as usize
-    // }
 }
 
 impl<

From 3dc108624754f7cf3623f2deb6d086f8a3322c40 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 18 Dec 2024 18:26:13 +0100
Subject: [PATCH 028/147] counters split out over in-mem, persisted

---
 proc_macros/src/lib.rs              | 20 ++++---
 src/bin/load_mrt.rs                 |  2 +-
 src/local_array/in_memory/tree.rs   |  2 +-
 src/local_array/persist/lsm_tree.rs | 40 ++++++++-----
 src/local_array/rib/rib.rs          | 91 +++++++++++++++++++++++++----
 src/prelude/mod.rs                  |  3 +-
 tests/full-table.rs                 |  2 +-
 7 files changed, 124 insertions(+), 36 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 97c96c6c..dea0e73e 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -1497,7 +1497,7 @@ pub fn create_store(
             ///
             /// Note that this method will actually traverse the complete
             /// tree.
-            pub fn prefixes_count(&self) -> usize {
+            pub fn prefixes_count(&self) -> UpsertCounters {
                 self.v4.get_prefixes_count()
                 + self.v6.get_prefixes_count()
             }
@@ -1507,7 +1507,7 @@ pub fn create_store(
             /// Note that this counter may be lower than the actual
             /// number in the store, due to contention at the time of
             /// reading the value.
-            pub fn prefixes_v4_count(&self) -> usize {
+            pub fn prefixes_v4_count(&self) -> UpsertCounters {
                 self.v4.get_prefixes_count()
             }
 
@@ -1517,16 +1517,17 @@ pub fn create_store(
             /// Note that this counter may be lower than the actual
             /// number in the store, due to contention at the time of
             /// reading the value.
-            pub fn prefixes_v4_count_for_len(&self, len: u8) -> usize {
-                self.v4.get_prefixes_count_for_len(len)
-            }
+            pub fn prefixes_v4_count_for_len(&self, len: u8)
+                -> UpsertCounters {
+                    self.v4.get_prefixes_count_for_len(len)
+                }
 
             /// Returns the number of all IPv6 prefixes in the store.
             ///
             /// Note that this counter may be lower than the actual
             /// number in the store, due to contention at the time of
             /// reading the value.
-            pub fn prefixes_v6_count(&self) -> usize {
+            pub fn prefixes_v6_count(&self) -> UpsertCounters {
                 self.v6.get_prefixes_count()
             }
 
@@ -1536,9 +1537,10 @@ pub fn create_store(
             /// Note that this counter may be lower than the actual
             /// number in the store, due to contention at the time of
             /// reading the value.
-            pub fn prefixes_v6_count_for_len(&self, len: u8) -> usize {
-                self.v6.get_prefixes_count_for_len(len)
-            }
+            pub fn prefixes_v6_count_for_len(&self, len: u8)
+                -> UpsertCounters {
+                    self.v6.get_prefixes_count_for_len(len)
+                }
 
             /// Returns the number of nodes in the store.
             ///
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index 14c17107..c8d1692a 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -563,7 +563,7 @@ fn main() {
     if let Some(store) = store {
         println!("store in-memory counters");
         println!("------------------------");
-        println!("prefixes:\t\t\t{:?}\n", store.prefixes_count());
+        println!("prefixes:\t\t\t{:?}\n", store.prefixes_count().in_memory());
 
         println!("store persistence counters");
         println!("--------------------------");
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 3519e3ec..6a47f482 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -199,7 +199,7 @@ use super::atomic_types::{
     NodeBuckets, PrefixBuckets, PrefixSet, StoredPrefix,
 };
 use crate::af::AddressFamily;
-use crate::rib::{Counters, Rib};
+use crate::rib::Counters;
 use crate::{
     impl_search_level, impl_search_level_for_mui, retrieve_node_mut_closure,
     store_node_closure,
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 64574963..9ff09d3e 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -6,6 +6,7 @@ use std::path::Path;
 use lsm_tree::AbstractTree;
 
 use crate::local_array::types::{PrefixId, RouteStatus};
+use crate::rib::Counters;
 use crate::{AddressFamily, MatchType, Meta, PublicRecord, QueryResult};
 
 pub struct PersistTree<
@@ -17,7 +18,11 @@ pub struct PersistTree<
     // The size in bytes of the complete key in the persisted storage, this
     // is PREFIX_SIZE bytes (4; 16) + mui size (4) + ltime (8)
     const KEY_SIZE: usize,
->(lsm_tree::Tree, PhantomData<AF>);
+> {
+    tree: lsm_tree::Tree,
+    counters: Counters,
+    _af: PhantomData<AF>,
+}
 
 impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
@@ -25,14 +30,15 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     pub fn new(
         persist_path: &Path,
     ) -> PersistTree<AF, PREFIX_SIZE, KEY_SIZE> {
-        PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>(
-            lsm_tree::Config::new(persist_path).open().unwrap(),
-            PhantomData,
-        )
+        PersistTree::<AF, PREFIX_SIZE, KEY_SIZE> {
+            tree: lsm_tree::Config::new(persist_path).open().unwrap(),
+            counters: Counters::default(),
+            _af: PhantomData,
+        }
     }
 
     pub fn insert(&self, key: [u8; KEY_SIZE], value: &[u8]) -> (u32, u32) {
-        self.0.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
+        self.tree.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
     }
 
     pub fn get_records_for_prefix<M: Meta>(
@@ -40,7 +46,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         prefix: PrefixId<AF>,
     ) -> Vec<PublicRecord<M>> {
         let prefix_b = &prefix.as_bytes::<PREFIX_SIZE>();
-        (*self.0.prefix(prefix_b))
+        (*self.tree.prefix(prefix_b))
             .into_iter()
             .map(|kv| {
                 let kv = kv.unwrap();
@@ -59,7 +65,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         key: &[u8],
     ) -> Vec<(inetnum::addr::Prefix, PublicRecord<M>)> {
-        (*self.0.prefix(key))
+        (*self.tree.prefix(key))
             .into_iter()
             .map(|kv| {
                 let kv = kv.unwrap();
@@ -105,11 +111,11 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     }
 
     pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
-        let segment = self.0.flush_active_memtable(0);
+        let segment = self.tree.flush_active_memtable(0);
 
         if let Ok(Some(segment)) = segment {
-            self.0.register_segments(&[segment])?;
-            self.0.compact(
+            self.tree.register_segments(&[segment])?;
+            self.tree.compact(
                 std::sync::Arc::new(lsm_tree::compaction::Leveled::default()),
                 0,
             )?;
@@ -119,11 +125,19 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     }
 
     pub fn approximate_len(&self) -> usize {
-        self.0.approximate_len()
+        self.tree.approximate_len()
     }
 
     pub fn disk_space(&self) -> u64 {
-        self.0.disk_space()
+        self.tree.disk_space()
+    }
+
+    pub fn get_prefixes_count(&self) -> usize {
+        self.counters.get_prefixes_count().iter().sum()
+    }
+
+    pub fn get_prefixes_count_for_len(&self, len: u8) -> usize {
+        self.counters.get_prefixes_count()[len as usize]
     }
 
     #[cfg(feature = "persist")]
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index bc9dd501..904e03fc 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -57,8 +57,12 @@ impl StoreConfig {
 
 #[derive(Debug)]
 pub struct Counters {
+    // number of created nodes in the in-mem tree
     nodes: AtomicUsize,
+    // number of unique prefixes in the store
     prefixes: [AtomicUsize; 129],
+    // number of unique (prefix, mui) values inserted in the in-mem tree
+    routes: AtomicUsize,
 }
 
 impl Counters {
@@ -98,6 +102,10 @@ impl Counters {
             })
             .collect()
     }
+
+    pub fn inc_routes_count(&self) {
+        self.routes.fetch_add(1, Ordering::Relaxed);
+    }
 }
 
 impl Default for Counters {
@@ -110,6 +118,59 @@ impl Default for Counters {
         Self {
             nodes: AtomicUsize::new(0),
             prefixes: prefixes.try_into().unwrap(),
+            routes: AtomicUsize::new(0),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct UpsertCounters {
+    // number of unique inserted prefixes|routes in the in-mem tree
+    in_memory_count: usize,
+    // number of unique persisted prefixes|routes
+    persisted_count: usize,
+    // total number of unique inserted prefixes|routes in the RIB
+    total_count: usize,
+}
+
+impl UpsertCounters {
+    pub fn in_memory(&self) -> usize {
+        self.in_memory_count
+    }
+
+    pub fn persisted(&self) -> usize {
+        self.persisted_count
+    }
+
+    pub fn total(&self) -> usize {
+        self.total_count
+    }
+}
+
+impl std::fmt::Display for UpsertCounters {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Unique Items in-memory:\t{}", self.in_memory_count)?;
+        write!(f, "Unique persisted Items:\t{}", self.persisted_count)?;
+        write!(f, "Total inserted Items:\t{}", self.total_count)
+    }
+}
+
+impl std::ops::AddAssign for UpsertCounters {
+    fn add_assign(&mut self, rhs: Self) {
+        self.in_memory_count += rhs.in_memory_count;
+        self.persisted_count += rhs.persisted_count;
+        self.total_count += rhs.total_count;
+    }
+}
+
+impl std::ops::Add for UpsertCounters {
+    type Output = UpsertCounters;
+
+    fn add(self, rhs: Self) -> Self::Output {
+        Self {
+            in_memory_count: self.in_memory_count + rhs.in_memory_count,
+            persisted_count: self.persisted_count + rhs.persisted_count,
+            total_count: self.total_count + rhs.total_count,
         }
     }
 }
@@ -608,18 +669,28 @@ impl<
         pfx.record_map.get_filtered_records(mui, bmin)
     }
 
-    pub fn get_prefixes_count(&self) -> (usize, usize) {
-        (
-            self.counters.get_prefixes_count().iter().sum(),
-            self.in_memory_tree.get_prefixes_count(),
-        )
+    pub fn get_prefixes_count(&self) -> UpsertCounters {
+        UpsertCounters {
+            in_memory_count: self.in_memory_tree.get_prefixes_count(),
+            persisted_count: self
+                .persist_tree
+                .as_ref()
+                .map_or(0, |p| p.get_prefixes_count()),
+            total_count: self.counters.get_prefixes_count().iter().sum(),
+        }
     }
 
-    pub fn get_prefixes_count_for_len(&self, len: u8) -> (usize, usize) {
-        (
-            self.counters.get_prefixes_count()[len as usize],
-            self.in_memory_tree.get_prefixes_count(),
-        )
+    pub fn get_prefixes_count_for_len(&self, len: u8) -> UpsertCounters {
+        UpsertCounters {
+            in_memory_count: self
+                .in_memory_tree
+                .get_prefixes_count_for_len(len),
+            persisted_count: self
+                .persist_tree
+                .as_ref()
+                .map_or(0, |p| p.get_prefixes_count_for_len(len)),
+            total_count: self.counters.get_prefixes_count()[len as usize],
+        }
     }
 
     // Stride related methods
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index bc703324..a54cece6 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -25,7 +25,8 @@ pub mod multi {
 
     pub use crate::rib::Rib;
     pub use crate::rib::{
-        Counters, PersistStrategy, StoreConfig, StoreStats, UpsertReport,
+        PersistStrategy, StoreConfig, StoreStats, UpsertCounters,
+        UpsertReport,
     };
 
     pub use routecore::bgp::path_selection::TiebreakerInfo;
diff --git a/tests/full-table.rs b/tests/full-table.rs
index 50136d2c..903fb2f2 100644
--- a/tests/full-table.rs
+++ b/tests/full-table.rs
@@ -207,7 +207,7 @@ mod tests {
             assert_eq!(searches_num, SEARCHES_NUM as u128);
             assert_eq!(inserts_num, INSERTS_NUM);
             assert_eq!(
-                tree_bitmap.prefixes_count(),
+                tree_bitmap.prefixes_count().total(),
                 GLOBAL_PREFIXES_VEC_SIZE
             );
             assert_eq!(found_counter, FOUND_PREFIXES);

From 03520af4199bef429fc5230aca3e78ace9e5e196 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 18 Dec 2024 19:28:46 +0100
Subject: [PATCH 029/147] move withdrawn_muis_bmin to rib

---
 src/local_array/in_memory/tree.rs |  3 ---
 src/local_array/rib/rib.rs        | 30 ++++++++++++------------------
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 6a47f482..a25fcd92 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -394,8 +394,6 @@ pub struct TreeBitMap<
 > {
     pub(crate) node_buckets: NB,
     pub(crate) prefix_buckets: PB,
-    // Global Roaring BitMap INdex that stores MUIs.
-    pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     counters: Counters,
     _af: PhantomData<AF>,
     _m: PhantomData<M>,
@@ -412,7 +410,6 @@ impl<
         let tree_bitmap = Self {
             node_buckets: NodeBuckets::init(),
             prefix_buckets: PB::init(),
-            withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
             _af: PhantomData,
             _m: PhantomData,
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 904e03fc..05d4f84b 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use inetnum::addr::Prefix;
 use log::{debug, error, info, log_enabled, trace};
 
-use crossbeam_epoch as epoch;
+use crossbeam_epoch::{self as epoch, Atomic};
 use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
 
@@ -215,9 +215,11 @@ pub struct Rib<
     const KEY_SIZE: usize,
 > {
     pub config: StoreConfig,
-    pub(crate) in_memory_tree: TreeBitMap<AF, M, NB, PB>,
+    pub(in crate::local_array) in_memory_tree: TreeBitMap<AF, M, NB, PB>,
     #[cfg(feature = "persist")]
     persist_tree: Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
+    // Global Roaring BitMap INdex that stores MUIs.
+    pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     pub counters: Counters,
 }
 
@@ -255,6 +257,7 @@ impl<
             config,
             in_memory_tree: TreeBitMap::<AF, M, NB, PB>::new()?,
             persist_tree,
+            withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
         };
 
@@ -515,8 +518,7 @@ impl<
         guard: &'a Guard,
     ) -> &'a RoaringBitmap {
         unsafe {
-            self.in_memory_tree
-                .withdrawn_muis_bmin
+            self.withdrawn_muis_bmin
                 .load(Ordering::Acquire, guard)
                 .deref()
         }
@@ -569,17 +571,14 @@ impl<
         mui: u32,
         guard: &Guard,
     ) -> Result<(), PrefixStoreError> {
-        let current = self
-            .in_memory_tree
-            .withdrawn_muis_bmin
-            .load(Ordering::Acquire, guard);
+        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
 
         let mut new = unsafe { current.as_ref() }.unwrap().clone();
         new.insert(mui);
 
         #[allow(clippy::assigning_clones)]
         loop {
-            match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
+            match self.withdrawn_muis_bmin.compare_exchange(
                 current,
                 Owned::new(new),
                 Ordering::AcqRel,
@@ -602,17 +601,14 @@ impl<
         mui: u32,
         guard: &Guard,
     ) -> Result<(), PrefixStoreError> {
-        let current = self
-            .in_memory_tree
-            .withdrawn_muis_bmin
-            .load(Ordering::Acquire, guard);
+        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
 
         let mut new = unsafe { current.as_ref() }.unwrap().clone();
         new.remove(mui);
 
         #[allow(clippy::assigning_clones)]
         loop {
-            match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
+            match self.withdrawn_muis_bmin.compare_exchange(
                 current,
                 Owned::new(new),
                 Ordering::AcqRel,
@@ -633,8 +629,7 @@ impl<
     // functions.
     pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
         unsafe {
-            self.in_memory_tree
-                .withdrawn_muis_bmin
+            self.withdrawn_muis_bmin
                 .load(Ordering::Acquire, guard)
                 .as_ref()
         }
@@ -647,8 +642,7 @@ impl<
     // functions.
     pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
         !unsafe {
-            self.in_memory_tree
-                .withdrawn_muis_bmin
+            self.withdrawn_muis_bmin
                 .load(Ordering::Acquire, guard)
                 .as_ref()
         }

From 373a95a1209c2f7f37e931089696c77f74549856 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 18 Dec 2024 21:08:54 +0100
Subject: [PATCH 030/147] move node-related stuff to node

---
 src/local_array/in_memory/atomic_stride.rs    |    2 +-
 src/local_array/in_memory/atomic_types.rs     |    4 +-
 src/local_array/in_memory/deprecated_query.rs |    6 +-
 src/local_array/in_memory/node.rs             |  142 ++-
 src/local_array/in_memory/tree.rs             |  176 +--
 src/local_array/iterators.rs                  |    5 +-
 src/local_array/node.rs                       | 1098 -----------------
 src/local_array/rib/rib.rs                    |    5 +-
 src/prelude/mod.rs                            |    2 +-
 9 files changed, 157 insertions(+), 1283 deletions(-)
 delete mode 100644 src/local_array/node.rs

diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 75d9e294..70d72c8b 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -9,7 +9,7 @@ use crate::af::Zero;
 use crate::synth_int::AtomicU128;
 use crate::{impl_primitive_atomic_stride, AddressFamily};
 
-use super::tree::StrideNodeId;
+use super::node::StrideNodeId;
 
 pub type Stride3 = u16;
 pub type Stride4 = u32;
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 254fd2d3..3da94d04 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -23,9 +23,9 @@ use crate::AddressFamily;
 
 use super::super::errors::PrefixStoreError;
 use super::atomic_stride;
-use super::node::TreeBitMapNode;
+use super::node::{StrideNodeId, TreeBitMapNode};
 use super::oncebox::OnceBoxSlice;
-use super::tree::{Stride, Stride3, Stride4, Stride5, StrideNodeId};
+use super::tree::{Stride, Stride3, Stride4, Stride5};
 
 // ----------- Node related structs -----------------------------------------
 
diff --git a/src/local_array/in_memory/deprecated_query.rs b/src/local_array/in_memory/deprecated_query.rs
index 467444b8..8c943e9e 100644
--- a/src/local_array/in_memory/deprecated_query.rs
+++ b/src/local_array/in_memory/deprecated_query.rs
@@ -2,18 +2,18 @@ use log::trace;
 
 use crate::af::AddressFamily;
 use crate::prelude::multi::PrefixSet;
-use crate::rib::Rib;
 use inetnum::addr::Prefix;
 
 use crate::{Meta, QueryResult};
 
-use crate::local_array::in_memory::node::TreeBitMapNode;
+use crate::local_array::in_memory::node::{SizedStrideRef, TreeBitMapNode};
 use crate::{MatchOptions, MatchType};
 
 use super::super::in_memory::atomic_types::StoredPrefix;
-use super::super::in_memory::tree::{SizedStrideRef, Stride, StrideNodeId};
+use super::super::in_memory::tree::Stride;
 use super::super::types::PrefixId;
 use super::atomic_types::{NodeBuckets, PrefixBuckets};
+use super::node::StrideNodeId;
 use super::tree::TreeBitMap;
 
 #[allow(dead_code)]
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 6f2c938a..c91a4f0f 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -9,7 +9,7 @@ use parking_lot_core::SpinWait;
 
 use super::super::bit_span::BitSpan;
 use super::super::iterators::{SizedNodeMoreSpecificIter, SizedPrefixIter};
-use super::tree::{AtomicBitmap, AtomicStride2, AtomicStride3, AtomicStride4, AtomicStride5, CasResult, NewNodeOrIndex, SizedStrideNode, Stride, Stride3, Stride4, Stride5, StrideNodeId};
+use super::tree::{AtomicBitmap, AtomicStride2, AtomicStride3, AtomicStride4, AtomicStride5, CasResult, Stride, Stride3, Stride4, Stride5};
 
 // pub use crate::in_memory_tree::*;
 use crate::af::Zero;
@@ -1085,3 +1085,143 @@ impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride5> {
         SizedPrefixIter::Stride5(self)
     }
 }
+
+//------------------- Sized Node Enums ------------------------------------
+
+// No, no, NO, NO, no, no! We're not going to Box this, because that's slow!
+// This enum is never used to store nodes/prefixes, it's only to be used in
+// generic code.
+#[allow(clippy::large_enum_variant)]
+#[derive(Debug)]
+pub enum SizedStrideNode<AF: AddressFamily> {
+    Stride3(TreeBitMapNode<AF, Stride3>),
+    Stride4(TreeBitMapNode<AF, Stride4>),
+    Stride5(TreeBitMapNode<AF, Stride5>),
+}
+
+impl<AF, S> Default for TreeBitMapNode<AF, S>
+where
+    AF: AddressFamily,
+    S: Stride,
+{
+    fn default() -> Self {
+        Self {
+            ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
+            pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
+            _af: PhantomData,
+        }
+    }
+}
+
+impl<AF> Default for SizedStrideNode<AF>
+where
+    AF: AddressFamily,
+{
+    fn default() -> Self {
+        SizedStrideNode::Stride3(TreeBitMapNode {
+            ptrbitarr: AtomicStride2(AtomicU8::new(0)),
+            pfxbitarr: AtomicStride3(AtomicU16::new(0)),
+            _af: PhantomData,
+        })
+    }
+}
+
+// Used to create a public iterator over all nodes.
+#[derive(Debug)]
+pub enum SizedStrideRef<'a, AF: AddressFamily> {
+    Stride3(&'a TreeBitMapNode<AF, Stride3>),
+    Stride4(&'a TreeBitMapNode<AF, Stride4>),
+    Stride5(&'a TreeBitMapNode<AF, Stride5>),
+}
+
+pub(crate) enum NewNodeOrIndex<AF: AddressFamily> {
+    NewNode(SizedStrideNode<AF>),
+    ExistingNode(StrideNodeId<AF>),
+    NewPrefix,
+    ExistingPrefix,
+}
+
+//--------------------- Per-Stride-Node-Id Type -----------------------------
+
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub struct StrideNodeId<AF: AddressFamily>(Option<(AF, u8)>);
+
+impl<AF: AddressFamily> StrideNodeId<AF> {
+    pub fn empty() -> Self {
+        Self(None)
+    }
+
+    pub fn dangerously_new_with_id_as_is(addr_bits: AF, len: u8) -> Self {
+        Self(Some((addr_bits, len)))
+    }
+
+    #[inline]
+    pub fn new_with_cleaned_id(addr_bits: AF, len: u8) -> Self {
+        Self(Some((addr_bits.truncate_to_len(len), len)))
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_none()
+    }
+
+    pub fn get_id(&self) -> (AF, u8) {
+        self.0.unwrap()
+    }
+    pub fn get_len(&self) -> u8 {
+        self.0.unwrap().1
+    }
+    pub fn set_len(mut self, len: u8) -> Self {
+        self.0.as_mut().unwrap().1 = len;
+        self
+    }
+
+    pub fn add_to_len(mut self, len: u8) -> Self {
+        self.0.as_mut().unwrap().1 += len;
+        self
+    }
+
+    #[inline]
+    pub fn truncate_to_len(self) -> Self {
+        let (addr_bits, len) = self.0.unwrap();
+        StrideNodeId::new_with_cleaned_id(addr_bits, len)
+    }
+
+    // clean out all bits that are set beyond the len. This function should
+    // be used before doing any ORing to add a nibble.
+    #[inline]
+    pub fn unwrap_with_cleaned_id(&self) -> (AF, u8) {
+        let (addr_bits, len) = self.0.unwrap();
+        (addr_bits.truncate_to_len(len), len)
+    }
+
+    pub fn add_nibble(&self, nibble: u32, nibble_len: u8) -> Self {
+        let (addr_bits, len) = self.unwrap_with_cleaned_id();
+        let res = addr_bits.add_nibble(len, nibble, nibble_len);
+        Self(Some(res))
+    }
+
+    pub fn into_inner(self) -> Option<(AF, u8)> {
+        self.0
+    }
+}
+
+impl<AF: AddressFamily> std::fmt::Display for StrideNodeId<AF> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}",
+            self.0
+                .map(|x| format!("{}-{}", x.0, x.1))
+                .unwrap_or_else(|| "-".to_string())
+        )
+    }
+}
+
+impl<AF: AddressFamily> std::convert::From<StrideNodeId<AF>>
+    for PrefixId<AF>
+{
+    fn from(id: StrideNodeId<AF>) -> Self {
+        let (addr_bits, len) = id.0.unwrap();
+        PrefixId::new(addr_bits, len)
+    }
+}
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index a25fcd92..8496b3d5 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -187,10 +187,8 @@ use crate::local_array::bit_span::BitSpan;
 use crate::local_array::in_memory::atomic_types::StoredNode;
 use crate::prefix_record::Meta;
 use crate::prelude::multi::{NodeSet, PrefixId};
-use crossbeam_epoch::Atomic;
 use crossbeam_utils::Backoff;
 use log::{debug, log_enabled, trace};
-use roaring::RoaringBitmap;
 
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
 use std::{fmt::Debug, marker::PhantomData};
@@ -208,181 +206,13 @@ use crate::{
 use super::super::errors::PrefixStoreError;
 pub(crate) use super::atomic_stride::*;
 
-use super::node::TreeBitMapNode;
+use super::node::{
+    SizedStrideNode, SizedStrideRef, StrideNodeId, TreeBitMapNode,
+};
 
 #[cfg(feature = "cli")]
 use ansi_term::Colour;
 
-//------------------- Sized Node Enums ------------------------------------
-
-// No, no, NO, NO, no, no! We're not going to Box this, because that's slow!
-// This enum is never used to store nodes/prefixes, it's only to be used in
-// generic code.
-#[allow(clippy::large_enum_variant)]
-#[derive(Debug)]
-pub enum SizedStrideNode<AF: AddressFamily> {
-    Stride3(TreeBitMapNode<AF, Stride3>),
-    Stride4(TreeBitMapNode<AF, Stride4>),
-    Stride5(TreeBitMapNode<AF, Stride5>),
-}
-
-impl<AF, S> Default for TreeBitMapNode<AF, S>
-where
-    AF: AddressFamily,
-    S: Stride,
-{
-    fn default() -> Self {
-        Self {
-            ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
-            pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
-            _af: PhantomData,
-        }
-    }
-}
-
-impl<AF> Default for SizedStrideNode<AF>
-where
-    AF: AddressFamily,
-{
-    fn default() -> Self {
-        SizedStrideNode::Stride3(TreeBitMapNode {
-            ptrbitarr: AtomicStride2(AtomicU8::new(0)),
-            pfxbitarr: AtomicStride3(AtomicU16::new(0)),
-            _af: PhantomData,
-        })
-    }
-}
-
-// Used to create a public iterator over all nodes.
-#[derive(Debug)]
-pub enum SizedStrideRef<'a, AF: AddressFamily> {
-    Stride3(&'a TreeBitMapNode<AF, Stride3>),
-    Stride4(&'a TreeBitMapNode<AF, Stride4>),
-    Stride5(&'a TreeBitMapNode<AF, Stride5>),
-}
-
-pub(crate) enum NewNodeOrIndex<AF: AddressFamily> {
-    NewNode(SizedStrideNode<AF>),
-    ExistingNode(StrideNodeId<AF>),
-    NewPrefix,
-    ExistingPrefix,
-}
-
-//--------------------- Per-Stride-Node-Id Type -----------------------------
-
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub struct StrideNodeId<AF: AddressFamily>(Option<(AF, u8)>);
-
-impl<AF: AddressFamily> StrideNodeId<AF> {
-    pub fn empty() -> Self {
-        Self(None)
-    }
-
-    pub fn dangerously_new_with_id_as_is(addr_bits: AF, len: u8) -> Self {
-        Self(Some((addr_bits, len)))
-    }
-
-    #[inline]
-    pub fn new_with_cleaned_id(addr_bits: AF, len: u8) -> Self {
-        Self(Some((addr_bits.truncate_to_len(len), len)))
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.0.is_none()
-    }
-
-    pub fn get_id(&self) -> (AF, u8) {
-        self.0.unwrap()
-    }
-    pub fn get_len(&self) -> u8 {
-        self.0.unwrap().1
-    }
-    pub fn set_len(mut self, len: u8) -> Self {
-        self.0.as_mut().unwrap().1 = len;
-        self
-    }
-
-    pub fn add_to_len(mut self, len: u8) -> Self {
-        self.0.as_mut().unwrap().1 += len;
-        self
-    }
-
-    #[inline]
-    pub fn truncate_to_len(self) -> Self {
-        let (addr_bits, len) = self.0.unwrap();
-        StrideNodeId::new_with_cleaned_id(addr_bits, len)
-    }
-
-    // clean out all bits that are set beyond the len. This function should
-    // be used before doing any ORing to add a nibble.
-    #[inline]
-    pub fn unwrap_with_cleaned_id(&self) -> (AF, u8) {
-        let (addr_bits, len) = self.0.unwrap();
-        (addr_bits.truncate_to_len(len), len)
-    }
-
-    pub fn add_nibble(&self, nibble: u32, nibble_len: u8) -> Self {
-        let (addr_bits, len) = self.unwrap_with_cleaned_id();
-        let res = addr_bits.add_nibble(len, nibble, nibble_len);
-        Self(Some(res))
-    }
-
-    pub fn into_inner(self) -> Option<(AF, u8)> {
-        self.0
-    }
-}
-
-impl<AF: AddressFamily> std::fmt::Display for StrideNodeId<AF> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{}",
-            self.0
-                .map(|x| format!("{}-{}", x.0, x.1))
-                .unwrap_or_else(|| "-".to_string())
-        )
-    }
-}
-
-impl<AF: AddressFamily> std::convert::From<StrideNodeId<AF>>
-    for PrefixId<AF>
-{
-    fn from(id: StrideNodeId<AF>) -> Self {
-        let (addr_bits, len) = id.0.unwrap();
-        PrefixId::new(addr_bits, len)
-    }
-}
-
-//------------------------- Node Collections --------------------------------
-
-// #[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy, Clone)]
-// pub enum StrideType {
-//     Stride3,
-//     Stride4,
-//     Stride5,
-// }
-
-// impl From<u8> for StrideType {
-//     fn from(level: u8) -> Self {
-//         match level {
-//             3 => StrideType::Stride3,
-//             4 => StrideType::Stride4,
-//             5 => StrideType::Stride5,
-//             _ => panic!("Invalid stride level {}", level),
-//         }
-//     }
-// }
-
-// impl std::fmt::Display for StrideType {
-//     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-//         match self {
-//             StrideType::Stride3 => write!(f, "S3"),
-//             StrideType::Stride4 => write!(f, "S4"),
-//             StrideType::Stride5 => write!(f, "S5"),
-//         }
-//     }
-// }
-
 //--------------------- TreeBitMap ------------------------------------------
 
 #[derive(Debug)]
diff --git a/src/local_array/iterators.rs b/src/local_array/iterators.rs
index 43f54a99..0f3cd15b 100644
--- a/src/local_array/iterators.rs
+++ b/src/local_array/iterators.rs
@@ -8,9 +8,10 @@
 // individual nodes. The Node Iterators live in the node.rs file.
 
 use super::in_memory::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
-use super::in_memory::tree::{Stride3, Stride4, Stride5, StrideNodeId};
+use super::in_memory::node::StrideNodeId;
+use super::in_memory::tree::{Stride3, Stride4, Stride5};
 use super::types::PrefixId;
-use crate::local_array::in_memory::tree::SizedStrideRef;
+use crate::local_array::in_memory::node::SizedStrideRef;
 use crate::local_array::types::RouteStatus;
 use crate::prefix_record::PublicRecord;
 use crate::rib;
diff --git a/src/local_array/node.rs b/src/local_array/node.rs
deleted file mode 100644
index 040f7a9e..00000000
--- a/src/local_array/node.rs
+++ /dev/null
@@ -1,1098 +0,0 @@
-use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
-use std::{
-    fmt::Debug,
-    marker::PhantomData,
-};
-
-use log::trace;
-use parking_lot_core::SpinWait;
-
-// pub use super::atomic_stride::*;
-use super::bit_span::BitSpan;
-use super::store::iterators::SizedNodeMoreSpecificIter;
-use crate::local_array::store::iterators::SizedPrefixIter;
-
-pub use crate::local_array::tree::*;
-use crate::af::Zero;
-use crate::af::AddressFamily;
-
-//------------ TreeBitMap Node ----------------------------------------------
-
-// The treebitmap turned into a "trie-bitmap", really. A Node in the
-// treebitmap now only holds a ptrbitarr bitmap and a pfxbitarr bitmap, that
-// indicate whether a node or a prefix exists in that spot. The corresponding
-// node Ids and prefix ids are calculated from their position in the array.
-// Nodes do *NOT* have a clue where they are in the tree, so they don't know
-// the node id they represent. Instead, the node id is calculated from the
-// position in the tree. That's why several methods take a `base_prefix` as a
-// an argument: it represents the ID of the node itself.
-//
-// The elision of both the collection of children nodes and the prefix nodes
-// in a treebitmap node is enabled by the storage backend for the
-// multi-threaded store, since holds its entries keyed on the [node|prefix]
-// id. (in contrast with arrays or `vec`s, that have 
-pub struct TreeBitMapNode<
-    AF,
-    S,
-> where
-    Self: Sized,
-    S: Stride,
-    AF: AddressFamily,
-{
-    pub ptrbitarr: <S as Stride>::AtomicPtrSize,
-    pub pfxbitarr: <S as Stride>::AtomicPfxSize,
-    pub _af: PhantomData<AF>,
-}
-
-impl<AF, S> Debug
-    for TreeBitMapNode<AF, S>
-where
-    AF: AddressFamily,
-    S: Stride,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("TreeBitMapNode")
-            .field("ptrbitarr", &self.ptrbitarr.load())
-            .field("pfxbitarr", &self.pfxbitarr.load())
-            .finish()
-    }
-}
-
-impl<AF, S>
-    std::fmt::Display for TreeBitMapNode<AF, S>
-where
-    AF: AddressFamily,
-    S: Stride
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "TreeBitMapNode {{ ptrbitarr: {:?}, pfxbitarr: {:?} }}",
-            self.ptrbitarr.load(),
-            self.pfxbitarr.load(),
-        )
-    }
-}
-
-impl<AF, S>
-    TreeBitMapNode<AF, S>
-where
-    AF: AddressFamily,
-    S: Stride
-{
-<<<<<<< Updated upstream
-    pub(crate) fn new() -> Self {
-        TreeBitMapNode {
-            ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
-            pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
-            _af: PhantomData
-        }
-    }
-
-=======
-    pub(crate) fn empty() -> Self {
-        TreeBitMapNode { 
-            ptrbitarr: <S as Stride>::AtomicPtrSize::new(), 
-            pfxbitarr: <S as Stride>::AtomicPfxSize::new(),
-            _af: PhantomData
-        }
-    }
-    
->>>>>>> Stashed changes
-    // ------- Iterators ----------------------------------------------------
-
-    // Iterate over all the child node of this node
-    pub(crate) fn ptr_iter(&self, base_prefix: StrideNodeId<AF>) -> 
-        NodeChildIter<AF,S> {
-            NodeChildIter::<AF,S> {
-                base_prefix,
-                ptrbitarr: self.ptrbitarr.load(),
-                bit_span: BitSpan::new(0, 1),
-                _af: PhantomData,
-            }
-    }
-
-    // Iterate over all the prefix ids contained in this node.
-    // Note that this function is *not* used by the iterator that iterates
-    // over all prefixes. That one doesn't have to use the tree at all, but
-    // uses the store directly.
-    pub(crate) fn pfx_iter(&self, base_prefix: StrideNodeId<AF>) -> 
-        NodePrefixIter<AF, S> {
-        NodePrefixIter::<AF, S> {
-            pfxbitarr: self.pfxbitarr.load(),
-            base_prefix,
-            bit_span: BitSpan::new(0,1 ),
-            _af: PhantomData,
-            _s: PhantomData,
-        }
-    }
-
-    // Iterate over the more specific prefixes ids contained in this node
-    pub(crate) fn more_specific_pfx_iter(&self, base_prefix: StrideNodeId<AF>, 
-    start_bit_span: BitSpan, skip_self: bool) -> 
-        NodeMoreSpecificsPrefixIter<AF, S> {
-        NodeMoreSpecificsPrefixIter::<AF, S> {
-            pfxbitarr: self.pfxbitarr.load(),
-            base_prefix,
-            start_bit_span,
-            cursor: start_bit_span,
-            skip_self,
-            _s: PhantomData,
-        }
-    }
-
-    // Iterate over the nodes that contain more specifics for the requested
-    // base_prefix and corresponding bit_span.
-    pub(crate) fn more_specific_ptr_iter(&self, base_prefix: StrideNodeId<AF>, 
-    start_bit_span: BitSpan) -> 
-        NodeMoreSpecificChildIter<AF, S> {
-        NodeMoreSpecificChildIter::<AF, S> {
-            ptrbitarr: self.ptrbitarr.load(),
-            base_prefix,
-            start_bit_span,
-            cursor: None,
-        }
-    }
-
-
-    // ------- Search by Traversal methods -----------------------------------
-
-    // Inspects the stride (nibble, nibble_len) to see it there's already a 
-    // child node (if not at the last stride) or a prefix (if it's the last
-    // stride).
-    //
-    // Returns a tuple of which the first element is one of:
-    // - A newly created child node.
-    // - The index of the existing child node in the global `nodes` vec
-    // - A newly created Prefix
-    // - The index of the existing prefix in the global `prefixes` vec
-    // and the second element is the number of accumulated retries for the
-    // compare_exchange of both ptrbitarr and pfxbitarr.
-    pub(crate) fn eval_node_or_prefix_at(
-        &self,
-        nibble: u32,
-        nibble_len: u8,
-        // all the bits of the search prefix, but with the length set to
-        // the length of this stride. So bits are set beyond its length.
-        base_prefix: StrideNodeId<AF>,
-        stride_len: u8,
-        next_stride: Option<&u8>,
-        is_last_stride: bool,
-    ) -> (NewNodeOrIndex<AF>, u32) {
-
-        // THE CRITICAL SECTION
-        //
-        // UPDATING ptrbitarr & pfxbitarr
-        //
-        // This section is not as critical as creating/updating a
-        // a prefix. We need to set one bit only, and if somebody
-        // beat us to it that's fine, we'll figure that out when
-        // we try to write the prefix's serial number later on.
-        // The one thing that can go wrong here is that we are
-        // using an old ptrbitarr and overwrite bits set in the
-        // meantime elsewhere in the bitarray.
-        let mut retry_count = 0;
-        let ptrbitarr = self.ptrbitarr.load();
-        let pfxbitarr = self.pfxbitarr.load();
-        let bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let new_node: SizedStrideNode<AF>;
-
-        // Check that we're not at the last stride (pfx.len <= stride_end),
-        // Note that next_stride may have a value, but we still don't want to
-        // continue, because we've exceeded the length of the prefix to
-        // be inserted.
-        // Also note that a nibble_len < S::BITS (a smaller than full nibble)
-        // does indeed indicate the last stride has been reached, but the
-        // reverse is *not* true, i.e. a full nibble can also be the last
-        // stride. Hence the `is_last_stride` argument
-        if !is_last_stride {
-
-            // We are not at the last stride
-            // Check it the ptr bit is already set in this position
-            if (S::into_stride_size(ptrbitarr) & bit_pos) == 
-                <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType>::zero() {
-                // Nope, set it and create a child node
-
-                match next_stride.unwrap() {
-                    3_u8 => {
-                        new_node = SizedStrideNode::Stride3(TreeBitMapNode {
-                            ptrbitarr: AtomicStride2(AtomicU8::new(0)),
-                            pfxbitarr: AtomicStride3(AtomicU16::new(0)),
-                            // pfx_vec: PrefixSet::empty(14),
-                            _af: PhantomData,
-                        });
-                    }
-                    4_u8 => {
-                        new_node = SizedStrideNode::Stride4(TreeBitMapNode {
-                            ptrbitarr: AtomicStride3(AtomicU16::new(0)),
-                            pfxbitarr: AtomicStride4(AtomicU32::new(0)),
-                            // pfx_vec: PrefixSet::empty(30),
-                            _af: PhantomData,
-                        });
-                    }
-                    5_u8 => {
-                        new_node = SizedStrideNode::Stride5(TreeBitMapNode {
-                            ptrbitarr: AtomicStride4(AtomicU32::new(0)),
-                            pfxbitarr: AtomicStride5(AtomicU64::new(0)),
-                            // pfx_vec: PrefixSet::empty(62),
-                            _af: PhantomData,
-                        });
-                    }
-                    _ => {
-                        panic!("can't happen");
-                    }
-                };
-
-                // THE CRITICAL SECTION
-                //
-                // UPDATING pfxbitarr
-                //
-                // preventing using an old ptrbitarr and overwrite bits set
-                // in the meantime elsewhere in the bitarray.
-                let mut a_ptrbitarr = self.ptrbitarr.compare_exchange(ptrbitarr,
-                    S::into_ptrbitarr_size(
-                    bit_pos | S::into_stride_size(ptrbitarr),
-                ));
-                let mut spinwait = SpinWait::new();
-                loop {
-                    match a_ptrbitarr {
-                        CasResult(Ok(_)) => {
-                            break;
-                        }
-                        CasResult(Err(newer_array)) => {
-                            // Someone beat us to it, so we need to use the
-                            // newer array.
-                            retry_count += 1;
-                            a_ptrbitarr = self.ptrbitarr.compare_exchange(newer_array,
-                                S::into_ptrbitarr_size(
-                                bit_pos | S::into_stride_size(newer_array),
-                            ));
-                        }
-                    };
-                    spinwait.spin_no_yield();
-                }
-
-                return (NewNodeOrIndex::NewNode(
-                    new_node
-                ), retry_count);
-            }
-        } else {
-            // only at the last stride do we create the bit in the prefix
-            // bitmap, and only if it doesn't exist already
-            if pfxbitarr & bit_pos
-                == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType as std::ops::BitAnd>::Output::zero()
-            {
-
-                // THE CRITICAL SECTION
-                //
-                // UPDATING pfxbitarr
-                //
-                // preventing using an old pfxbitarr and overwrite bits set
-                // in the meantime elsewhere in the bitarray.
-                let mut a_pfxbitarr = 
-                self.pfxbitarr.compare_exchange(
-                    pfxbitarr, bit_pos | pfxbitarr
-                );
-                let mut spinwait = SpinWait::new();
-
-                loop {
-                    match a_pfxbitarr {
-                        CasResult(Ok(_)) => {
-                            break;
-                        }
-                        CasResult(Err(newer_array)) => {
-                            // Someone beat us to it, so we need to use the
-                            // newer array.
-                            retry_count += 1;
-                            a_pfxbitarr = self.pfxbitarr.compare_exchange(
-                                newer_array, bit_pos | newer_array
-                            );
-                        }
-                    };
-                    spinwait.spin_no_yield();
-                }
-
-                return (NewNodeOrIndex::NewPrefix, retry_count);
-            }
-            return (NewNodeOrIndex::ExistingPrefix, retry_count);
-        }
-
-        // Nodes always live at the last length of a stride (i.e. the last 
-        // nibble), so we add the stride length to the length of the
-        // base_prefix (which is always the start length of the stride).
-        (NewNodeOrIndex::ExistingNode(
-            base_prefix.add_to_len(stride_len).truncate_to_len()
-        ), retry_count)
-    }
-
-    //-------- Search nibble functions --------------------------------------
-
-    // This function looks for the longest marching prefix in the provided
-    // nibble, by iterating over all the bits in it and comparing that with
-    // the appropriate bytes from the requested prefix. It mutates the 
-    // `less_specifics_vec` that was passed in to hold all the prefixes found
-    // along the way.
-    pub(crate) fn search_stride_for_longest_match_at(
-        &self,
-        search_pfx: PrefixId<AF>,
-        mut nibble: u32,
-        nibble_len: u8,
-        start_bit: u8,
-        less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
-    ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
-        let pfxbitarr = self.pfxbitarr.load();
-        let ptrbitarr = self.ptrbitarr.load();
-        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_pfx = None;
-
-        trace!("start longest_match search");
-        for n_l in 1..(nibble_len + 1) {
-            // Move the bit in the right position.
-            nibble =
-                AddressFamily::get_nibble(search_pfx.get_net(), start_bit, n_l);
-            bit_pos = S::get_bit_pos(nibble, n_l);
-
-            // Check if the prefix has been set, if so select the prefix.
-            // This is not necessarily the final prefix that will be
-            // returned.
-
-            // Check it there's a prefix matching in this bitmap for this 
-            // nibble.
-            trace!("pfxbitarr {:032b}", pfxbitarr);
-
-            if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-                let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + n_l), start_bit + n_l);
-                // f_pfx.set_serial(self.get_pfx_serial(f_pfx, nibble, n_l, guard).load(Ordering::Relaxed));
-
-                // Receiving a less_specifics_vec means that the user wants
-                // to have all the last-specific prefixes returned, so add
-                // the found prefix.
-                trace!("gather pfx in less_specifics {:?}", f_pfx);
-                trace!("ls_vec {:?}", less_specifics_vec);
-                if let Some(ls_vec) = less_specifics_vec {
-                    trace!("len {}", search_pfx.get_len());
-                    trace!("start_bit {}", start_bit);
-                    trace!("n_l {}", n_l);
-                    trace!("smaller length? {}", search_pfx.get_len() > start_bit + n_l);
-                    trace!("{}", (S::into_stride_size(ptrbitarr)
-                            & bit_pos)
-                            == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero());
-                    if search_pfx.get_len() > start_bit + n_l
-                        && (S::into_stride_size(ptrbitarr)
-                            & bit_pos)
-                            == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-                    {
-                        ls_vec.push(f_pfx);
-                    }
-                }
-
-                found_pfx = Some(f_pfx);
-            }
-        }
-
-        let base_prefix =
-            StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit);
-
-        // Check if this the last stride, or if they're no more children to
-        // go to, if so return what we found up until now.
-        if search_pfx.get_len() <= start_bit + nibble_len
-            || (S::into_stride_size(ptrbitarr) & bit_pos) == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-        // No children or at the end, return the definitive LMP we found.
-        {
-            return (
-                None,      /* no more children */
-                found_pfx, /* The definitive LMP if any */
-            );
-        }
-
-        // There's another child, return it together with the preliminary LMP
-        // we found.
-        (
-            // The identifier of the node that has children of the next
-            // stride.
-            Some(base_prefix.add_nibble(nibble, nibble_len)),
-            found_pfx,
-        )
-    }
-
-    // This function looks for the exactly matching prefix in the provided
-    // nibble. It doesn't need to iterate over anything it just compares
-    // the complete nibble, with the appropriate bits in the requested
-    // prefix. Although this is rather efficient, there's no way to collect
-    // less-specific prefixes from the search prefix.
-    pub(crate) fn search_stride_for_exact_match_at(
-        &'_ self,
-        search_pfx: PrefixId<AF>,
-        nibble: u32,
-        nibble_len: u8,
-        start_bit: u8,
-        _: &mut Option<Vec<PrefixId<AF>>>,
-    ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
-        let pfxbitarr = self.pfxbitarr.load();
-        let ptrbitarr = self.ptrbitarr.load();
-        // This is an exact match, so we're only considering the position of
-        // the full nibble.
-        let bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_pfx = None;
-        let mut found_child = None;
-
-        // Is this the last nibble?
-        // Otherwise we're not looking for a prefix (exact matching only
-        // lives at last nibble)
-        match search_pfx.get_len() <= start_bit + nibble_len {
-            // We're at the last nibble.
-            true => {
-                // Check for an actual prefix at the right position, i.e. 
-                // consider the complete nibble.
-                if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-                    let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + nibble_len), start_bit + nibble_len);
-                    found_pfx = Some(f_pfx);
-                }
-            }
-            // We're not at the last nibble.
-            false => {
-                // Check for a child node at the right position, i.e.
-                // consider the complete nibble.
-                if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-                {
-                    found_child = Some(
-                        StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)
-                    );
-                }
-            }
-        }
-
-        (
-            found_child, /* The node that has children in the next stride, if
-                            any */
-            found_pfx,   /* The exactly matching prefix, if any */
-        )
-    }
-
-    // This function looks for the exactly matching prefix in the provided
-    // nibble, just like the one above, but this *does* iterate over all the
-    // bytes in the nibble to collect the less-specific prefixes of the the 
-    // search prefix. This is of course slower, so it should only be used 
-    // when the user explicitly requests less-specifics.
-    pub(crate) fn search_stride_for_exact_match_with_less_specifics_at(
-        &self,
-        search_pfx: PrefixId<AF>,
-        mut nibble: u32,
-        nibble_len: u8,
-        start_bit: u8,
-        less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
-    ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
-        let pfxbitarr = self.pfxbitarr.load();
-        let ptrbitarr = self.ptrbitarr.load();
-        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_pfx = None;
-
-        let ls_vec = less_specifics_vec
-            .as_mut()
-            .expect(concat!("You shouldn't call this function without",
-            "a `less_specifics_vec` buffer. Supply one when calling this function",
-            "or use `search_stride_for_exact_match_at`"));
-
-        for n_l in 1..(nibble_len + 1) {
-            // Move the bit in the right position.
-            nibble =
-                AddressFamily::get_nibble(search_pfx.get_net(), start_bit, n_l);
-            bit_pos = S::get_bit_pos(nibble, n_l);
-
-            // Check if the prefix has been set, if so select the prefix.
-            // This is not necessarily the final prefix that will be
-            // returned.
-
-            // Check it there's a prefix matching in this bitmap for this 
-            // nibble.
-            if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-                // since we want an exact match only, we will fill the prefix
-                // field only if we're exactly at the last bit of the nibble
-                if n_l == nibble_len {
-                    let f_pfx =
-                        PrefixId::new(
-                            search_pfx.get_net().truncate_to_len(start_bit + n_l), start_bit + n_l);
-                        found_pfx = Some(f_pfx);
-                }
-
-                // Receiving a less_specifics_vec means that the user wants to
-                // have all the last-specific prefixes returned, so add the
-                // found prefix.
-                let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + n_l), start_bit + n_l);
-                ls_vec.push(f_pfx);
-            }
-        }
-
-        if found_pfx.is_none() {
-            // no prefix here, clear out all of the prefixes we found along
-            // the way, since it doesn't make sense to return less-specifics
-            // if we don't have a exact match.
-            ls_vec.clear();
-        }
-
-        // Check if this the last stride, or if they're no more children to 
-        // go to, if so return what we found up until now.
-        match search_pfx.get_len() <= start_bit + nibble_len
-            || (S::into_stride_size(ptrbitarr) & bit_pos)
-                == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType as std::ops::BitAnd>::Output::zero()
-        {
-            // No children or at the end, return the definitive LMP we found.
-            true => (
-                None,      /* no more children */
-                found_pfx, /* The definitive LMP if any */
-            ),
-            // There's another child, we won't return the found_pfx, since
-            // we're not at the last nibble and we want an exact match only.
-            false => (
-                Some(StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)),
-                None,
-            ),
-        }
-    }
-
-    // Search a stride for more-specific prefixes and child nodes containing
-    // more specifics for `search_prefix`.
-    pub(crate) fn add_more_specifics_at(
-        &self,
-        nibble: u32,
-        nibble_len: u8,
-        base_prefix: StrideNodeId<AF>,
-    ) -> (
-        Vec<StrideNodeId<AF>>, /* child nodes with more more-specifics in
-                                  this stride */
-        Vec<PrefixId<AF>>,     /* more-specific prefixes in this stride */
-    ) {
-        trace!("start adding more specifics");
-        let pfxbitarr = self.pfxbitarr.load();
-        let ptrbitarr = self.ptrbitarr.load();
-        trace!("ptrbitarr {:032b}", ptrbitarr);
-        trace!("pfxbitarr {:032b}", pfxbitarr);
-        let mut found_children_with_more_specifics = vec![];
-        let mut found_more_specifics_vec: Vec<PrefixId<AF>> = vec![];
-
-        // This is an exact match, so we're only considering the position of
-        // the full nibble.
-        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_child = None;
-
-        // Is there also a child node here?
-        // Note that even without a child node, there may be more specifics 
-        // further up in this pfxbitarr or children in this ptrbitarr.
-        if (S::into_stride_size(ptrbitarr) & bit_pos)
-            > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
-            )
-        {
-            found_child = Some(base_prefix.add_nibble(nibble, nibble_len));
-        }
-
-        if let Some(child) = found_child {
-            found_children_with_more_specifics.push(child);
-        }
-
-        // We're expanding the search for more-specifics bit-by-bit.
-        // `ms_nibble_len` is the number of bits including the original
-        // nibble we're considering, e.g. if our prefix has a length of 25
-        // and we've all strides sized 4, we would end up with a last
-        // nibble_len of 1. `ms_nibble_len` will expand then from 2 up and
-        // till 4.
-        // 
-        // ex.:
-        // nibble: 1 , (nibble_len: 1)
-        // Iteration:
-        // ms_nibble_len=1,n_l=0: 10, n_l=1: 11
-        // ms_nibble_len=2,n_l=0: 100, n_l=1: 101, n_l=2: 110, n_l=3: 111
-        // ms_nibble_len=3,n_l=0: 1000, n_l=1: 1001, n_l=2: 1010, ..., 
-        // n_l=7: 1111
-
-        for ms_nibble_len in nibble_len + 1..=S::STRIDE_LEN {
-            // iterate over all the possible values for this `ms_nibble_len`,
-            // e.g. two bits can have 4 different values.
-            for n_l in 0..(1 << (ms_nibble_len - nibble_len)) {
-                // move the nibble left with the amount of bits we're going
-                // to loop over. e.g. a stride of size 4 with a nibble 0000
-                // 0000 0000 0011 becomes 0000 0000 0000 1100, then it will
-                // iterate over ...1100,...1101,...1110,...1111
-                let ms_nibble =
-                    (nibble << (ms_nibble_len - nibble_len)) + n_l as u32;
-                bit_pos = S::get_bit_pos(ms_nibble, ms_nibble_len);
-
-                if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-                {
-                    found_children_with_more_specifics.push(
-                    base_prefix.add_nibble(ms_nibble, ms_nibble_len)
-                    );
-                }
-
-                if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-                    found_more_specifics_vec.push(
-                        base_prefix.add_nibble(ms_nibble, ms_nibble_len).into()                    )
-                }
-            }
-        }
-
-        trace!("found_children_with_more_specifics {:?}", found_children_with_more_specifics);
-        trace!("found_more_specifics_vec {:?}", found_more_specifics_vec);
-
-        (
-            // We're done here, the caller should now go over all nodes in
-            // found_children_with_more_specifics vec and add ALL prefixes
-            // found in there.
-            found_children_with_more_specifics,
-            found_more_specifics_vec,
-        )
-    }
-}
-
-
-// ------------ Iterator methods --------------------------------------------
-
-// ----------- NodeChildIter ------------------------------------------------
-
-// create an iterator over all child nodes id
-//
-// we don't have a collection of local nodes anymore, since the id of the
-// node are deterministically generated, as the prefix+len they represent
-// in the treebitmap. This has both the advantage of using less memory,
-// and being easier to use in a concurrently updated tree. The
-// disadvantage is that we have to look up the child nodes on the fly
-// when we want to iterate over all children of a node.
-//
-// ptr child nodes only exist at the last nibble of the stride size
-// (`child_len`). Since children  in the first nibbles are leaf nodes.
-// leaf nodes will only be prefixes. So if we have a first stride of
-// size 5, all ptr nodes wil have StrideNodeIds with len = 5.
-//
-// Ex.:
-//
-// Stride no.          1       2       3      4       5       6       7       
-// StrideSize          5       5       4      3       3       3       3      
-// child pfxs len      /1-5   /5-10    /10-14 /15-17  /18-20  /21-23  /24-26
-// child Nodes len     /5      /10     /14    /17     /20     /23     /26 
-//
-// Stride no.          8      9
-// StrideSize          3      3
-// child pfxs len      /27-29 /30-32
-// child Nodes len     /29    /32
-
-#[derive(Debug, Copy, Clone)]
-pub(crate) struct NodeChildIter<AF: AddressFamily, S: Stride> {
-   base_prefix: StrideNodeId<AF>,
-   ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
-   bit_span: BitSpan, // start with 0
-   _af: PhantomData<AF>,
-}
-
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
-    NodeChildIter<AF, S>
-{
-    type Item = StrideNodeId<AF>;
-    fn next(&mut self) -> Option<Self::Item> {
-        // iterate over all the possible values for this stride length, e.g.
-        // two bits can have 4 different values.
-        for cursor in self.bit_span.bits..(1 << S::STRIDE_LEN) {
-            // move the bit_span left with the amount of bits we're going to
-            // loop over.
-            // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
-            // becomes 0000 0000 0000 1100, then it will iterate over 
-            // ...1100,...1101,...1110,...1111
-            let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
-            if (S::into_stride_size(self.ptrbitarr) & bit_pos) >
-                <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-            {
-                self.bit_span.bits = cursor + 1;
-                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
-            }    
-            
-        }
-        None
-    }
-}
-
-// ----------- NodeMoreSpecificChildIter ------------------------------------
-
-// Create an iterator over all the child nodes that hold a more specific
-// prefixes of the specified start_bit_span. This basically the same Iterator
-// as the ChildNodeIter, except that it stops (potentially) earlier, to avoid
-// including nodes with adjacent prefixes. Starting an iterator with a 
-// `start_bit_span` of { bits: 0, len: 0 } will return all child nodes of 
-// this node. In that case you could also use the `NodeChildIter` instead.
-//
-// inputs
-//
-// `base_prefix`
-// This iterator take a `base_prefix` since the nodes themselves have no
-// knowledge of their own prefixes, those are inferred by their position in
-// the tree (therefore, it's actually a Trie). Note that `base_prefix` +
-// `bit_span` define the actual starting prefix for this iterator.
-//
-// `ptrbitarr` 
-// is the bitmap that holds the slots that have child nodes. 
-//
-// `start_bit_span` 
-// is the bit span that is going to be used as a starting point for the 
-// iterator.
-// 
-// `cursor`
-// holds the current cursor offset from the start_bit_span.bits, the sum of
-// these describe the current position in the bitmap. Used for re-entry into
-// the iterator. A new iterator should start with None.
-//
-// How this works
-//
-// The iterator starts at the start_bit_span.bits position in the bitmap and
-// advances until it reaches either a one in the bitmap, or the maximum
-// position for the particular more-specifics for this bit_span.
-//
-// e.x.
-// The stride size is 5 and the starting bit span is {bits: 2, len: 4} (0010)
-// The starting point is therefore the bit_array 0010. The iterator will go
-// over 0010 0 and 0010 1. The next bits to consider would be 0011 0 which
-// would not fit our starting bit_span of 0010. So we have to stop after 2
-// iterations. This means that the number of iterations is determined by the
-// difference between the number of bits in the stride size (5) and the the
-// number of bits in the start_bit_span (4). The number of iterations in the
-// above example is therefore 1 << (5 - 4) = 2. Remember that a ptrbitarr
-// holds only one stride size (the largest for its stride size), so we're
-// done now.
-#[derive(Debug, Copy, Clone)]
-pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily, S: Stride> {
-   base_prefix: StrideNodeId<AF>,
-   ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
-   start_bit_span: BitSpan,
-   cursor: Option<u32>,
-}
-
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
-    NodeMoreSpecificChildIter<AF, S>
-{
-    type Item = StrideNodeId<AF>;
-    fn next(&mut self) -> Option<Self::Item> {
-        // Early exits
-
-        // Empty bitmap
-        if self.ptrbitarr == <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType::zero() {
-            trace!("empty ptrbitrarr. this iterator is done.");
-            return None;
-        }
-
-        // Previous iteration incremented the cursor beyond the stride size.
-        if let Some(cursor) = self.cursor { 
-            if cursor >= (1 << (S::STRIDE_LEN - self.start_bit_span.len)) {
-                trace!("cursor.bits >= (1 << (S::STRIDE_LEN - self.start_bit_span.len))");
-                trace!("cursor: {}", cursor);
-                trace!("start_bit_span: {} {}", self.start_bit_span.bits, self.start_bit_span.len);
-                return None;
-            }
-        }
-
-        // No early exits, we're in business.
-        trace!("NodeMoreSpecificChildIter");
-        trace!("base_prefix {}", self.base_prefix);
-        trace!("stride_size {}", S::STRIDE_LEN);
-        trace!("start_bit_span bits {} len {} ", self.start_bit_span.bits, self.start_bit_span.len);
-        trace!("cursor bits {:?}", self.cursor);
-        trace!("          x1  4   8  12  16  20  24  28  32");
-        trace!("ptrbitarr {:032b}", self.ptrbitarr);
-
-        let start = if let Some(bits) = self.cursor { bits } else { self.start_bit_span.bits };
-        // We either stop if we have reached the maximum number of bits that
-        // we should check for this bit_span or we stop at the end of the
-        // stride (in case of a start_bit_span.bits == 0).
-        let stop = <u32>::min((1 << (S::STRIDE_LEN - self.start_bit_span.len)) + start, (1 << S::STRIDE_LEN) - 1); 
-
-        trace!("start {:?} stop {}", start, stop);
-        for cursor in start..=stop {
-            // move the bit_span left with the amount of bits we're going to loop over.
-            // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
-            // becomes 0000 0000 0000 1100, then it will iterate over 
-            // ...1100,...1101,...1110,...1111
-            let bit_pos = 
-                S::get_bit_pos(
-                    cursor, S::STRIDE_LEN);
-            trace!("cmpbitarr x{:032b} {}", bit_pos, stop - start);
-            if (S::into_stride_size(self.ptrbitarr) & bit_pos) >
-                <<S as Stride>::AtomicPfxSize as AtomicBitmap
-                >::InnerType::zero()
-            {
-                trace!("bingo!");
-                self.cursor = Some(cursor + 1);
-
-                trace!("next bit_span {} {} with cursor {:?}", self.start_bit_span.bits, self.start_bit_span.len, self.cursor);
-                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
-            }
-        }
-        trace!("No more nodes. End of the iterator.");
-        None
-    }
-}
-
-impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride3> {
-    pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
-        SizedNodeMoreSpecificIter::<AF>::Stride3(self)
-    }
-}
-
-impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride4> {
-    pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
-        SizedNodeMoreSpecificIter::<AF>::Stride4(self)
-    }
-}
-
-impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride5> {
-    pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
-        SizedNodeMoreSpecificIter::<AF>::Stride5(self)
-    }
-}
-
-
-// ----------- NodePrefixIter -----------------------------------------------
-
-// Create an iterator of all prefix ids hosted by this node.
-
-// Partition for stride 3
-//
-// pfxbitarr (AF::BITS)  0 1 2  3  4  5  6   7   8   9  10  11  12  13  14 
-// bit_span (binary)     * 0 1 00 01 10 11 000 001 010 011 100 101 110 111 * *
-// bit_span (dec.)       * 0 1  0  1  2  3   0   1   2   3   4   5   6   7 * *
-// len                   0 1   2           3                               
-//
-// pfxbitarr (example)   1 0 0 0  0  0  1  1   1   0   0   0   0   0   0   0 0
-// pos (example)         0 0 0 0  0  0  0  0   1   0   0   0   0   0   0   0 0                 
-//
-// Ex.:
-// `pos` describes the bit that is currently under consideration. 
-// 
-// `pfxbitarr` is the bitmap that contains the prefixes. Every 1 in the
-// bitmap means that the prefix is hosted by this node. Moreover, the
-// position in the bitmap describes the address part of the prefix, given
-// a `base prefix`. The described prefix is the bits of the `base_prefix`
-// bitmap appended by the `bit span` bits.
-//
-// The length of the prefix is
-// described by sum of the length of the base_prefix and the `len`
-// variable.
-//
-// The `bit_span` variable starts counting at every new prefix length.
-pub(crate) struct NodePrefixIter<AF: AddressFamily, S: Stride> {
-    base_prefix: StrideNodeId<AF>,
-    pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
-    bit_span: BitSpan, // start with 0
-    _af: PhantomData<AF>,
-    _s: PhantomData<S>,
-}
-
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator for 
-    NodePrefixIter<AF, S> {
-        type Item = PrefixId<AF>;
-
-        fn next(&mut self) -> Option<Self::Item> {
-        // iterate over all the possible values for this stride length, e.g.
-        // two bits can have 4 different values.
-        for cursor in self.bit_span.bits..(1 << S::STRIDE_LEN) {
-            
-            let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
-            if self.pfxbitarr & bit_pos >
-                <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-            {
-                self.bit_span.bits = cursor + 1;
-                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN).into());
-            }    
-            
-        }
-        None
-    }
-}
-
-// Creates an Iterator that returns all prefixes that exist in a node that
-// are a more-specific prefix of the `base_prefix` + `start_bit_span`.
-//
-// Inputs
-// 
-// `base_prefix`
-// This iterator take a `base_prefix` since the nodes themselves have no
-// knowledge of their own prefixes, those are inferred by their position in
-// the tree (therefore, it's actually a Trie). Note that `base_prefix` +
-// `bit_span` define the actual starting prefix for this iterator.
-//
-// `pfxbitarr` 
-// is the bitmap that holds the slots that have prefixes.
-//
-// `start_bit_span` 
-// is the bit span that is going to be used as a starting point for the 
-// iterator.
-// 
-// `cursor`
-// holds the current cursor offset from the start_bit_span.bits, the sum of
-// these describe the current position in the bitmap. Used for re-entry into
-// the iterator. A new iterator should start with None.
-//
-// How this works
-//
-// The iterator starts at the start_bit_span.bits position in the bitmap and
-// advances until it reaches either a one in the bitmap, or the maximum
-// position for the particular more-specifics for this bit_span. When it
-// reaches the maximum position it determines whether there are more stride-
-// sizes available in this bitmap. If there are, it advances to the next
-// stride-size in the first position. If not it terminates the iterator.
-//
-// e.x.
-// The stride size is 5 and the starting bit span is {bits: 1, len: 3} (001)
-// This means that the stride size that we have to consider are 4 and 5. 3 
-// being the size of the current bit_span and 5 being the size of the total
-// stride. 
-// The starting point is therefore the bit_array 001. The iterator will go
-// over 001 00, 001 01, 001 10 and 001 11. The next bits to consider would be
-//  010 00 which would not fit our starting bit_span of 0010. So we have to
-// stop after 2 iterations. This means that the number of iterations is
-// determined by the difference between the number of bits in the stride size
-// (5) and the the number of bits in the start_bit_span (4). The number of 
-// iterations in the above example is therefore 1 << (5 - 3) = 4.
-// Unlike the MoreSpecificPrefixIter, we will have to consider more lengths
-// than just the bit_span len. We will have to jump a few pfxbitarr bits and
-// move to the next stride size in the bitmap, starting at bit_array 0010, or
-// the bit_span { bits: 2, len: 3 }, a.k.a. 0010 << 1. But now we will have
-// to go over a different amount of 1 << (5 - 4) = 2 iterations to reap the
-// next bit_spans of 0010 0 and 0010 1.
-
-pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily, S: Stride> {
-    // immutables
-    base_prefix: StrideNodeId<AF>,
-    pfxbitarr: <<S as crate::local_array::atomic_stride::Stride>::AtomicPfxSize 
-        as crate::local_array::atomic_stride::AtomicBitmap>::InnerType,
-    // we need to keep around only the `bits` part of the `bit_span`
-    // technically, (it needs resetting the current state to it after each
-    // prefix-length), but we'll keep the start-length as well for clarity
-    // and increment it on a different field ('cur_len').
-    start_bit_span: BitSpan,
-    cursor: BitSpan,
-    skip_self: bool,
-    _s: PhantomData<S>,
-}
-
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator for 
-    NodeMoreSpecificsPrefixIter<AF, S> {
-        type Item = PrefixId<AF>;
-
-        fn next(&mut self) -> Option<Self::Item> {
-
-            // Easy early exit conditions
-
-            // Empty bitmap
-            if self.pfxbitarr == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-                trace!("empty pfxbitarr. This iterator is done.");
-                return None;
-            }
-
-            // No early exits, We're in business.
-            trace!("len_offset {}", ((1<< self.cursor.len) - 1));
-            trace!("start_bit {}", self.start_bit_span.bits);
-            trace!("number of check bits in len {}", 
-                (1 << (self.cursor.len - self.start_bit_span.len)));
-
-            trace!("next more specifics prefix iter start bits {} len {}",
-                self.start_bit_span.bits, self.start_bit_span.len);
-
-            let mut res = None;
-
-            // Move to the next len if we're at the first prefix-length that matches,
-            // if `skip_self` is set. Typically this flag is set for the first stride.
-            // In the consecutive strides, we don't want to skip the base prefix, since
-            // that base_prefix is a more specific prefix of the one requested in the
-            // first stride.
-            if self.skip_self && (1 << (self.cursor.len - self.start_bit_span.len)) == 1 {
-                // self.cursor.len += (self.start_bit_span.bits & 1) as u8;
-                trace!("skipping self");
-                self.start_bit_span.bits <<= 1;
-                self.cursor.bits = self.start_bit_span.bits;
-                self.cursor.len += 1;
-                self.skip_self = false;
-                trace!("new start bits {} len {}", self.start_bit_span.bits, self.start_bit_span.len);
-                trace!("new cursor bits {} len {}", self.cursor.bits, self.cursor.len);
-            }
-
-            // Previous iteration or the skip_self routine may have
-            // incremented the cursor beyond the end of the stride size.
-            if self.cursor.len > S::STRIDE_LEN {
-                trace!("cursor.len > S::STRIDE_LEN. This iterator is done.");
-                return None;
-            }
-            
-            loop {
-                trace!("          x1  4   8  12  16  20  24  28  32  36  40  44  48  52  56  60  64");
-                trace!("cmpnibble {:064b} ({} + {}) len {} stride_size {}", 
-                    S::get_bit_pos(self.cursor.bits, self.cursor.len), 
-                    (1<< self.cursor.len) - 1, 
-                    self.cursor.bits, 
-                    self.cursor.len + self.base_prefix.get_len(),
-                    S::STRIDE_LEN
-                );
-
-                trace!("pfxbitarr {:064b}", self.pfxbitarr);
-
-                if (S::get_bit_pos(self.cursor.bits, self.cursor.len) | self.pfxbitarr) == self.pfxbitarr {
-                    trace!("found prefix with len {} at pos {} pfx len {}",
-                        self.cursor.len, 
-                        self.cursor.bits,
-                        self.base_prefix.get_len() + self.cursor.len,
-                    );
-                    res = Some(self.base_prefix
-                        .add_nibble(self.cursor.bits, self.cursor.len).into());
-                    trace!("found prefix {:?}", res);
-                }
-
-                // Determine when we're at the end of the bits applicable to
-                // this combo of this start_bit_span.
-                // bitspan offset: 
-                // self.start_bit_span.bits
-                // number of matches in this length:
-                // 1 << (self.cursor.len - self.start_bit_span.len)
-                let max_pos_offset =  
-                    self.start_bit_span.bits + 
-                    (1 << (self.cursor.len - self.start_bit_span.len)) - 1;
-
-                trace!("max_pos_offset {} > cursor bit_pos {}?", max_pos_offset, self.cursor.bits);
-                trace!("number of check bits in len {}", (1 << (self.cursor.len - self.start_bit_span.len)));
-
-                // case 1. At the beginning or inside a prefix-length.
-                if max_pos_offset > self.cursor.bits {
-                    self.cursor.bits += 1;
-                } 
-                // case 2. At the end of a prefix-lengths in this stride.
-                else if self.cursor.len < S::STRIDE_LEN {
-                    trace!("move len to {}", self.cursor.len + 1);
-                    self.start_bit_span.bits <<= 1;
-                    self.cursor.bits = self.start_bit_span.bits;
-                    self.cursor.len += 1;
-                }
-                // case 4. At the end of a prefix-length, but not at the end of the pfxbitarr.
-                else {
-                    self.start_bit_span.bits <<= 1;
-                    self.cursor.bits = self.start_bit_span.bits;
-                    self.cursor.len += 1;
-                    trace!("end of stride, next cursor bits {} len {}", self.cursor.bits, self.cursor.len);
-                    return res;
-                }
-
-                trace!("some res {:?}", res);
-                if res.is_some() { return res; }
-            }
-        }    
-}
-
-impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride3> {
-    pub fn wrap(self) -> SizedPrefixIter<AF> {
-        SizedPrefixIter::<AF>::Stride3(self)
-    }
-}
-
-impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride4> {
-    pub fn wrap(self) -> SizedPrefixIter<AF> {
-        SizedPrefixIter::<AF>::Stride4(self)
-    }
-}
-
-impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride5> {
-    pub fn wrap(self) -> SizedPrefixIter<AF> {
-        SizedPrefixIter::Stride5(self)
-    }
-}
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 05d4f84b..b900f8f1 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -9,7 +9,8 @@ use crossbeam_epoch::{self as epoch, Atomic};
 use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
 
-use crate::local_array::in_memory::tree::{StrideNodeId, TreeBitMap};
+use crate::local_array::in_memory::node::SizedStrideRef;
+use crate::local_array::in_memory::tree::TreeBitMap;
 use crate::local_array::types::PrefixId;
 use crate::stats::CreatedNodes;
 use crate::{
@@ -20,7 +21,7 @@ use crate::local_array::in_memory::atomic_types::PrefixBuckets;
 use crate::local_array::in_memory::atomic_types::{
     NodeBuckets, StoredPrefix,
 };
-use crate::local_array::in_memory::tree::{NewNodeOrIndex, SizedStrideRef};
+use crate::local_array::in_memory::node::{NewNodeOrIndex, StrideNodeId};
 
 // Make sure to also import the other methods for the Rib, so the proc macro
 // create_store can use them.
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index a54cece6..c67e94d2 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -19,7 +19,7 @@ pub mod multi {
     pub use crate::local_array::in_memory::atomic_types::{
         NodeBuckets, NodeSet, PrefixBuckets, PrefixSet,
     };
-    pub use crate::local_array::in_memory::tree::StrideNodeId;
+    pub use crate::local_array::in_memory::node::StrideNodeId;
     pub use crate::local_array::types::{PrefixId, RouteStatus};
     pub use crate::prefix_record::PublicRecord as Record;
 

From 5d8a1e6cc147ec069e78df7c2855fa2534c78fe8 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 19 Dec 2024 14:45:06 +0100
Subject: [PATCH 031/147] split out rib -> inmem - pesist upsert

---
 examples/full_table_multiple_trees_json.rs |  36 +--
 proc_macros/src/lib.rs                     |   3 -
 src/local_array/errors.rs                  |  14 +-
 src/local_array/in_memory/atomic_types.rs  | 184 +++++++------
 src/local_array/in_memory/macros.rs        |  57 ++--
 src/local_array/in_memory/tree.rs          | 250 ++++++++++++++++-
 src/local_array/persist/lsm_tree.rs        |   8 +-
 src/local_array/rib/rib.rs                 | 304 +++++----------------
 tests/best-path.rs                         |   2 +-
 9 files changed, 472 insertions(+), 386 deletions(-)

diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index 333d75cb..35a88f30 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -107,24 +107,24 @@ fn main() -> Result<(), Box<dyn Error>> {
 
             println!("{{");
             println!("\"type\": \"treebitmap_univec\",");
-            println!(
-                "\"strides v4 \": {:?},",
-                &tree_bitmap
-                    .v4
-                    .get_stride_sizes()
-                    .iter()
-                    .map_while(|s| if s > &0 { Some(*s) } else { None })
-                    .collect::<Vec<_>>()
-            );
-            println!(
-                "\"strides v6 \": {:?},",
-                &tree_bitmap
-                    .v6
-                    .get_stride_sizes()
-                    .iter()
-                    .map_while(|s| if s > &0 { Some(*s) } else { None })
-                    .collect::<Vec<_>>()
-            );
+            // println!(
+            //     "\"strides v4 \": {:?},",
+            //     &tree_bitmap
+            //         .v4
+            //         .get_stride_sizes()
+            //         .iter()
+            //         .map_while(|s| if s > &0 { Some(*s) } else { None })
+            //         .collect::<Vec<_>>()
+            // );
+            // println!(
+            //     "\"strides v6 \": {:?},",
+            //     &tree_bitmap
+            //         .v6
+            //         .get_stride_sizes()
+            //         .iter()
+            //         .map_while(|s| if s > &0 { Some(*s) } else { None })
+            //         .collect::<Vec<_>>()
+            // );
             println!("\"run_no\": {},", n);
             println!("\"inserts_num\": {},", inserts_num);
             println!("\"insert_duration_nanos\": {},", dur_insert_nanos);
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index dea0e73e..852de14a 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -1443,7 +1443,6 @@ pub fn create_store(
                 )
             }
 
-
             /// Change the status of all records for this `multi_uniq_id` to
             /// Withdrawn.
             ///
@@ -1469,8 +1468,6 @@ pub fn create_store(
                 res_v4.and(res_v6)
             }
 
-
-
             // Whether the global status for IPv4 prefixes and the specified
             // `multi_uniq_id` is set to `Withdrawn`.
             pub fn mui_is_withdrawn_v4(
diff --git a/src/local_array/errors.rs b/src/local_array/errors.rs
index f38c9298..38001254 100644
--- a/src/local_array/errors.rs
+++ b/src/local_array/errors.rs
@@ -10,6 +10,7 @@ pub enum PrefixStoreError {
     BestPathNotFound,
     RecordNotInMemory,
     PersistFailed,
+    StatusUnknown,
 }
 
 impl std::error::Error for PrefixStoreError {}
@@ -28,7 +29,11 @@ impl fmt::Display for PrefixStoreError {
                 write!(f, "Error: Store isn't ready yet.")
             }
             PrefixStoreError::PathSelectionOutdated => {
-                write!(f, "Error: The Path Selection process is based on outdated paths.")
+                write!(
+                    f,
+                    "Error: The Path Selection process is based on \
+                    outdated paths."
+                )
             }
             PrefixStoreError::PrefixNotFound => {
                 write!(f, "Error: The Prefix cannot be found.")
@@ -53,6 +58,13 @@ impl fmt::Display for PrefixStoreError {
                     persisted."
                 )
             }
+            PrefixStoreError::StatusUnknown => {
+                write!(
+                    f,
+                    "Warning: The record is persisted, but the upsert \
+                    counters cannot be reported for persist only strategy."
+                )
+            }
         }
     }
 }
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 3da94d04..12563cbb 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -510,97 +510,115 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     // record.multi_uniq_id. Returns the number of entries in the HashMap
     // after updating it, if it's more than 1. Returns None if this is the
     // first entry.
-    pub(crate) fn upsert_record<
-        AF: AddressFamily,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    >(
+    pub(crate) fn upsert_record(
         &self,
-        prefix: PrefixId<AF>,
+        // prefix: PrefixId<AF>,
         new_rec: PublicRecord<M>,
-        persistence: &Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
-        strategy: PersistStrategy,
-    ) -> Result<(Option<usize>, usize), PrefixStoreError> {
+        persist_status: PersistStatus,
+        // persistence: &Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
+        // strategy: PersistStrategy,
+    ) -> Result<(Option<(MultiMapValue<M>, usize)>, usize), PrefixStoreError>
+    {
         let (mut record_map, retry_count) = self.guard_with_retry(0);
         let key = new_rec.multi_uniq_id;
 
-        match (strategy, record_map.get_mut(&key)) {
-            // New record for (prefix, mui) in memory.
-
-            // We store the record in memory only.
-            (
-                PersistStrategy::PersistHistory | PersistStrategy::MemoryOnly,
-                None,
-            ) => {
-                record_map.insert(
-                    key,
-                    MultiMapValue::from((
-                        new_rec,
-                        PersistStatus::not_persisted(),
-                    )),
-                );
-
-                Ok((None, retry_count))
-            }
-            // We only persist the record.
-            (PersistStrategy::PersistOnly, None) => {
-                if let Some(persistence) = persistence {
-                    persistence.persist_record(prefix, key, &new_rec);
-                    Ok((None, retry_count))
-                } else {
-                    Err(PrefixStoreError::PersistFailed)
-                }
-            }
-            // We store both in memory and persist it.
-            (PersistStrategy::WriteAhead, None) => {
-                if let Some(persistence) = persistence {
-                    persistence.persist_record(prefix, key, &new_rec);
-                    let mmv = MultiMapValue::from((
-                        new_rec,
-                        PersistStatus::persisted(),
-                    ));
-                    record_map.insert(key, mmv);
-
-                    Ok((None, retry_count))
-                } else {
-                    Err(PrefixStoreError::PersistFailed)
-                }
-            }
-
-            // Existing record for (prefix, mui) in memory.
-
-            // We store the record in memory only, and discard the old record.
-            (PersistStrategy::MemoryOnly, Some(exist_rec)) => {
-                *exist_rec = MultiMapValue::from((
-                    new_rec,
-                    PersistStatus::not_persisted(),
-                ));
-
-                Ok((Some(record_map.len()), retry_count))
-            }
-            // We only persist record, so how come there's one in memory?
-            // Should not happen.
-            (PersistStrategy::PersistOnly, Some(_)) => {
-                panic!("Encountered illegally stored record");
+        match record_map.contains_key(&key) {
+            true => {
+                let old_rec = record_map
+                    .insert(
+                        key,
+                        MultiMapValue::from((new_rec, persist_status)),
+                    )
+                    .map(|r| (r, record_map.len()));
+                Ok((old_rec, retry_count))
             }
-            // We store the new record in memory and persist the old record.
-            (
-                PersistStrategy::PersistHistory | PersistStrategy::WriteAhead,
-                Some(exist_rec),
-            ) => {
-                if let Some(persistence) = persistence {
-                    persistence.persist_record(prefix, key, &new_rec);
-                    *exist_rec = MultiMapValue::from((
-                        new_rec,
-                        PersistStatus::persisted(),
-                    ));
-
-                    Ok((Some(record_map.len()), retry_count))
-                } else {
-                    Err(PrefixStoreError::PersistFailed)
-                }
+            false => {
+                let new_rec = MultiMapValue::from((new_rec, persist_status));
+                let old_rec = record_map.insert(key, new_rec);
+                assert!(old_rec.is_none());
+                Ok((None, retry_count))
             }
         }
+
+        // Ok((record_map.len(), retry_count))
+
+        // match (strategy, record_map.get_mut(&key)) {
+        //     // New record for (prefix, mui) in memory.
+
+        //     // We store the record in memory only.
+        //     (
+        //         PersistStrategy::PersistHistory | PersistStrategy::MemoryOnly,
+        //         None,
+        //     ) => {
+        //         record_map.insert(
+        //             key,
+        //             MultiMapValue::from((
+        //                 new_rec,
+        //                 PersistStatus::not_persisted(),
+        //             )),
+        //         );
+
+        //         Ok((None, retry_count))
+        //     }
+        //     // We only persist the record.
+        //     (PersistStrategy::PersistOnly, None) => {
+        //         if let Some(persistence) = persistence {
+        //             persistence.persist_record(prefix, key, &new_rec);
+        //             Ok((None, retry_count))
+        //         } else {
+        //             Err(PrefixStoreError::PersistFailed)
+        //         }
+        //     }
+        //     // We store both in memory and persist it.
+        //     (PersistStrategy::WriteAhead, None) => {
+        //         if let Some(persistence) = persistence {
+        //             persistence.persist_record(prefix, key, &new_rec);
+        //             let mmv = MultiMapValue::from((
+        //                 new_rec,
+        //                 PersistStatus::persisted(),
+        //             ));
+        //             record_map.insert(key, mmv);
+
+        //             Ok((None, retry_count))
+        //         } else {
+        //             Err(PrefixStoreError::PersistFailed)
+        //         }
+        //     }
+
+        //     // Existing record for (prefix, mui) in memory.
+
+        //     // We store the record in memory only, and discard the old record.
+        //     (PersistStrategy::MemoryOnly, Some(exist_rec)) => {
+        //         *exist_rec = MultiMapValue::from((
+        //             new_rec,
+        //             PersistStatus::not_persisted(),
+        //         ));
+
+        //         Ok((Some(record_map.len()), retry_count))
+        //     }
+        //     // We only persist record, so how come there's one in memory?
+        //     // Should not happen.
+        //     (PersistStrategy::PersistOnly, Some(_)) => {
+        //         panic!("Encountered illegally stored record");
+        //     }
+        //     // We store the new record in memory and persist the old record.
+        //     (
+        //         PersistStrategy::PersistHistory | PersistStrategy::WriteAhead,
+        //         Some(exist_rec),
+        //     ) => {
+        //         if let Some(persistence) = persistence {
+        //             persistence.persist_record(prefix, key, &new_rec);
+        //             *exist_rec = MultiMapValue::from((
+        //                 new_rec,
+        //                 PersistStatus::persisted(),
+        //             ));
+
+        //             Ok((Some(record_map.len()), retry_count))
+        //         } else {
+        //             Err(PrefixStoreError::PersistFailed)
+        //         }
+        //     }
+        // }
     }
 }
 
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index c953f05a..371344fb 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -11,8 +11,8 @@ macro_rules! insert_match {
         $nibble: expr; // nibble is a variable-length bitarray (1,2,4,8,etc)
         $is_last_stride: expr;
         $pfx: ident; // the whole search prefix
-        $record: ident; // the record holding the metadata
-        $update_path_selections: ident; // boolean indicate whether to update the path selections for this route
+        $mui: ident; // the reccord holding the metadata
+        // $update_path_selections: ident; // boolean indicate whether to update the path selections for this route
         $truncate_len: ident; // the start of the length of this stride
         $stride_len: ident; // the length of this stride
         $cur_i: expr; // the id of the current node in this stride
@@ -47,8 +47,8 @@ macro_rules! insert_match {
             // retry_count from this macro.
             let local_retry_count = 0;
             // retrieve_node_mut updates the bitmap index if necessary.
-            if let Some(current_node) = $self.in_memory_tree.retrieve_node_mut(
-                $cur_i, $record.multi_uniq_id) {
+            if let Some(current_node) = $self.retrieve_node_mut(
+                $cur_i, $mui) {
                 match current_node {
                     $(
                         SizedStrideRef::$variant(current_node) => {
@@ -83,13 +83,13 @@ macro_rules! insert_match {
                                             $truncate_len + $nibble_len
                                        );
 
-                                    // store the new node in the global
-                                    // store. It returns the created id
-                                    // and the number of retries before
+                                    // store the new node in the in_memory
+                                    // part of the RIB. It returns the created
+                                    // id and the number of retries before
                                     // success.
-                                    match $self.in_memory_tree.store_node(
+                                    match $self.store_node(
                                         new_id,
-                                        $record.multi_uniq_id, n
+                                        $mui, n
                                     ) {
                                         Ok((node_id, s_retry_count)) => {
                                             Ok((
@@ -126,38 +126,23 @@ macro_rules! insert_match {
                                     ))
                                 },
                                 (NewNodeOrIndex::NewPrefix, retry_count) => {
-                                    return $self.upsert_prefix(
-                                        $pfx,
-                                        $record,
-                                        $update_path_selections,
-                                        $guard
-                                    ).and_then(|mut r| {
-                                            r.cas_count +=
-                                                $acc_retry_count as usize +
-                                                local_retry_count as usize +
-                                                retry_count as usize;
-                                            Ok(r)
-                                        })
-                                    // Log
-                                    // $self.stats[$stats_level].
-                                    //inc_prefix_count($level);
-                                }
+                                    break
+                                        $acc_retry_count +
+                                        local_retry_count +
+                                        retry_count
+
+                                },
                                 (
                                     NewNodeOrIndex::ExistingPrefix,
                                      retry_count
                                  ) =>
                                      {
-                                        return $self.upsert_prefix(
-                                            $pfx,
-                                            $record,
-                                            $update_path_selections,$guard
-                                        ).and_then(|mut r| {
-                                            r.cas_count +=
-                                                $acc_retry_count as usize +
-                                                local_retry_count as usize +
-                                                retry_count as usize;
-                                            Ok(r)
-                                        })
+                                    break
+                                        $acc_retry_count +
+                                        local_retry_count +
+                                        retry_count
+
+
                                 }
                             }   // end of eval_node_or_prefix_at
                         }
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 8496b3d5..f8f62188 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -187,28 +187,29 @@ use crate::local_array::bit_span::BitSpan;
 use crate::local_array::in_memory::atomic_types::StoredNode;
 use crate::prefix_record::Meta;
 use crate::prelude::multi::{NodeSet, PrefixId};
+use crossbeam_epoch::Guard;
 use crossbeam_utils::Backoff;
-use log::{debug, log_enabled, trace};
+use log::{debug, error, log_enabled, trace};
 
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
 use std::{fmt::Debug, marker::PhantomData};
 
 use super::atomic_types::{
-    NodeBuckets, PrefixBuckets, PrefixSet, StoredPrefix,
+    MultiMapValue, NodeBuckets, PersistStatus, PrefixBuckets, PrefixSet,
+    StoredPrefix,
 };
 use crate::af::AddressFamily;
-use crate::rib::Counters;
+use crate::rib::{Counters, UpsertReport};
 use crate::{
-    impl_search_level, impl_search_level_for_mui, retrieve_node_mut_closure,
-    store_node_closure,
+    impl_search_level, impl_search_level_for_mui, insert_match,
+    retrieve_node_mut_closure, store_node_closure, PublicRecord,
 };
 
 use super::super::errors::PrefixStoreError;
 pub(crate) use super::atomic_stride::*;
+use crate::local_array::in_memory::node::{NewNodeOrIndex, StrideNodeId};
 
-use super::node::{
-    SizedStrideNode, SizedStrideRef, StrideNodeId, TreeBitMapNode,
-};
+use super::node::{SizedStrideNode, SizedStrideRef, TreeBitMapNode};
 
 #[cfg(feature = "cli")]
 use ansi_term::Colour;
@@ -278,6 +279,234 @@ impl<
         Ok(tree_bitmap)
     }
 
+    pub(crate) fn set_prefix_exists(
+        &self,
+        pfx: PrefixId<AF>,
+        mui: u32,
+        // update_path_selections: Option<M::TBI>,
+    ) -> Result<u32, PrefixStoreError> {
+        if pfx.get_len() == 0 {
+            return self.update_default_route_prefix_meta(mui);
+            // return Ok(res);
+        }
+
+        let mut stride_end: u8 = 0;
+        let mut cur_i = self.get_root_node_id();
+        let mut level: u8 = 0;
+        let mut acc_retry_count = 0;
+
+        let a = loop {
+            let stride = self.get_stride_sizes()[level as usize];
+            stride_end += stride;
+            let nibble_len = if pfx.get_len() < stride_end {
+                stride + pfx.get_len() - stride_end
+            } else {
+                stride
+            };
+
+            let nibble = AF::get_nibble(
+                pfx.get_net(),
+                stride_end - stride,
+                nibble_len,
+            );
+            let is_last_stride = pfx.get_len() <= stride_end;
+            let stride_start = stride_end - stride;
+            // let back_off = crossbeam_utils::Backoff::new();
+
+            // insert_match! returns the node_id of the next node to be
+            // traversed. It was created if it did not exist.
+            let node_result = insert_match![
+                // applicable to the whole outer match in the macro
+                self;
+                guard;
+                nibble_len;
+                nibble;
+                is_last_stride;
+                pfx;
+                mui;
+                // record;
+                // perform an update for the paths in this record
+                // update_path_selections;
+                // the length at the start of the stride a.k.a. start_bit
+                stride_start;
+                stride;
+                cur_i;
+                level;
+                acc_retry_count;
+                // Strides to create match arm for; stats level
+                Stride3; 0,
+                Stride4; 1,
+                Stride5; 2
+            ];
+
+            match node_result {
+                Ok((next_id, retry_count)) => {
+                    cur_i = next_id;
+                    level += 1;
+                    acc_retry_count += retry_count;
+                }
+                Err(err) => {
+                    if log_enabled!(log::Level::Error) {
+                        error!(
+                            "{} failing to store (intermediate) node {}.
+                             Giving up this node. This shouldn't happen!",
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread"),
+                            cur_i,
+                        );
+                        error!(
+                            "{} {}",
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread"),
+                            err
+                        );
+                    }
+                }
+            }
+        };
+
+        Ok(a)
+    }
+
+    pub(crate) fn upsert_prefix(
+        &self,
+        prefix: PrefixId<AF>,
+        record: PublicRecord<M>,
+        persist_status: PersistStatus,
+        update_path_selections: Option<M::TBI>,
+        guard: &Guard,
+    ) -> Result<(UpsertReport, Option<MultiMapValue<M>>), PrefixStoreError>
+    {
+        let mut prefix_is_new = true;
+        let mut mui_is_new = true;
+
+        let (mui_count, cas_count) = match self
+            .non_recursive_retrieve_prefix_mut(prefix)
+        {
+            // There's no StoredPrefix at this location yet. Create a new
+            // PrefixRecord and try to store it in the empty slot.
+            (stored_prefix, false) => {
+                if log_enabled!(log::Level::Debug) {
+                    debug!(
+                        "{} store: Create new prefix record",
+                        std::thread::current()
+                            .name()
+                            .unwrap_or("unnamed-thread")
+                    );
+                }
+
+                let (mui_count, retry_count) = stored_prefix
+                    .record_map
+                    .upsert_record(record, persist_status)?;
+
+                // See if someone beat us to creating the record.
+                if mui_count.is_some() {
+                    mui_is_new = false;
+                    prefix_is_new = false;
+                } else {
+                    // No, we were the first, we created a new prefix
+                    self.counters.inc_prefixes_count(prefix.get_len());
+                }
+
+                (mui_count, retry_count)
+            }
+            // There already is a StoredPrefix with a record at this
+            // location.
+            (stored_prefix, true) => {
+                if log_enabled!(log::Level::Debug) {
+                    debug!(
+                        "{} store: Found existing prefix record for {}/{}",
+                        std::thread::current()
+                            .name()
+                            .unwrap_or("unnamed-thread"),
+                        prefix.get_net(),
+                        prefix.get_len()
+                    );
+                }
+                prefix_is_new = false;
+
+                // Update the already existing record_map with our
+                // caller's record.
+                stored_prefix.set_ps_outdated(guard)?;
+
+                let (mui_count, retry_count) = stored_prefix
+                    .record_map
+                    .upsert_record(record, persist_status)?;
+                mui_is_new = mui_count.is_none();
+
+                if let Some(tbi) = update_path_selections {
+                    stored_prefix
+                        .calculate_and_store_best_backup(&tbi, guard)?;
+                }
+
+                (mui_count, retry_count)
+            }
+        };
+
+        let count = mui_count.as_ref().map(|m| m.1).unwrap_or(1);
+        Ok((
+            UpsertReport {
+                prefix_new: prefix_is_new,
+                cas_count,
+                mui_new: mui_is_new,
+                mui_count: count,
+            },
+            mui_count.map(|m| m.0),
+        ))
+    }
+
+    // Yes, we're hating this. But, the root node has no room for a serial of
+    // the prefix 0/0 (the default route), which doesn't even matter, unless,
+    // UNLESS, somebody wants to store a default route. So we have to store a
+    // serial for this prefix. The normal place for a serial of any prefix is
+    // on the pfxvec of its paren. But, hey, guess what, the
+    // default-route-prefix lives *on* the root node, and, you know, the root
+    // node doesn't have a parent. We can:
+    // - Create a type RootTreeBitmapNode with a ptrbitarr with a size one
+    //   bigger than a "normal" TreeBitMapNod for the first stride size. no we
+    //   have to iterate over the root-node type in all matches on
+    //   stride_size, just because we have exactly one instance of the
+    //   RootTreeBitmapNode. So no.
+    // - Make the `get_pfx_index` method on the implementations of the
+    //   `Stride` trait check for a length of zero and branch if it is and
+    //   return the serial of the root node. Now each and every call to this
+    //   method will have to check a condition for exactly one instance of
+    //   RootTreeBitmapNode. So again, no.
+    // - The root node only gets used at the beginning of a search query or an
+    //   insert. So if we provide two specialised methods that will now how to
+    //   search for the default-route prefix and now how to set serial for
+    //  that prefix and make sure we start searching/inserting with one of
+    //   those specialized methods we're good to go.
+    fn update_default_route_prefix_meta(
+        &self,
+        mui: u32,
+    ) -> Result<u32, PrefixStoreError> {
+        trace!("Updating the default route...");
+
+        if let Some(root_node) =
+            self.retrieve_node_mut(self.get_root_node_id(), mui)
+        {
+            match root_node {
+                SizedStrideRef::Stride3(_) => self
+                    .node_buckets
+                    .get_store3(self.get_root_node_id())
+                    .update_rbm_index(mui),
+                SizedStrideRef::Stride4(_) => self
+                    .node_buckets
+                    .get_store4(self.get_root_node_id())
+                    .update_rbm_index(mui),
+                SizedStrideRef::Stride5(_) => self
+                    .node_buckets
+                    .get_store5(self.get_root_node_id())
+                    .update_rbm_index(mui),
+            }
+        } else {
+            Err(PrefixStoreError::StoreNotReadyError)
+        }
+    }
+
     // Create a new node in the store with payload `next_node`.
     //
     // Next node will be ignored if a node with the same `id` already exists,
@@ -449,8 +678,8 @@ impl<
         }
     }
 
-    // retrieve a node, but only its bitmap index contains the specified mui.
-    // Used for iterators per mui.
+    // retrieve a node, but only if its bitmap index contains the specified
+    // mui. Used for iterators per mui.
     #[allow(clippy::type_complexity)]
     pub(crate) fn retrieve_node_for_mui(
         &self,
@@ -541,7 +770,6 @@ impl<
             );
 
             // probe the slot with the index that's the result of the hashing.
-            // let locked_prefix = prefix_set.0.get(index);
             let stored_prefix = match prefix_set.0.get(index) {
                 Some(p) => {
                     trace!("prefix set found.");
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 9ff09d3e..13965aba 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -85,8 +85,8 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             .collect::<Vec<_>>()
     }
 
-    fn match_prefix_in_persisted_store<'a, M: Meta>(
-        &'a self,
+    fn match_prefix_in_persisted_store<M: Meta>(
+        &self,
         search_pfx: PrefixId<AF>,
         mui: Option<u32>,
     ) -> QueryResult<M> {
@@ -221,13 +221,13 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     pub(crate) fn persist_record<M: Meta>(
         &self,
         prefix: PrefixId<AF>,
-        mui: u32,
+        // mui: u32,
         record: &PublicRecord<M>,
     ) {
         self.insert(
             PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
                 prefix,
-                mui,
+                record.multi_uniq_id,
                 record.ltime,
                 record.status,
             ),
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index b900f8f1..4429a373 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -17,10 +17,12 @@ use crate::{
     local_array::errors::PrefixStoreError, prefix_record::PublicRecord,
 };
 
-use crate::local_array::in_memory::atomic_types::PrefixBuckets;
 use crate::local_array::in_memory::atomic_types::{
     NodeBuckets, StoredPrefix,
 };
+use crate::local_array::in_memory::atomic_types::{
+    PersistStatus, PrefixBuckets,
+};
 use crate::local_array::in_memory::node::{NewNodeOrIndex, StrideNodeId};
 
 // Make sure to also import the other methods for the Rib, so the proc macro
@@ -267,93 +269,31 @@ impl<
 
     pub fn insert(
         &self,
-        pfx: PrefixId<AF>,
+        prefix: PrefixId<AF>,
         record: PublicRecord<M>,
         update_path_selections: Option<M::TBI>,
     ) -> Result<UpsertReport, PrefixStoreError> {
         let guard = &epoch::pin();
-
-        if pfx.get_len() == 0 {
-            let res = self.update_default_route_prefix_meta(record, guard)?;
-            return Ok(res);
-        }
-
-        let mut stride_end: u8 = 0;
-        let mut cur_i = self.in_memory_tree.get_root_node_id();
-        let mut level: u8 = 0;
-        let mut acc_retry_count = 0;
-
-        loop {
-            let stride =
-                self.in_memory_tree.get_stride_sizes()[level as usize];
-            stride_end += stride;
-            let nibble_len = if pfx.get_len() < stride_end {
-                stride + pfx.get_len() - stride_end
-            } else {
-                stride
-            };
-
-            let nibble = AF::get_nibble(
-                pfx.get_net(),
-                stride_end - stride,
-                nibble_len,
-            );
-            let is_last_stride = pfx.get_len() <= stride_end;
-            let stride_start = stride_end - stride;
-            // let back_off = crossbeam_utils::Backoff::new();
-
-            // insert_match! returns the node_id of the next node to be
-            // traversed. It was created if it did not exist.
-            let node_result = insert_match![
-                // applicable to the whole outer match in the macro
-                self;
-                guard;
-                nibble_len;
-                nibble;
-                is_last_stride;
-                pfx;
-                record;
-                // perform an update for the paths in this record
-                update_path_selections;
-                // the length at the start of the stride a.k.a. start_bit
-                stride_start;
-                stride;
-                cur_i;
-                level;
-                acc_retry_count;
-                // Strides to create match arm for; stats level
-                Stride3; 0,
-                Stride4; 1,
-                Stride5; 2
-            ];
-
-            match node_result {
-                Ok((next_id, retry_count)) => {
-                    cur_i = next_id;
-                    level += 1;
-                    acc_retry_count += retry_count;
-                }
-                Err(err) => {
-                    if log_enabled!(log::Level::Error) {
-                        error!(
-                            "{} failing to store (intermediate) node {}.
-                             Giving up this node. This shouldn't happen!",
-                            std::thread::current()
-                                .name()
-                                .unwrap_or("unnamed-thread"),
-                            cur_i,
-                        );
-                        error!(
-                            "{} {}",
-                            std::thread::current()
-                                .name()
-                                .unwrap_or("unnamed-thread"),
-                            err
-                        );
+        self.in_memory_tree
+            .set_prefix_exists(prefix, record.multi_uniq_id)
+            .and_then(|c1| {
+                self.upsert_prefix(
+                    prefix,
+                    record,
+                    update_path_selections,
+                    guard,
+                )
+                .map(|mut c| {
+                    if c.prefix_new {
+                        self.counters.inc_prefixes_count(prefix.get_len());
                     }
-                }
-            }
-        }
+                    if c.mui_new {
+                        self.counters.inc_routes_count();
+                    }
+                    c.cas_count += c1 as usize;
+                    c
+                })
+            })
     }
 
     fn upsert_prefix(
@@ -363,151 +303,64 @@ impl<
         update_path_selections: Option<M::TBI>,
         guard: &Guard,
     ) -> Result<UpsertReport, PrefixStoreError> {
-        let mut prefix_is_new = true;
-        let mut mui_is_new = true;
-
-        let (mui_count, cas_count) = match self
-            .in_memory_tree
-            .non_recursive_retrieve_prefix_mut(prefix)
-        {
-            // There's no StoredPrefix at this location yet. Create a new
-            // PrefixRecord and try to store it in the empty slot.
-            (locked_prefix, false) => {
-                if log_enabled!(log::Level::Debug) {
-                    debug!(
-                        "{} store: Create new prefix record",
-                        std::thread::current()
-                            .name()
-                            .unwrap_or("unnamed-thread")
-                    );
-                }
+        let mui = record.multi_uniq_id;
+        match self.config.persist_strategy {
+            PersistStrategy::WriteAhead => {
+                if let Some(persist_tree) = &self.persist_tree {
+                    persist_tree.persist_record(prefix, &record);
 
-                let (mui_count, retry_count) =
-                    locked_prefix.record_map.upsert_record(
-                        prefix,
-                        record,
-                        &self.persist_tree,
-                        self.config.persist_strategy,
-                    )?;
-
-                // See if someone beat us to creating the record.
-                if mui_count.is_some() {
-                    mui_is_new = false;
-                    prefix_is_new = false;
+                    self.in_memory_tree
+                        .upsert_prefix(
+                            prefix,
+                            record,
+                            PersistStatus::persisted(),
+                            update_path_selections,
+                            guard,
+                        )
+                        .map(|(report, _old_rec)| report)
                 } else {
-                    // No, we were the first, we created a new prefix
-                    self.counters.inc_prefixes_count(prefix.get_len());
+                    Err(PrefixStoreError::StoreNotReadyError)
                 }
-
-                (mui_count, retry_count)
             }
-            // There already is a StoredPrefix with a record at this
-            // location.
-            (stored_prefix, true) => {
-                if log_enabled!(log::Level::Debug) {
-                    debug!(
-                        "{} store: Found existing prefix record for {}/{}",
-                        std::thread::current()
-                            .name()
-                            .unwrap_or("unnamed-thread"),
-                        prefix.get_net(),
-                        prefix.get_len()
-                    );
-                }
-                prefix_is_new = false;
-
-                // Update the already existing record_map with our
-                // caller's record.
-                stored_prefix.set_ps_outdated(guard)?;
-
-                let (mui_count, retry_count) =
-                    stored_prefix.record_map.upsert_record(
-                        prefix,
-                        record,
-                        &self.persist_tree,
-                        self.config.persist_strategy,
-                    )?;
-                mui_is_new = mui_count.is_none();
-
-                if let Some(tbi) = update_path_selections {
-                    stored_prefix
-                        .calculate_and_store_best_backup(&tbi, guard)?;
+            PersistStrategy::PersistHistory => self
+                .in_memory_tree
+                .upsert_prefix(
+                    prefix,
+                    record,
+                    PersistStatus::not_persisted(),
+                    update_path_selections,
+                    guard,
+                )
+                .map(|(report, old_rec)| {
+                    if let Some(rec) = old_rec {
+                        if let Some(persist_tree) = &self.persist_tree {
+                            persist_tree.persist_record(
+                                prefix,
+                                &PublicRecord::from((mui, &rec)),
+                            );
+                        }
+                    }
+                    report
+                }),
+            PersistStrategy::MemoryOnly => self
+                .in_memory_tree
+                .upsert_prefix(
+                    prefix,
+                    record,
+                    PersistStatus::not_persisted(),
+                    update_path_selections,
+                    guard,
+                )
+                .map(|(report, _)| report),
+            PersistStrategy::PersistOnly => {
+                if let Some(persist_tree) = &self.persist_tree {
+                    persist_tree.persist_record(prefix, &record);
+                    Err(PrefixStoreError::StatusUnknown)
+                } else {
+                    Err(PrefixStoreError::PersistFailed)
                 }
-
-                (mui_count, retry_count)
             }
-        };
-
-        Ok(UpsertReport {
-            prefix_new: prefix_is_new,
-            cas_count,
-            mui_new: mui_is_new,
-            mui_count: mui_count.unwrap_or(1),
-        })
-    }
-
-    // Yes, we're hating this. But, the root node has no room for a serial of
-    // the prefix 0/0 (the default route), which doesn't even matter, unless,
-    // UNLESS, somebody wants to store a default route. So we have to store a
-    // serial for this prefix. The normal place for a serial of any prefix is
-    // on the pfxvec of its paren. But, hey, guess what, the
-    // default-route-prefix lives *on* the root node, and, you know, the root
-    // node doesn't have a parent. We can:
-    // - Create a type RootTreeBitmapNode with a ptrbitarr with a size one
-    //   bigger than a "normal" TreeBitMapNod for the first stride size. no we
-    //   have to iterate over the root-node type in all matches on
-    //   stride_size, just because we have exactly one instance of the
-    //   RootTreeBitmapNode. So no.
-    // - Make the `get_pfx_index` method on the implementations of the
-    //   `Stride` trait check for a length of zero and branch if it is and
-    //   return the serial of the root node. Now each and every call to this
-    //   method will have to check a condition for exactly one instance of
-    //   RootTreeBitmapNode. So again, no.
-    // - The root node only gets used at the beginning of a search query or an
-    //   insert. So if we provide two specialised methods that will now how to
-    //   search for the default-route prefix and now how to set serial for
-    //  that prefix and make sure we start searching/inserting with one of
-    //   those specialized methods we're good to go.
-    fn update_default_route_prefix_meta(
-        &self,
-        record: PublicRecord<M>,
-        guard: &epoch::Guard,
-    ) -> Result<UpsertReport, PrefixStoreError> {
-        trace!("Updating the default route...");
-
-        if let Some(root_node) = self.in_memory_tree.retrieve_node_mut(
-            self.in_memory_tree.get_root_node_id(),
-            record.multi_uniq_id,
-        ) {
-            match root_node {
-                SizedStrideRef::Stride3(_) => {
-                    self.in_memory_tree
-                        .node_buckets
-                        .get_store3(self.in_memory_tree.get_root_node_id())
-                        .update_rbm_index(record.multi_uniq_id)?;
-                }
-                SizedStrideRef::Stride4(_) => {
-                    self.in_memory_tree
-                        .node_buckets
-                        .get_store4(self.in_memory_tree.get_root_node_id())
-                        .update_rbm_index(record.multi_uniq_id)?;
-                }
-                SizedStrideRef::Stride5(_) => {
-                    self.in_memory_tree
-                        .node_buckets
-                        .get_store5(self.in_memory_tree.get_root_node_id())
-                        .update_rbm_index(record.multi_uniq_id)?;
-                }
-            };
-        };
-
-        self.upsert_prefix(
-            PrefixId::new(AF::zero(), 0),
-            record,
-            // Do not update the path selection for the default route.
-            None,
-            guard,
-        )
+        }
     }
 
     pub fn get_nodes_count(&self) -> usize {
@@ -688,13 +541,6 @@ impl<
         }
     }
 
-    // Stride related methods
-
-    // Pass through the in_memory stride sizes, for printing purposes
-    pub fn get_stride_sizes(&self) -> &[u8] {
-        self.in_memory_tree.node_buckets.get_stride_sizes()
-    }
-
     //-------- Persistence ---------------------------------------------------
 
     pub fn persist_strategy(&self) -> PersistStrategy {
diff --git a/tests/best-path.rs b/tests/best-path.rs
index fa37ce2d..1f4f986a 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -46,7 +46,7 @@ impl AsRef<[u8]> for Ipv4Route {
 }
 
 impl From<Vec<u8>> for Ipv4Route {
-    fn from(value: Vec<u8>) -> Self {
+    fn from(_value: Vec<u8>) -> Self {
         todo!()
     }
 }

From 7c78a9103b7ede0a395234afbd2e98ba5b390053 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 19 Dec 2024 14:54:37 +0100
Subject: [PATCH 032/147] clean

---
 src/local_array/in_memory/atomic_types.rs | 2 --
 src/local_array/rib/rib.rs                | 6 ++----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 12563cbb..c3b336bc 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -13,12 +13,10 @@ use log::{debug, log_enabled, trace};
 use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
 
-use crate::local_array::persist::lsm_tree::PersistTree;
 use crate::local_array::types::{PrefixId, RouteStatus};
 // use crate::local_array::in_memory_tree::*;
 use crate::prefix_record::PublicRecord;
 use crate::prelude::Meta;
-use crate::rib::PersistStrategy;
 use crate::AddressFamily;
 
 use super::super::errors::PrefixStoreError;
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 4429a373..63f2674c 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -3,13 +3,12 @@ use std::path::Path;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use inetnum::addr::Prefix;
-use log::{debug, error, info, log_enabled, trace};
+use log::info;
 
 use crossbeam_epoch::{self as epoch, Atomic};
 use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
 
-use crate::local_array::in_memory::node::SizedStrideRef;
 use crate::local_array::in_memory::tree::TreeBitMap;
 use crate::local_array::types::PrefixId;
 use crate::stats::CreatedNodes;
@@ -23,14 +22,13 @@ use crate::local_array::in_memory::atomic_types::{
 use crate::local_array::in_memory::atomic_types::{
     PersistStatus, PrefixBuckets,
 };
-use crate::local_array::in_memory::node::{NewNodeOrIndex, StrideNodeId};
 
 // Make sure to also import the other methods for the Rib, so the proc macro
 // create_store can use them.
 pub use crate::local_array::iterators;
 pub use crate::local_array::query;
 
-use crate::{insert_match, IPv4, IPv6, MatchType, Meta, QueryResult};
+use crate::{IPv4, IPv6, MatchType, Meta, QueryResult};
 
 use crate::AddressFamily;
 

From 5744b852d46ae31b753174b52a1e333eb366a5a8 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 19 Dec 2024 17:20:10 +0100
Subject: [PATCH 033/147] some minor shuffling between files

---
 src/local_array/in_memory/tree.rs   |  2 +-
 src/local_array/iterators.rs        | 27 +++++++++-----
 src/local_array/persist/lsm_tree.rs | 42 +++++++++++++---------
 src/local_array/query.rs            | 26 +++++++++++---
 src/local_array/rib/rib.rs          | 56 ++++++-----------------------
 src/rotonda_store.rs                | 20 +++++++++--
 6 files changed, 96 insertions(+), 77 deletions(-)

diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index f8f62188..01bbc285 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -515,7 +515,7 @@ impl<
     // Returns: a tuple with the node_id of the created node and the number of
     // retry_count
     #[allow(clippy::type_complexity)]
-    pub(crate) fn store_node(
+    fn store_node(
         &self,
         id: StrideNodeId<AF>,
         multi_uniq_id: u32,
diff --git a/src/local_array/iterators.rs b/src/local_array/iterators.rs
index 0f3cd15b..6ce002a0 100644
--- a/src/local_array/iterators.rs
+++ b/src/local_array/iterators.rs
@@ -1,17 +1,28 @@
 // ----------- Store Iterators ----------------------------------------------
 //
-// This file hosts the iterators for the CustomAllocStorage type and the
-// implementations for the methods that start'em.
-// Note that these iterators are only the iterators that go over the
-// storage (and some over the TreeBitMap nodes, the parent of the store),
-// as such all the iterators here are composed of iterators over the
-// individual nodes. The Node Iterators live in the node.rs file.
+// This file hosts the iterators for the Rib and implementations for the
+// methods that start'em. There are 3 Iterators:
+//
+// 1. an iterator `PrefixIter` that iterates over ALL of the prefix buckets of
+// the CHT backing the TreeBitMap.
+//
+// 2. a MoreSpecificsIterator that starts from a prefix in the prefix buckets
+// for that particular prefix length, but uses the node in the TreeBitMap to
+// find its more specifics.
+//
+// 3. a LessSpecificIterator, that just reduces the prefix size bit-by-bit and
+// looks in the prefix buckets for the diminuishing prefix.
+//
+// The Iterators that start from the root node of the TreeBitMap (which
+// is the only option for the single-threaded TreeBitMap) live in the
+// deprecated_node.rs file. They theoretically should be slower and cause more
+// contention, since every lookup has to go through the levels near the root
+// in the TreeBitMap.
 
 use super::in_memory::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
-use super::in_memory::node::StrideNodeId;
+use super::in_memory::node::{SizedStrideRef, StrideNodeId};
 use super::in_memory::tree::{Stride3, Stride4, Stride5};
 use super::types::PrefixId;
-use crate::local_array::in_memory::node::SizedStrideRef;
 use crate::local_array::types::RouteStatus;
 use crate::prefix_record::PublicRecord;
 use crate::rib;
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 13965aba..f53bdc2e 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -85,7 +85,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             .collect::<Vec<_>>()
     }
 
-    fn match_prefix_in_persisted_store<M: Meta>(
+    pub(crate) fn match_prefix<M: Meta>(
         &self,
         search_pfx: PrefixId<AF>,
         mui: Option<u32>,
@@ -96,17 +96,27 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         } else {
             search_pfx.as_bytes::<PREFIX_SIZE>().to_vec()
         };
-
-        QueryResult {
-            prefix: Some(search_pfx.into_pub()),
-            match_type: MatchType::ExactMatch,
-            prefix_meta: self
-                .get_records_for_key(&key)
-                .into_iter()
-                .map(|(_, rec)| rec)
-                .collect::<Vec<_>>(),
-            less_specifics: None,
-            more_specifics: None,
+        let recs = self.get_records_for_key(&key);
+
+        if !recs.is_empty() {
+            QueryResult {
+                prefix: Some(search_pfx.into_pub()),
+                match_type: MatchType::ExactMatch,
+                prefix_meta: recs
+                    .into_iter()
+                    .map(|(_, rec)| rec)
+                    .collect::<Vec<_>>(),
+                less_specifics: None,
+                more_specifics: None,
+            }
+        } else {
+            QueryResult {
+                prefix: Some(search_pfx.into_pub()),
+                match_type: MatchType::EmptyMatch,
+                prefix_meta: vec![],
+                less_specifics: None,
+                more_specifics: None,
+            }
         }
     }
 
@@ -212,10 +222,10 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         )
     }
 
-    #[cfg(feature = "persist")]
-    pub fn parse_prefix(bytes: &[u8]) -> [u8; PREFIX_SIZE] {
-        *bytes.first_chunk::<PREFIX_SIZE>().unwrap()
-    }
+    // #[cfg(feature = "persist")]
+    // pub fn parse_prefix(bytes: &[u8]) -> [u8; PREFIX_SIZE] {
+    //     *bytes.first_chunk::<PREFIX_SIZE>().unwrap()
+    // }
 
     #[cfg(feature = "persist")]
     pub(crate) fn persist_record<M: Meta>(
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 54e0ee21..dd2ef915 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -54,7 +54,12 @@ where
                 None
             },
             prefix_meta: prefix
-                .map(|r| self.get_filtered_records(r, mui, guard))
+                .map(|r| {
+                    r.record_map.get_filtered_records(
+                        mui,
+                        self.withdrawn_muis_bmin(guard),
+                    )
+                })
                 .unwrap_or_default(),
             match_type: MatchType::EmptyMatch,
             less_specifics: None,
@@ -95,7 +100,12 @@ where
                 None
             },
             prefix_meta: prefix
-                .map(|r| self.get_filtered_records(r, mui, guard))
+                .map(|r| {
+                    r.record_map.get_filtered_records(
+                        mui,
+                        self.withdrawn_muis_bmin(guard),
+                    )
+                })
                 .unwrap_or_default(),
             match_type: MatchType::EmptyMatch,
             less_specifics: less_specifics_vec.map(|iter| iter.collect()),
@@ -135,7 +145,11 @@ where
     ) -> QueryResult<M> {
         match self.config.persist_strategy() {
             PersistStrategy::PersistOnly => {
-                self.match_prefix_in_persisted_store(search_pfx, options.mui)
+                if let Some(persist_tree) = &self.persist_tree {
+                    persist_tree.match_prefix(search_pfx, options.mui)
+                } else {
+                    QueryResult::empty()
+                }
             }
             _ => self.match_prefix_in_memory(search_pfx, options, guard),
         }
@@ -161,7 +175,11 @@ where
                         // with globally withdrawn muis, and with local
                         // statuses
                         // set to Withdrawn.
-                        self.get_filtered_records(pfx, options.mui, guard)
+                        pfx.record_map
+                            .get_filtered_records(
+                                options.mui,
+                                self.withdrawn_muis_bmin(guard),
+                            )
                             .into_iter()
                             .collect()
                     } else {
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 63f2674c..8a91c430 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -218,7 +218,8 @@ pub struct Rib<
     pub config: StoreConfig,
     pub(in crate::local_array) in_memory_tree: TreeBitMap<AF, M, NB, PB>,
     #[cfg(feature = "persist")]
-    persist_tree: Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
+    pub(in crate::local_array) persist_tree:
+        Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
     // Global Roaring BitMap INdex that stores MUIs.
     pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     pub counters: Counters,
@@ -504,16 +505,16 @@ impl<
 
     // Helper to filter out records that are not-active (Inactive or
     // Withdrawn), or whose mui appears in the global withdrawn index.
-    pub(crate) fn get_filtered_records(
-        &self,
-        pfx: &StoredPrefix<AF, M>,
-        mui: Option<u32>,
-        guard: &Guard,
-    ) -> Vec<PublicRecord<M>> {
-        let bmin = self.withdrawn_muis_bmin(guard);
+    // pub(crate) fn get_filtered_records(
+    //     &self,
+    //     pfx: &StoredPrefix<AF, M>,
+    //     mui: Option<u32>,
+    //     guard: &Guard,
+    // ) -> Vec<PublicRecord<M>> {
+    //     let bmin = self.withdrawn_muis_bmin(guard);
 
-        pfx.record_map.get_filtered_records(mui, bmin)
-    }
+    //     pfx.record_map.get_filtered_records(mui, bmin)
+    // }
 
     pub fn get_prefixes_count(&self) -> UpsertCounters {
         UpsertCounters {
@@ -545,41 +546,6 @@ impl<
         self.config.persist_strategy
     }
 
-    pub fn match_prefix_in_persisted_store(
-        &'a self,
-        search_pfx: PrefixId<AF>,
-        mui: Option<u32>,
-    ) -> QueryResult<M> {
-        let key: Vec<u8> = if let Some(mui) = mui {
-            PersistTree::<AF,
-        PREFIX_SIZE, KEY_SIZE>::prefix_mui_persistence_key(search_pfx, mui)
-        } else {
-            search_pfx.as_bytes::<PREFIX_SIZE>().to_vec()
-        };
-
-        if let Some(persist) = &self.persist_tree {
-            QueryResult {
-                prefix: Some(search_pfx.into_pub()),
-                match_type: MatchType::ExactMatch,
-                prefix_meta: persist
-                    .get_records_for_key(&key)
-                    .into_iter()
-                    .map(|(_, rec)| rec)
-                    .collect::<Vec<_>>(),
-                less_specifics: None,
-                more_specifics: None,
-            }
-        } else {
-            QueryResult {
-                prefix: None,
-                match_type: MatchType::EmptyMatch,
-                prefix_meta: vec![],
-                less_specifics: None,
-                more_specifics: None,
-            }
-        }
-    }
-
     pub fn get_records_for_prefix(
         &self,
         prefix: &Prefix,
diff --git a/src/rotonda_store.rs b/src/rotonda_store.rs
index 3fd262d6..3247078a 100644
--- a/src/rotonda_store.rs
+++ b/src/rotonda_store.rs
@@ -155,19 +155,33 @@ impl<M: Meta> Iterator for PrefixSingleRecordIter<'_, M> {
 /// See [MultiThreadedStore::match_prefix] for more details.
 
 #[derive(Clone, Debug)]
-pub struct QueryResult<M: crate::prefix_record::Meta> {
+pub struct QueryResult<M: Meta> {
     /// The match type of the resulting prefix
     pub match_type: MatchType,
     /// The resulting prefix record
     pub prefix: Option<Prefix>,
     /// The meta data associated with the resulting prefix record
     pub prefix_meta: Vec<PublicRecord<M>>,
-    /// The less-specifics of the resulting prefix together with their meta data
+    /// The less-specifics of the resulting prefix together with their meta
+    /// data
     pub less_specifics: Option<RecordSet<M>>,
-    /// The more-specifics of the resulting prefix together with their meta data
+    /// The more-specifics of the resulting prefix together with their meta
+    //// data
     pub more_specifics: Option<RecordSet<M>>,
 }
 
+impl<M: Meta> QueryResult<M> {
+    pub fn empty() -> Self {
+        QueryResult {
+            match_type: MatchType::EmptyMatch,
+            prefix: None,
+            prefix_meta: vec![],
+            less_specifics: None,
+            more_specifics: None,
+        }
+    }
+}
+
 impl<M: Meta> fmt::Display for QueryResult<M> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         let pfx_str = match self.prefix {

From f3b2470f998fd8955ffc2c1a18eadd21daec804f Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 24 Dec 2024 15:49:40 +0100
Subject: [PATCH 034/147] IncludeHistory + prefix_exists in memory

---
 examples/exact_matches.rs                     |   3 +-
 examples/exact_matches_single.rs              |   5 +-
 examples/full_table_multiple_trees_json.rs    |   1 +
 examples/more_specifics.rs                    |   1 +
 examples/multi_no_thread.rs                   |   1 +
 examples/multi_thread_1.rs                    |   1 +
 examples/multi_thread_2.rs                    |   1 +
 examples/multi_thread_3.rs                    |   1 +
 examples/multi_thread_4.rs                    |   3 +-
 examples/multi_thread_multi_prefix.rs         |   3 +-
 examples/multi_thread_single_prefix.rs        |   3 +-
 examples/treebitmap.rs                        |   1 +
 proc_macros/src/lib.rs                        |  13 +-
 src/bin/cli.rs                                |   5 +-
 src/bin/load_mrt.rs                           |   8 +-
 src/local_array/in_memory/atomic_types.rs     |   6 +-
 src/local_array/in_memory/deprecated_query.rs |  52 +-
 src/local_array/in_memory/macros.rs           |  15 +-
 src/local_array/in_memory/mod.rs              |   5 +-
 src/local_array/in_memory/node.rs             |  48 ++
 src/local_array/in_memory/query.rs            | 612 ++++++++++++++++++
 src/local_array/in_memory/tree.rs             |  90 ++-
 src/local_array/persist/lsm_tree.rs           | 221 ++++++-
 src/local_array/query.rs                      |  19 +-
 src/local_array/rib/rib.rs                    |  25 +-
 src/local_vec/tests/full_table_single.rs      |   2 +
 src/local_vec/tests/more_specifics_single.rs  |   2 +
 src/prelude/mod.rs                            |   2 +-
 src/rotonda_store.rs                          |  14 +
 tests/best-path.rs                            |   2 +
 tests/concurrency.rs                          |   7 +-
 tests/full-table.rs                           |   2 +
 tests/more-more-specifics.rs                  |   2 +
 tests/more-specifics.rs                       |   1 +
 tests/treebitmap.rs                           |  14 +
 tests/treebitmap_v6.rs                        |   7 +
 36 files changed, 1093 insertions(+), 105 deletions(-)
 create mode 100644 src/local_array/in_memory/query.rs

diff --git a/examples/exact_matches.rs b/examples/exact_matches.rs
index 02e0dcdd..16d3a5b3 100644
--- a/examples/exact_matches.rs
+++ b/examples/exact_matches.rs
@@ -1,6 +1,6 @@
 use rotonda_store::meta_examples::NoMeta;
 use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
+use rotonda_store::{prelude::*, IncludeHistory};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let guard = &epoch::pin();
@@ -348,6 +348,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_less_specifics: false,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
diff --git a/examples/exact_matches_single.rs b/examples/exact_matches_single.rs
index c0b91e67..d2fd69dd 100644
--- a/examples/exact_matches_single.rs
+++ b/examples/exact_matches_single.rs
@@ -1,6 +1,6 @@
+use rotonda_store::meta_examples::NoMeta;
 use rotonda_store::prelude::*;
 use rotonda_store::SingleThreadedStore;
-use rotonda_store::meta_examples::NoMeta;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let v4 = vec![8];
@@ -345,7 +345,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_withdrawn: false,
                 include_less_specifics: false,
                 include_more_specifics: false,
-                mui: None
+                mui: None,
+                include_history: IncludeHistory::None,
             },
         );
         println!("exact match: {:?}", s_spfx);
diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index 35a88f30..a0c69074 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -92,6 +92,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                                     include_less_specifics: false,
                                     include_more_specifics: false,
                                     mui: None,
+                                    include_history: IncludeHistory::None,
                                 },
                                 guard,
                             );
diff --git a/examples/more_specifics.rs b/examples/more_specifics.rs
index 12cef348..e982cbe9 100644
--- a/examples/more_specifics.rs
+++ b/examples/more_specifics.rs
@@ -288,6 +288,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_less_specifics: true,
                 include_more_specifics: true,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
diff --git a/examples/multi_no_thread.rs b/examples/multi_no_thread.rs
index 300adfe6..b6a6291b 100644
--- a/examples/multi_no_thread.rs
+++ b/examples/multi_no_thread.rs
@@ -58,6 +58,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_less_specifics: true,
             include_more_specifics: true,
             mui: None,
+            include_history: IncludeHistory::None,
         },
         guard,
     );
diff --git a/examples/multi_thread_1.rs b/examples/multi_thread_1.rs
index 7b992f26..441542c7 100644
--- a/examples/multi_thread_1.rs
+++ b/examples/multi_thread_1.rs
@@ -55,6 +55,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_less_specifics: true,
                 include_more_specifics: true,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
diff --git a/examples/multi_thread_2.rs b/examples/multi_thread_2.rs
index 510cfc00..53301668 100644
--- a/examples/multi_thread_2.rs
+++ b/examples/multi_thread_2.rs
@@ -71,6 +71,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_less_specifics: true,
             include_more_specifics: true,
             mui: None,
+            include_history: IncludeHistory::None,
         },
         guard,
     );
diff --git a/examples/multi_thread_3.rs b/examples/multi_thread_3.rs
index 13885b71..0be99aa2 100644
--- a/examples/multi_thread_3.rs
+++ b/examples/multi_thread_3.rs
@@ -97,6 +97,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_less_specifics: true,
             include_more_specifics: true,
             mui: None,
+            include_history: IncludeHistory::None,
         },
         guard,
     );
diff --git a/examples/multi_thread_4.rs b/examples/multi_thread_4.rs
index ac04db46..627fc8ad 100644
--- a/examples/multi_thread_4.rs
+++ b/examples/multi_thread_4.rs
@@ -125,7 +125,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_withdrawn: true,
             include_less_specifics: true,
             include_more_specifics: true,
-            mui: None
+            mui: None,
+            include_history: IncludeHistory::None,
         },
         guard,
     );
diff --git a/examples/multi_thread_multi_prefix.rs b/examples/multi_thread_multi_prefix.rs
index 08000fb0..908a8428 100644
--- a/examples/multi_thread_multi_prefix.rs
+++ b/examples/multi_thread_multi_prefix.rs
@@ -71,7 +71,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                     include_withdrawn: true,
                                     include_less_specifics: true,
                                     include_more_specifics: true,
-                                    mui: None
+                                    mui: None,
+                                    include_history: IncludeHistory::None,
                                 },
                                 guard,
                             ).prefix_meta;
diff --git a/examples/multi_thread_single_prefix.rs b/examples/multi_thread_single_prefix.rs
index e9dcf5e3..db3af6e1 100644
--- a/examples/multi_thread_single_prefix.rs
+++ b/examples/multi_thread_single_prefix.rs
@@ -67,7 +67,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                             include_withdrawn: true,
                             include_less_specifics: true,
                             include_more_specifics: true,
-                            mui: None
+                            mui: None,
+                            include_history: IncludeHistory::None,
                         },
                         guard,
                     ).prefix_meta;
diff --git a/examples/treebitmap.rs b/examples/treebitmap.rs
index f7389de6..b208e01b 100644
--- a/examples/treebitmap.rs
+++ b/examples/treebitmap.rs
@@ -298,6 +298,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_less_specifics: true,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 852de14a..13c4a080 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -1572,9 +1572,9 @@ pub fn create_store(
             pub fn print_funky_stats(&self) {
                 println!("");
                 println!("Stats for IPv4 multi-threaded store\n");
-                println!("{}", self.v4);
+                println!("{}", self.v4.in_memory_tree);
                 println!("Stats for IPv6 multi-threaded store\n");
-                println!("{}", self.v6);
+                println!("{}", self.v6.in_memory_tree);
             }
 
             // The Store statistics.
@@ -1591,11 +1591,14 @@ pub fn create_store(
                 self.config.persist_strategy()
             }
 
-            pub fn get_records_for_prefix(&self, prefix: &Prefix) ->
+            pub fn get_records_for_prefix(
+                &self, prefix: &Prefix,
+                mui: Option<u32>
+            ) ->
                 Vec<Record<M>> {
                 match prefix.is_v4() {
-                    true => self.v4.get_records_for_prefix(prefix),
-                    false => self.v6.get_records_for_prefix(prefix)
+                    true => self.v4.get_records_for_prefix(prefix, mui),
+                    false => self.v6.get_records_for_prefix(prefix, mui)
                 }
             }
 
diff --git a/src/bin/cli.rs b/src/bin/cli.rs
index eac5661b..05de15b5 100644
--- a/src/bin/cli.rs
+++ b/src/bin/cli.rs
@@ -228,6 +228,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                         include_less_specifics: true,
                                         include_more_specifics: true,
                                         mui: None,
+                                        include_history: IncludeHistory::None,
                                     },
                                     guard,
                                 );
@@ -290,7 +291,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                             include_withdrawn: true,
                                             include_less_specifics: true,
                                             include_more_specifics: true,
-                                            mui: None
+                                            mui: None,
+                                            include_history:
+                                                IncludeHistory::None
                                         },
                                         guard
                                     )
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index c8d1692a..17b26c86 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -588,11 +588,15 @@ fn main() {
         mib_total as f64 / t_total.elapsed().as_secs() as f64
     );
 
+    if let Some(s) = store {
+        s.print_funky_stats();
+    }
+
     if args.verify {
         println!("\nverifying disk persistence...");
         let mut max_len = 0;
         for pfx in persisted_prefixes {
-            let values = store.unwrap().get_records_for_prefix(&pfx);
+            let values = store.unwrap().get_records_for_prefix(&pfx, None);
             if values.is_empty() {
                 eprintln!("Found empty prefix on disk");
                 eprintln!("prefix: {}", pfx);
@@ -600,7 +604,7 @@ fn main() {
             }
             if values.len() > max_len {
                 max_len = values.len();
-                let recs = store.unwrap().get_records_for_prefix(&pfx);
+                let recs = store.unwrap().get_records_for_prefix(&pfx, None);
                 println!("LEN {} prefix: {}", max_len, pfx);
                 for rec in recs {
                     let pa = OwnedPathAttributes::from((
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index c3b336bc..d96d78d8 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -66,16 +66,16 @@ impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
     pub fn update_rbm_index(
         &self,
         multi_uniq_id: u32,
-    ) -> Result<u32, crate::prelude::multi::PrefixStoreError>
+    ) -> Result<(u32, bool), crate::prelude::multi::PrefixStoreError>
     where
         S: atomic_stride::Stride,
         AF: crate::AddressFamily,
     {
         let try_count = 0;
         let mut rbm = self.1.write().unwrap();
-        rbm.insert(multi_uniq_id);
+        let absent = rbm.insert(multi_uniq_id);
 
-        Ok(try_count)
+        Ok((try_count, !absent))
     }
 
     pub fn remove_from_rbm_index(
diff --git a/src/local_array/in_memory/deprecated_query.rs b/src/local_array/in_memory/deprecated_query.rs
index 8c943e9e..0a912c2b 100644
--- a/src/local_array/in_memory/deprecated_query.rs
+++ b/src/local_array/in_memory/deprecated_query.rs
@@ -82,7 +82,7 @@ where
     // This function assembles all entries in the `pfx_vec` of all child nodes
     // of the `start_node` into one vec, starting from itself and then
     // recursively assembling adding all `pfx_vec`s of its children.
-    fn get_all_more_specifics_for_node(
+    fn _get_all_more_specifics_for_node(
         &self,
         start_node_id: StrideNodeId<AF>,
         found_pfx_vec: &mut Vec<PrefixId<AF>>,
@@ -95,7 +95,7 @@ where
                 );
 
                 for child_node in n.ptr_iter(start_node_id) {
-                    self.get_all_more_specifics_for_node(
+                    self._get_all_more_specifics_for_node(
                         child_node,
                         found_pfx_vec,
                     );
@@ -107,7 +107,7 @@ where
                 );
 
                 for child_node in n.ptr_iter(start_node_id) {
-                    self.get_all_more_specifics_for_node(
+                    self._get_all_more_specifics_for_node(
                         child_node,
                         found_pfx_vec,
                     );
@@ -119,7 +119,7 @@ where
                 );
 
                 for child_node in n.ptr_iter(start_node_id) {
-                    self.get_all_more_specifics_for_node(
+                    self._get_all_more_specifics_for_node(
                         child_node,
                         found_pfx_vec,
                     );
@@ -135,7 +135,7 @@ where
     // specified bit position in a ptr_vec of `current_node` into a vec,
     // then adds all prefixes of these children recursively into a vec and
     // returns that.
-    fn get_all_more_specifics_from_nibble<S: Stride>(
+    fn _get_all_more_specifics_from_nibble<S: Stride>(
         &self,
         current_node: &TreeBitMapNode<AF, S>,
         nibble: u32,
@@ -149,7 +149,7 @@ where
         );
 
         for child_node in cnvec.iter() {
-            self.get_all_more_specifics_for_node(*child_node, &mut msvec);
+            self._get_all_more_specifics_for_node(*child_node, &mut msvec);
         }
         Some(msvec)
     }
@@ -175,7 +175,7 @@ where
     // nibble              1010 1011 1100 1101 1110 1111    x
     // nibble len offset      4(contd.)
 
-    fn match_prefix_by_tree_traversal(
+    fn _match_prefix_by_tree_traversal(
         &'a self,
         search_pfx: PrefixId<AF>,
         options: &MatchOptions,
@@ -202,18 +202,10 @@ where
             //     }
 
             //     _serial => {
-            let prefix_meta = self
-                .retrieve_prefix(PrefixId::new(AF::zero(), 0))
-                .map(|sp| sp.0.record_map.as_records())
-                .unwrap_or_default();
             return QueryResult {
-                prefix: Prefix::new(
-                    search_pfx.get_net().into_ipaddr(),
-                    search_pfx.get_len(),
-                )
-                .ok(),
-                prefix_meta,
-                match_type: MatchType::ExactMatch,
+                prefix: None,
+                prefix_meta: vec![],
+                match_type: MatchType::EmptyMatch,
                 less_specifics: None,
                 more_specifics: None,
             };
@@ -337,7 +329,7 @@ where
                             if last_stride {
                                 if options.include_more_specifics {
                                     more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
+                                        ._get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
                                             nibble_len,
@@ -356,7 +348,7 @@ where
                             if last_stride {
                                 if options.include_more_specifics {
                                     more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
+                                        ._get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
                                             nibble_len,
@@ -375,7 +367,7 @@ where
                         (None, Some(pfx_idx)) => {
                             if options.include_more_specifics {
                                 more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
+                                    ._get_all_more_specifics_from_nibble(
                                         current_node,
                                         nibble,
                                         nibble_len,
@@ -400,7 +392,7 @@ where
                                     // match arm more then once, we return
                                     // early here.
                                     more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
+                                        ._get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
                                             nibble_len,
@@ -455,7 +447,7 @@ where
                             if last_stride {
                                 if options.include_more_specifics {
                                     more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
+                                        ._get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
                                             nibble_len,
@@ -474,7 +466,7 @@ where
                             if last_stride {
                                 if options.include_more_specifics {
                                     more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
+                                        ._get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
                                             nibble_len,
@@ -490,7 +482,7 @@ where
                         (None, Some(pfx_idx)) => {
                             if options.include_more_specifics {
                                 more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
+                                    ._get_all_more_specifics_from_nibble(
                                         current_node,
                                         nibble,
                                         nibble_len,
@@ -509,7 +501,7 @@ where
                                     // To make sure we don't process this match arm more then once, we
                                     // return early here.
                                     more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
+                                        ._get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
                                             nibble_len,
@@ -562,7 +554,7 @@ where
                             if last_stride {
                                 if options.include_more_specifics {
                                     more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
+                                        ._get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
                                             nibble_len,
@@ -581,7 +573,7 @@ where
                             if last_stride {
                                 if options.include_more_specifics {
                                     more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
+                                        ._get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
                                             nibble_len,
@@ -597,7 +589,7 @@ where
                         (None, Some(pfx_idx)) => {
                             if options.include_more_specifics {
                                 more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
+                                    ._get_all_more_specifics_from_nibble(
                                         current_node,
                                         nibble,
                                         nibble_len,
@@ -614,7 +606,7 @@ where
                             match options.match_type {
                                 MatchType::EmptyMatch => {
                                     more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
+                                        ._get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
                                             nibble_len,
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index 371344fb..96c88d1e 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -126,10 +126,15 @@ macro_rules! insert_match {
                                     ))
                                 },
                                 (NewNodeOrIndex::NewPrefix, retry_count) => {
-                                    break
+                                    $self.counters.inc_prefixes_count(
+                                        $pfx.get_len()
+                                    );
+                                    break (
                                         $acc_retry_count +
                                         local_retry_count +
-                                        retry_count
+                                        retry_count,
+                                        false
+                                    )
 
                                 },
                                 (
@@ -137,10 +142,12 @@ macro_rules! insert_match {
                                      retry_count
                                  ) =>
                                      {
-                                    break
+                                    break (
                                         $acc_retry_count +
                                         local_retry_count +
-                                        retry_count
+                                        retry_count,
+                                        true
+                                    )
 
 
                                 }
diff --git a/src/local_array/in_memory/mod.rs b/src/local_array/in_memory/mod.rs
index 164b8dc7..9b6ac496 100644
--- a/src/local_array/in_memory/mod.rs
+++ b/src/local_array/in_memory/mod.rs
@@ -4,8 +4,9 @@ pub(crate) mod node;
 mod oncebox;
 pub(crate) mod tree;
 
-#[deprecated]
-mod deprecated_query;
+// #[deprecated]
+// mod deprecated_query;
+mod query;
 
 #[macro_use]
 mod macros;
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index c91a4f0f..0aedb6e5 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -458,6 +458,54 @@ where
         )
     }
 
+
+    pub(crate) fn prefix_exists_in_stride(
+        &'_ self,
+        search_pfx: PrefixId<AF>,
+        nibble: u32,
+        nibble_len: u8,
+        start_bit: u8,
+    ) -> (Option<StrideNodeId<AF>>, bool) {
+        let pfxbitarr = self.pfxbitarr.load();
+        let ptrbitarr = self.ptrbitarr.load();
+        // This is an exact match, so we're only considering the position of
+        // the full nibble.
+        let bit_pos = S::get_bit_pos(nibble, nibble_len);
+        let mut found_pfx = false;
+        let mut found_child = None;
+
+        // Is this the last nibble?
+        // Otherwise we're not looking for a prefix (exact matching only
+        // lives at last nibble)
+        match search_pfx.get_len() <= start_bit + nibble_len {
+            // We're at the last nibble.
+            true => {
+                // Check for an actual prefix at the right position, i.e. 
+                // consider the complete nibble.
+                if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
+                    found_pfx = true;
+                }
+            }
+            // We're not at the last nibble.
+            false => {
+                // Check for a child node at the right position, i.e.
+                // consider the complete nibble.
+                if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+                {
+                    found_child = Some(
+                        StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)
+                    );
+                }
+            }
+        }
+
+        (
+            found_child, /* The node that has children in the next stride, if
+                            any */
+            found_pfx,   /* The exactly matching prefix, if any */
+        )
+    }
+
     // This function looks for the exactly matching prefix in the provided
     // nibble, just like the one above, but this *does* iterate over all the
     // bytes in the nibble to collect the less-specific prefixes of the the 
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
new file mode 100644
index 00000000..1f8734f3
--- /dev/null
+++ b/src/local_array/in_memory/query.rs
@@ -0,0 +1,612 @@
+use log::trace;
+
+use crate::af::AddressFamily;
+use crate::rib::query::TreeQueryResult;
+
+use crate::Meta;
+
+use crate::local_array::in_memory::node::{SizedStrideRef, TreeBitMapNode};
+use crate::{MatchOptions, MatchType};
+
+use super::super::in_memory::tree::Stride;
+use super::super::types::PrefixId;
+use super::atomic_types::{NodeBuckets, PrefixBuckets};
+use super::node::StrideNodeId;
+use super::tree::TreeBitMap;
+
+impl<AF, M, NB, PB> TreeBitMap<AF, M, NB, PB>
+where
+    AF: AddressFamily,
+    M: Meta,
+    NB: NodeBuckets<AF>,
+    PB: PrefixBuckets<AF, M>,
+{
+    // This function assembles all entries in the `pfx_vec` of all child nodes
+    // of the `start_node` into one vec, starting from itself and then
+    // recursively assembling adding all `pfx_vec`s of its children.
+    fn get_all_more_specifics_for_node(
+        &self,
+        start_node_id: StrideNodeId<AF>,
+        found_pfx_vec: &mut Vec<PrefixId<AF>>,
+    ) {
+        trace!("{:?}", self.retrieve_node(start_node_id));
+        match self.retrieve_node(start_node_id) {
+            Some(SizedStrideRef::Stride3(n)) => {
+                found_pfx_vec.extend(
+                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
+                );
+
+                for child_node in n.ptr_iter(start_node_id) {
+                    self.get_all_more_specifics_for_node(
+                        child_node,
+                        found_pfx_vec,
+                    );
+                }
+            }
+            Some(SizedStrideRef::Stride4(n)) => {
+                found_pfx_vec.extend(
+                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
+                );
+
+                for child_node in n.ptr_iter(start_node_id) {
+                    self.get_all_more_specifics_for_node(
+                        child_node,
+                        found_pfx_vec,
+                    );
+                }
+            }
+            Some(SizedStrideRef::Stride5(n)) => {
+                found_pfx_vec.extend(
+                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
+                );
+
+                for child_node in n.ptr_iter(start_node_id) {
+                    self.get_all_more_specifics_for_node(
+                        child_node,
+                        found_pfx_vec,
+                    );
+                }
+            }
+            _ => {
+                panic!("can't find node {}", start_node_id);
+            }
+        }
+    }
+
+    // This function assembles the prefixes of a child node starting on a
+    // specified bit position in a ptr_vec of `current_node` into a vec,
+    // then adds all prefixes of these children recursively into a vec and
+    // returns that.
+    fn get_all_more_specifics_from_nibble<S: Stride>(
+        &self,
+        current_node: &TreeBitMapNode<AF, S>,
+        nibble: u32,
+        nibble_len: u8,
+        base_prefix: StrideNodeId<AF>,
+    ) -> Option<Vec<PrefixId<AF>>> {
+        let (cnvec, mut msvec) = current_node.add_more_specifics_at(
+            nibble,
+            nibble_len,
+            base_prefix,
+        );
+
+        for child_node in cnvec.iter() {
+            self.get_all_more_specifics_for_node(*child_node, &mut msvec);
+        }
+        Some(msvec)
+    }
+
+    // In a LMP search we have to go over all the nibble lengths in the
+    // stride up until the value of the actual nibble length were looking for
+    // (until we reach stride length for all strides that aren't the last)
+    // and see if the prefix bit in that position is set. Note that this does
+    // not search for prefixes with length 0 (which would always match).
+    // So for matching a nibble 1010, we have to search for 1, 10, 101 and
+    // 1010 on resp. position 1, 5, 12 and 25:
+    //                       ↓          ↓                         ↓
+    // nibble              * 0 1 00 01 10 11 000 001 010 011 100 101 110 111
+    // nibble len offset   0 1    2            3
+    //
+    // (contd.)
+    // pfx bit arr (u32)     15   16   17   18   19   20   21   22   23   24
+    // nibble              0000 0001 0010 0011 0100 0101 0110 0111 1000 1001
+    // nibble len offset      4
+    //
+    // (contd.)               ↓
+    // pfx bit arr (u32)     25   26   27   28   29   30   31
+    // nibble              1010 1011 1100 1101 1110 1111    x
+    // nibble len offset      4(contd.)
+
+    pub(crate) fn match_prefix_by_tree_traversal(
+        &self,
+        search_pfx: PrefixId<AF>,
+        options: &MatchOptions,
+        // guard: &'a Guard,
+    ) -> TreeQueryResult<AF> {
+        // --- The Default Route Prefix -------------------------------------
+
+        // The Default Route Prefix unfortunately does not fit in tree as we
+        // have it. There's no room for it in the pfxbitarr of the root node,
+        // since that can only contain serial numbers for prefixes that are
+        // children of the root node. We, however, want the default prefix
+        // which lives on the root node itself! We are *not* going to return
+        // all of the prefixes in the tree as more-specifics.
+        if search_pfx.get_len() == 0 {
+            // match self.load_default_route_prefix_serial() {
+            //     0 => {
+            //         return QueryResult {
+            //             prefix: None,
+            //             prefix_meta: vec![],
+            //             match_type: MatchType::EmptyMatch,
+            //             less_specifics: None,
+            //             more_specifics: None,
+            //         };
+            //     }
+
+            //     _serial => {
+            return TreeQueryResult {
+                prefix: None,
+                match_type: MatchType::EmptyMatch,
+                less_specifics: None,
+                more_specifics: None,
+            };
+            // }
+            // }
+        }
+
+        let mut stride_end = 0;
+
+        let root_node_id = self.get_root_node_id();
+        let mut node = match self.get_stride_for_id(root_node_id) {
+            3 => self.retrieve_node(root_node_id).unwrap(),
+            4 => self.retrieve_node(root_node_id).unwrap(),
+            _ => self.retrieve_node(root_node_id).unwrap(),
+        };
+
+        let mut nibble;
+        let mut nibble_len;
+
+        //---- result values ------------------------------------------------
+
+        // These result values are kept in mutable variables, and assembled
+        // at the end into a QueryResult struct. This proved to result in the
+        // most efficient code, where we don't have to match on
+        // SizedStrideNode over and over. The `match_type` field in the
+        // QueryResult is computed at the end.
+
+        // The final prefix
+        let mut match_prefix_idx: Option<PrefixId<AF>> = None;
+
+        // The indexes of the less-specifics
+        let mut less_specifics_vec = if options.include_less_specifics {
+            Some(Vec::<PrefixId<AF>>::new())
+        } else {
+            None
+        };
+
+        // The indexes of the more-specifics.
+        let mut more_specifics_vec = if options.include_more_specifics {
+            Some(Vec::<PrefixId<AF>>::new())
+        } else {
+            None
+        };
+
+        //---- Stride Processing --------------------------------------------
+
+        // We're going to iterate over all the strides in the treebitmap (so
+        // up to the last bit in the max prefix length for that tree). When
+        // a final prefix is found or we get to the end of the strides,
+        // depending on the options.match_type (the type requested by the
+        // user). we ALWAYS break out of the loop. WE ALWAYS BREAK OUT OF THE
+        // LOOP. Just before breaking some processing is done inside the loop
+        // before the break (looking up more-specifics mainly), which looks a
+        // bit repetitious, but again it's been done like that to avoid
+        // having to match over a SizedStrideNode again in the
+        // `post-processing` section.
+
+        for stride in self.get_stride_sizes() {
+            stride_end += stride;
+
+            let last_stride = search_pfx.get_len() < stride_end;
+
+            nibble_len = if last_stride {
+                stride + search_pfx.get_len() - stride_end
+            } else {
+                *stride
+            };
+
+            // Shift left and right to set the bits to zero that are not
+            // in the nibble we're handling here.
+            nibble = AddressFamily::get_nibble(
+                search_pfx.get_net(),
+                stride_end - stride,
+                nibble_len,
+            );
+
+            match node {
+                SizedStrideRef::Stride3(current_node) => {
+                    let search_fn = match options.match_type {
+                        MatchType::ExactMatch => {
+                            if options.include_less_specifics {
+                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
+                            } else {
+                                TreeBitMapNode::search_stride_for_exact_match_at
+                            }
+                        }
+                        MatchType::LongestMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                        MatchType::EmptyMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                    };
+
+                    // This whole match assumes that:
+                    // - if the first value in the return tuple of
+                    //   `search_fn` holds a value, then we need to continue
+                    //   searching by following the node contained in the
+                    //   value.
+                    // - The second value in the tuple holds the prefix that
+                    //   was found.
+                    // The less_specifics_vec is mutated by `search_fn` to
+                    // hold the prefixes found along the way, in the cases
+                    // where `include_less_specifics` was requested by the
+                    // user.
+                    match search_fn(
+                        current_node,
+                        search_pfx,
+                        nibble,
+                        nibble_len,
+                        stride_end - stride,
+                        &mut less_specifics_vec,
+                    ) {
+                        // This and the next match will handle all
+                        // intermediary nodes, but they might also handle
+                        // exit nodes.
+                        (Some(n), Some(pfx_idx)) => {
+                            match_prefix_idx = Some(pfx_idx);
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        (Some(n), None) => {
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        // This handles exact and longest matches: there are
+                        // no more children, but there is a prefix on this
+                        // node.
+                        (None, Some(pfx_idx)) => {
+                            if options.include_more_specifics {
+                                more_specifics_vec = self
+                                    .get_all_more_specifics_from_nibble(
+                                        current_node,
+                                        nibble,
+                                        nibble_len,
+                                        StrideNodeId::new_with_cleaned_id(
+                                            search_pfx.get_net(),
+                                            stride_end - stride,
+                                        ),
+                                    );
+                            }
+                            match_prefix_idx = Some(pfx_idx);
+                            break;
+                        }
+                        // This handles cases where there's no prefix (and no
+                        // child) for exact match or longest match, the empty
+                        // match - which doesn't care about actually finding
+                        // a prefix - just continues in search of
+                        // more-specifics.
+                        (None, None) => {
+                            match options.match_type {
+                                MatchType::EmptyMatch => {
+                                    // To make sure we don't process this
+                                    // match arm more then once, we return
+                                    // early here.
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+
+                                    match_prefix_idx = None;
+                                    break;
+                                }
+                                MatchType::LongestMatch => {}
+                                MatchType::ExactMatch => {
+                                    match_prefix_idx = None;
+                                }
+                            }
+                            break;
+                        }
+                    }
+                }
+                //---- From here only repetitions for all strides -----------
+                // For comments see the code above for the Stride3 arm.
+                SizedStrideRef::Stride4(current_node) => {
+                    let search_fn = match options.match_type {
+                        MatchType::ExactMatch => {
+                            if options.include_less_specifics {
+                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
+                            } else {
+                                TreeBitMapNode::search_stride_for_exact_match_at
+                            }
+                        }
+                        MatchType::LongestMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                        MatchType::EmptyMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                    };
+                    match search_fn(
+                        current_node,
+                        search_pfx,
+                        nibble,
+                        nibble_len,
+                        stride_end - stride,
+                        &mut less_specifics_vec,
+                    ) {
+                        (Some(n), Some(pfx_idx)) => {
+                            match_prefix_idx = Some(pfx_idx);
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        (Some(n), None) => {
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        (None, Some(pfx_idx)) => {
+                            if options.include_more_specifics {
+                                more_specifics_vec = self
+                                    .get_all_more_specifics_from_nibble(
+                                        current_node,
+                                        nibble,
+                                        nibble_len,
+                                        StrideNodeId::new_with_cleaned_id(
+                                            search_pfx.get_net(),
+                                            stride_end - stride,
+                                        ),
+                                    );
+                            }
+                            match_prefix_idx = Some(pfx_idx);
+                            break;
+                        }
+                        (None, None) => {
+                            match options.match_type {
+                                MatchType::EmptyMatch => {
+                                    // To make sure we don't process this
+                                    // match arm more then once, we return
+                                    // early here.
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+
+                                    match_prefix_idx = None;
+                                    break;
+                                }
+                                MatchType::LongestMatch => {}
+                                MatchType::ExactMatch => {
+                                    match_prefix_idx = None;
+                                }
+                            }
+                            break;
+                        }
+                    }
+                }
+                SizedStrideRef::Stride5(current_node) => {
+                    let search_fn = match options.match_type {
+                        MatchType::ExactMatch => {
+                            if options.include_less_specifics {
+                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
+                            } else {
+                                TreeBitMapNode::search_stride_for_exact_match_at
+                            }
+                        }
+                        MatchType::LongestMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                        MatchType::EmptyMatch => {
+                            TreeBitMapNode::search_stride_for_longest_match_at
+                        }
+                    };
+                    match search_fn(
+                        current_node,
+                        search_pfx,
+                        nibble,
+                        nibble_len,
+                        stride_end - stride,
+                        &mut less_specifics_vec,
+                    ) {
+                        (Some(n), Some(pfx_idx)) => {
+                            match_prefix_idx = Some(pfx_idx);
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        (Some(n), None) => {
+                            node = self.retrieve_node(n).unwrap();
+
+                            if last_stride {
+                                if options.include_more_specifics {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+                                }
+                                break;
+                            }
+                        }
+                        (None, Some(pfx_idx)) => {
+                            if options.include_more_specifics {
+                                more_specifics_vec = self
+                                    .get_all_more_specifics_from_nibble(
+                                        current_node,
+                                        nibble,
+                                        nibble_len,
+                                        StrideNodeId::new_with_cleaned_id(
+                                            search_pfx.get_net(),
+                                            stride_end - stride,
+                                        ),
+                                    );
+                            }
+                            match_prefix_idx = Some(pfx_idx);
+                            break;
+                        }
+                        (None, None) => {
+                            match options.match_type {
+                                MatchType::EmptyMatch => {
+                                    more_specifics_vec = self
+                                        .get_all_more_specifics_from_nibble(
+                                            current_node,
+                                            nibble,
+                                            nibble_len,
+                                            StrideNodeId::new_with_cleaned_id(
+                                                search_pfx.get_net(),
+                                                stride_end - stride,
+                                            ),
+                                        );
+
+                                    match_prefix_idx = None;
+                                    break;
+                                }
+                                MatchType::LongestMatch => {}
+                                MatchType::ExactMatch => {
+                                    match_prefix_idx = None;
+                                }
+                            }
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        //------------------ end of Stride branch arm repetition ------------
+
+        //------------------ post-processing --------------------------------
+
+        // If the above loop finishes (so not hitting a break) we have
+        // processed all strides and have found a child node and maybe a
+        // prefix. Now we will look up more-specifics for longest-matching
+        // prefixes that were found in the last stride only. Note that still
+        // any of the match_types (as specified by the user, not the return
+        // type) may end up here.
+
+        let match_type = MatchType::EmptyMatch;
+        if let Some(prefix) = match_prefix_idx {
+            if prefix.get_len() == search_pfx.get_len() {
+                MatchType::ExactMatch
+            } else {
+                MatchType::LongestMatch
+            }
+        } else {
+            MatchType::EmptyMatch
+        };
+
+        TreeQueryResult {
+            prefix: match_prefix_idx,
+            match_type,
+            less_specifics: if options.include_less_specifics {
+                less_specifics_vec
+            } else {
+                None
+            },
+            more_specifics: if options.include_more_specifics {
+                more_specifics_vec
+            } else {
+                None
+            },
+        }
+    }
+}
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 01bbc285..ba255515 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -284,7 +284,7 @@ impl<
         pfx: PrefixId<AF>,
         mui: u32,
         // update_path_selections: Option<M::TBI>,
-    ) -> Result<u32, PrefixStoreError> {
+    ) -> Result<(u32, bool), PrefixStoreError> {
         if pfx.get_len() == 0 {
             return self.update_default_route_prefix_meta(mui);
             // return Ok(res);
@@ -295,7 +295,7 @@ impl<
         let mut level: u8 = 0;
         let mut acc_retry_count = 0;
 
-        let a = loop {
+        let retry_and_exists = loop {
             let stride = self.get_stride_sizes()[level as usize];
             stride_end += stride;
             let nibble_len = if pfx.get_len() < stride_end {
@@ -367,7 +367,86 @@ impl<
             }
         };
 
-        Ok(a)
+        Ok(retry_and_exists)
+    }
+
+    pub fn prefix_exists(&self, prefix_id: PrefixId<AF>) -> bool {
+        // if prefix_id.get_len() == 0 {
+        //     return self.update_default_route_prefix_meta(mui);
+        // }
+
+        let mut stride_end: u8 = 0;
+        let mut cur_i = self.get_root_node_id();
+        let mut level: u8 = 0;
+
+        loop {
+            let stride = self.get_stride_sizes()[level as usize];
+            stride_end += stride;
+            let nibble_len = if prefix_id.get_len() < stride_end {
+                stride + prefix_id.get_len() - stride_end
+            } else {
+                stride
+            };
+
+            let nibble = AF::get_nibble(
+                prefix_id.get_net(),
+                stride_end - stride,
+                nibble_len,
+            );
+            let stride_start = stride_end - stride;
+
+            let node_result = match self.retrieve_node(cur_i) {
+                Some(node) => match node {
+                    SizedStrideRef::Stride3(n) => {
+                        let (child_node, found) = n.prefix_exists_in_stride(
+                            prefix_id,
+                            nibble,
+                            nibble_len,
+                            stride_start,
+                        );
+                        if found {
+                            return true;
+                        } else {
+                            child_node
+                        }
+                    }
+                    SizedStrideRef::Stride4(n) => {
+                        let (child_node, found) = n.prefix_exists_in_stride(
+                            prefix_id,
+                            nibble,
+                            nibble_len,
+                            stride_start,
+                        );
+                        if found {
+                            return true;
+                        } else {
+                            child_node
+                        }
+                    }
+                    SizedStrideRef::Stride5(n) => {
+                        let (child_node, found) = n.prefix_exists_in_stride(
+                            prefix_id,
+                            nibble,
+                            nibble_len,
+                            stride_start,
+                        );
+                        if found {
+                            return true;
+                        } else {
+                            child_node
+                        }
+                    }
+                },
+                None => {
+                    return false;
+                }
+            };
+
+            if let Some(next_id) = node_result {
+                cur_i = next_id;
+                level += 1;
+            }
+        }
     }
 
     pub(crate) fn upsert_prefix(
@@ -405,9 +484,6 @@ impl<
                 if mui_count.is_some() {
                     mui_is_new = false;
                     prefix_is_new = false;
-                } else {
-                    // No, we were the first, we created a new prefix
-                    self.counters.inc_prefixes_count(prefix.get_len());
                 }
 
                 (mui_count, retry_count)
@@ -482,7 +558,7 @@ impl<
     fn update_default_route_prefix_meta(
         &self,
         mui: u32,
-    ) -> Result<u32, PrefixStoreError> {
+    ) -> Result<(u32, bool), PrefixStoreError> {
         trace!("Updating the default route...");
 
         if let Some(root_node) =
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index f53bdc2e..ae5a87f5 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -3,11 +3,17 @@
 use std::marker::PhantomData;
 use std::path::Path;
 
+use inetnum::addr::Prefix;
 use lsm_tree::AbstractTree;
 
 use crate::local_array::types::{PrefixId, RouteStatus};
+use crate::prefix_record::PublicPrefixRecord;
+use crate::rib::query::TreeQueryResult;
 use crate::rib::Counters;
-use crate::{AddressFamily, MatchType, Meta, PublicRecord, QueryResult};
+use crate::{
+    AddressFamily, IncludeHistory, MatchOptions, Meta, PublicRecord,
+    QueryResult, RecordSet,
+};
 
 pub struct PersistTree<
     AF: AddressFamily,
@@ -44,8 +50,14 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     pub fn get_records_for_prefix<M: Meta>(
         &self,
         prefix: PrefixId<AF>,
+        mui: Option<u32>,
     ) -> Vec<PublicRecord<M>> {
-        let prefix_b = &prefix.as_bytes::<PREFIX_SIZE>();
+        let prefix_b = if let Some(mui) = mui {
+            &Self::prefix_mui_persistence_key(prefix, mui)
+        } else {
+            &prefix.as_bytes::<PREFIX_SIZE>()
+        };
+
         (*self.tree.prefix(prefix_b))
             .into_iter()
             .map(|kv| {
@@ -85,37 +97,184 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             .collect::<Vec<_>>()
     }
 
-    pub(crate) fn match_prefix<M: Meta>(
+    fn enrich_prefix_most_recent<M: Meta>(
         &self,
-        search_pfx: PrefixId<AF>,
+        prefix: Option<PrefixId<AF>>,
         mui: Option<u32>,
+    ) -> (Option<Prefix>, Vec<PublicRecord<M>>) {
+        match prefix {
+            Some(pfx) => {
+                let prefix_b = if let Some(mui) = mui {
+                    &Self::prefix_mui_persistence_key(pfx, mui)
+                } else {
+                    &pfx.as_bytes::<PREFIX_SIZE>()
+                };
+                (
+                    prefix.map(|p| p.into_pub()),
+                    (*self.tree.prefix(prefix_b))
+                        .into_iter()
+                        .last()
+                        .map(|kv| {
+                            let kv = kv.unwrap();
+                            let (_, mui, ltime, status) =
+                                Self::parse_key(kv.0.as_ref());
+                            vec![PublicRecord::new(
+                                mui,
+                                ltime,
+                                status.try_into().unwrap(),
+                                kv.1.as_ref().to_vec().into(),
+                            )]
+                        })
+                        .unwrap_or_default(),
+                )
+            }
+            None => (None, vec![]),
+        }
+    }
+
+    fn enrich_prefixes_most_recent<M: Meta>(
+        &self,
+        prefixes: Option<Vec<PrefixId<AF>>>,
+        mui: Option<u32>,
+    ) -> Option<RecordSet<M>> {
+        prefixes.map(|pfxs| {
+            pfxs.iter()
+                .flat_map(|pfx| {
+                    let prefix_b = if let Some(mui) = mui {
+                        &Self::prefix_mui_persistence_key(*pfx, mui)
+                    } else {
+                        &pfx.as_bytes::<PREFIX_SIZE>()
+                    };
+                    Some(PublicPrefixRecord::from((
+                        pfx.into_pub(),
+                        (*self.tree.prefix(prefix_b))
+                            .into_iter()
+                            .last()
+                            .map(|kv| {
+                                let kv = kv.unwrap();
+                                let (_, mui, ltime, status) =
+                                    Self::parse_key(kv.0.as_ref());
+                                vec![PublicRecord::new(
+                                    mui,
+                                    ltime,
+                                    status.try_into().unwrap(),
+                                    kv.1.as_ref().to_vec().into(),
+                                )]
+                            })
+                            .unwrap_or_default(),
+                    )))
+                })
+                .collect::<RecordSet<M>>()
+        })
+    }
+
+    fn enrich_prefix<M: Meta>(
+        &self,
+        prefix: Option<PrefixId<AF>>,
+        mui: Option<u32>,
+    ) -> (Option<Prefix>, Vec<PublicRecord<M>>) {
+        match prefix {
+            Some(pfx) => {
+                (Some(pfx.into_pub()), self.get_records_for_prefix(pfx, mui))
+            }
+            None => (None, vec![]),
+        }
+    }
+
+    fn enrich_prefixes<M: Meta>(
+        &self,
+        prefixes: Option<Vec<PrefixId<AF>>>,
+        mui: Option<u32>,
+    ) -> Option<RecordSet<M>> {
+        prefixes.map(|ls| {
+            ls.iter()
+                .flat_map(move |pfx| {
+                    Some(PublicPrefixRecord::from((
+                        *pfx,
+                        self.get_records_for_prefix(*pfx, mui),
+                    )))
+                })
+                .collect::<RecordSet<M>>()
+        })
+    }
+
+    fn sparse_record_set<M: Meta>(
+        &self,
+        prefixes: Option<Vec<PrefixId<AF>>>,
+    ) -> Option<RecordSet<M>> {
+        prefixes.map(|ls| {
+            ls.iter()
+                .flat_map(|pfx| {
+                    Some(PublicPrefixRecord::from((*pfx, vec![])))
+                })
+                .collect::<RecordSet<M>>()
+        })
+    }
+
+    pub(crate) fn match_prefix<M: Meta>(
+        &self,
+        search_pfxs: TreeQueryResult<AF>,
+        options: &MatchOptions,
     ) -> QueryResult<M> {
-        let key: Vec<u8> = if let Some(mui) = mui {
-            PersistTree::<AF,
-        PREFIX_SIZE, KEY_SIZE>::prefix_mui_persistence_key(search_pfx, mui)
-        } else {
-            search_pfx.as_bytes::<PREFIX_SIZE>().to_vec()
-        };
-        let recs = self.get_records_for_key(&key);
-
-        if !recs.is_empty() {
-            QueryResult {
-                prefix: Some(search_pfx.into_pub()),
-                match_type: MatchType::ExactMatch,
-                prefix_meta: recs
-                    .into_iter()
-                    .map(|(_, rec)| rec)
-                    .collect::<Vec<_>>(),
-                less_specifics: None,
-                more_specifics: None,
+        match options.include_history {
+            // All the records for all the prefixes
+            IncludeHistory::All => {
+                let (prefix, prefix_meta) =
+                    self.enrich_prefix(search_pfxs.prefix, options.mui);
+
+                QueryResult {
+                    prefix,
+                    prefix_meta,
+                    match_type: search_pfxs.match_type,
+                    less_specifics: self.enrich_prefixes(
+                        search_pfxs.less_specifics,
+                        options.mui,
+                    ),
+                    more_specifics: self.enrich_prefixes(
+                        search_pfxs.more_specifics,
+                        options.mui,
+                    ),
+                }
             }
-        } else {
-            QueryResult {
-                prefix: Some(search_pfx.into_pub()),
-                match_type: MatchType::EmptyMatch,
-                prefix_meta: vec![],
-                less_specifics: None,
-                more_specifics: None,
+            // Only the searc prefix itself has historical records attacched
+            // to it, other prefixes (less|more specifics), have no records
+            // attached. Not useful with the MemoryOnly strategy (historical
+            // records are neve kept in memory).
+            IncludeHistory::SearchPrefix => {
+                let (prefix, prefix_meta) =
+                    self.enrich_prefix(search_pfxs.prefix, options.mui);
+
+                QueryResult {
+                    prefix,
+                    prefix_meta,
+                    match_type: search_pfxs.match_type,
+                    less_specifics: self
+                        .sparse_record_set(search_pfxs.less_specifics),
+                    more_specifics: self
+                        .sparse_record_set(search_pfxs.more_specifics),
+                }
+            }
+            // Only the most recent record of the search prefix is returned
+            // with the prefixes. This is used for the PersistOnly strategy.
+            IncludeHistory::None => {
+                let (prefix, prefix_meta) = self.enrich_prefix_most_recent(
+                    search_pfxs.prefix,
+                    options.mui,
+                );
+
+                QueryResult {
+                    prefix,
+                    prefix_meta,
+                    match_type: search_pfxs.match_type,
+                    less_specifics: self.enrich_prefixes_most_recent(
+                        search_pfxs.less_specifics,
+                        options.mui,
+                    ),
+                    more_specifics: self.enrich_prefixes_most_recent(
+                        search_pfxs.more_specifics,
+                        options.mui,
+                    ),
+                }
             }
         }
     }
@@ -187,8 +346,8 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     pub fn prefix_mui_persistence_key(
         prefix_id: PrefixId<AF>,
         mui: u32,
-    ) -> Vec<u8> {
-        let mut key = vec![];
+    ) -> [u8; PREFIX_SIZE] {
+        let mut key = [0; PREFIX_SIZE];
         // prefix 5 or 17 bytes
         *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
 
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index dd2ef915..d221c1cf 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -146,7 +146,12 @@ where
         match self.config.persist_strategy() {
             PersistStrategy::PersistOnly => {
                 if let Some(persist_tree) = &self.persist_tree {
-                    persist_tree.match_prefix(search_pfx, options.mui)
+                    persist_tree.match_prefix(
+                        self.in_memory_tree.match_prefix_by_tree_traversal(
+                            search_pfx, options,
+                        ),
+                        options,
+                    )
                 } else {
                     QueryResult::empty()
                 }
@@ -163,6 +168,7 @@ where
     ) -> QueryResult<M> {
         // `non_recursive_retrieve_prefix` returns an exact match only, so no
         // longest matching prefix!
+        let withdrawn_muis_bmin = self.withdrawn_muis_bmin(guard);
         let mut stored_prefix = self
             .in_memory_tree
             .non_recursive_retrieve_prefix(search_pfx)
@@ -178,7 +184,7 @@ where
                         pfx.record_map
                             .get_filtered_records(
                                 options.mui,
-                                self.withdrawn_muis_bmin(guard),
+                                withdrawn_muis_bmin,
                             )
                             .into_iter()
                             .collect()
@@ -187,7 +193,7 @@ where
                         // the local statuses of the records with muis
                         // that appear in the specified bitmap index.
                         pfx.record_map.as_records_with_rewritten_status(
-                            self.withdrawn_muis_bmin(guard),
+                            withdrawn_muis_bmin,
                             RouteStatus::Withdrawn,
                         )
                     },
@@ -320,3 +326,10 @@ where
             })
     }
 }
+
+pub struct TreeQueryResult<AF: AddressFamily> {
+    pub match_type: MatchType,
+    pub prefix: Option<PrefixId<AF>>,
+    pub less_specifics: Option<Vec<PrefixId<AF>>>,
+    pub more_specifics: Option<Vec<PrefixId<AF>>>,
+}
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 8a91c430..9cedebee 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -16,9 +16,7 @@ use crate::{
     local_array::errors::PrefixStoreError, prefix_record::PublicRecord,
 };
 
-use crate::local_array::in_memory::atomic_types::{
-    NodeBuckets, StoredPrefix,
-};
+use crate::local_array::in_memory::atomic_types::NodeBuckets;
 use crate::local_array::in_memory::atomic_types::{
     PersistStatus, PrefixBuckets,
 };
@@ -28,14 +26,16 @@ use crate::local_array::in_memory::atomic_types::{
 pub use crate::local_array::iterators;
 pub use crate::local_array::query;
 
-use crate::{IPv4, IPv6, MatchType, Meta, QueryResult};
+use crate::{IPv4, IPv6, Meta};
 
 use crate::AddressFamily;
 
 //------------ StoreConfig ---------------------------------------------------
 
+/// Some description
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum PersistStrategy {
+    /// The current records are kept in history
     WriteAhead,
     PersistHistory,
     MemoryOnly,
@@ -216,7 +216,7 @@ pub struct Rib<
     const KEY_SIZE: usize,
 > {
     pub config: StoreConfig,
-    pub(in crate::local_array) in_memory_tree: TreeBitMap<AF, M, NB, PB>,
+    pub in_memory_tree: TreeBitMap<AF, M, NB, PB>,
     #[cfg(feature = "persist")]
     pub(in crate::local_array) persist_tree:
         Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
@@ -289,7 +289,7 @@ impl<
                     if c.mui_new {
                         self.counters.inc_routes_count();
                     }
-                    c.cas_count += c1 as usize;
+                    c.cas_count += c1.0 as usize;
                     c
                 })
             })
@@ -353,8 +353,16 @@ impl<
                 .map(|(report, _)| report),
             PersistStrategy::PersistOnly => {
                 if let Some(persist_tree) = &self.persist_tree {
+                    let (retry_count, exists) = self
+                        .in_memory_tree
+                        .set_prefix_exists(prefix, record.multi_uniq_id)?;
                     persist_tree.persist_record(prefix, &record);
-                    Err(PrefixStoreError::StatusUnknown)
+                    Ok(UpsertReport {
+                        cas_count: retry_count as usize,
+                        prefix_new: exists,
+                        mui_new: true,
+                        mui_count: 0,
+                    })
                 } else {
                     Err(PrefixStoreError::PersistFailed)
                 }
@@ -549,9 +557,10 @@ impl<
     pub fn get_records_for_prefix(
         &self,
         prefix: &Prefix,
+        mui: Option<u32>,
     ) -> Vec<PublicRecord<M>> {
         if let Some(p) = &self.persist_tree {
-            p.get_records_for_prefix(PrefixId::from(*prefix))
+            p.get_records_for_prefix(PrefixId::from(*prefix), mui)
         } else {
             vec![]
         }
diff --git a/src/local_vec/tests/full_table_single.rs b/src/local_vec/tests/full_table_single.rs
index 59664192..7afe94f3 100644
--- a/src/local_vec/tests/full_table_single.rs
+++ b/src/local_vec/tests/full_table_single.rs
@@ -81,6 +81,7 @@ mod full_table {
                         include_less_specifics: false,
                         include_more_specifics: false,
                         mui: None,
+                        include_history: IncludeHistory::None,
                     },
                 );
 
@@ -118,6 +119,7 @@ mod full_table {
                                 include_less_specifics: false,
                                 include_more_specifics: false,
                                 mui: None,
+                                include_history: IncludeHistory::None,
                             },
                         );
                         if let Some(_pfx) = res.prefix {
diff --git a/src/local_vec/tests/more_specifics_single.rs b/src/local_vec/tests/more_specifics_single.rs
index 4f61faef..0f473db5 100644
--- a/src/local_vec/tests/more_specifics_single.rs
+++ b/src/local_vec/tests/more_specifics_single.rs
@@ -57,6 +57,7 @@ mod tests {
                     include_less_specifics: false,
                     include_more_specifics: true,
                     mui: None,
+                    include_history: IncludeHistory::None,
                 },
             );
             println!("em/m-s: {:#?}", found_result);
@@ -134,6 +135,7 @@ mod tests {
                     include_less_specifics: false,
                     include_more_specifics: true,
                     mui: None,
+                    include_history: IncludeHistory::None,
                 },
             );
             println!("em/m-s: {}", found_result);
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index c67e94d2..df6c1ede 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -3,7 +3,7 @@ pub use uuid::Uuid;
 
 pub use crate::prefix_record::{Meta, PublicPrefixRecord as PrefixRecord};
 pub use crate::stride::{Stride3, Stride4, Stride5};
-pub use crate::{MatchOptions, MatchType, QueryResult};
+pub use crate::{IncludeHistory, MatchOptions, MatchType, QueryResult};
 pub use inetnum::addr::Prefix;
 
 pub mod multi {
diff --git a/src/rotonda_store.rs b/src/rotonda_store.rs
index 3247078a..098e7082 100644
--- a/src/rotonda_store.rs
+++ b/src/rotonda_store.rs
@@ -85,6 +85,13 @@ pub struct MatchOptions {
     /// Whether to return records for a specific multi_uniq_id, None indicates
     /// all records.
     pub mui: Option<u32>,
+    /// Whether to include historical records, i.e. records that have been
+    /// superceded by updates. `SearchPrefix` means only historical records
+    /// for the search prefix will be included (if present), `All` means
+    /// all retrieved prefixes, i.e. next to the search prefix, also the
+    /// historical records for less and more specific prefixes will be
+    /// included.
+    pub include_history: IncludeHistory,
 }
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
@@ -110,6 +117,13 @@ impl std::fmt::Display for MatchType {
     }
 }
 
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub enum IncludeHistory {
+    None,
+    SearchPrefix,
+    All,
+}
+
 //------------ PrefixRecordIter ---------------------------------------------
 
 // Converts from the InternalPrefixRecord to the (public) PrefixRecord
diff --git a/tests/best-path.rs b/tests/best-path.rs
index 1f4f986a..85b091df 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -5,6 +5,7 @@ use rotonda_store::prelude::multi::Record;
 use rotonda_store::prelude::multi::RouteStatus;
 use rotonda_store::rib::PersistStrategy;
 use rotonda_store::rib::StoreConfig;
+use rotonda_store::IncludeHistory;
 use rotonda_store::MatchOptions;
 use rotonda_store::Meta;
 use rotonda_store::MultiThreadedStore;
@@ -177,6 +178,7 @@ fn test_best_path_1() -> Result<(), Box<dyn std::error::Error>> {
             include_less_specifics: false,
             include_more_specifics: false,
             mui: None,
+            include_history: IncludeHistory::None,
         },
         &rotonda_store::epoch::pin(),
     );
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index da7d821f..8d760745 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -2,8 +2,8 @@ use std::{str::FromStr, sync::atomic::Ordering};
 
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
-    prelude::multi::RouteStatus, test_types::BeBytesAsn, MatchOptions,
-    MultiThreadedStore, PublicRecord as Record,
+    prelude::multi::RouteStatus, test_types::BeBytesAsn, IncludeHistory,
+    MatchOptions, MultiThreadedStore, PublicRecord as Record,
 };
 
 mod common {
@@ -331,6 +331,7 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
         include_less_specifics: false,
         include_more_specifics: false,
         mui: None,
+        include_history: IncludeHistory::None,
     };
 
     for pfx in pfx_vec_2 {
@@ -597,6 +598,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
         include_less_specifics: false,
         include_more_specifics: false,
         mui: None,
+        include_history: IncludeHistory::None,
     };
 
     for pfx in wd_pfxs {
@@ -619,6 +621,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
         include_less_specifics: false,
         include_more_specifics: true,
         mui: None,
+        include_history: IncludeHistory::None,
     };
 
     let pfx = Prefix::from_str("0.0.0.0/0").unwrap();
diff --git a/tests/full-table.rs b/tests/full-table.rs
index 903fb2f2..e1d7c297 100644
--- a/tests/full-table.rs
+++ b/tests/full-table.rs
@@ -132,6 +132,7 @@ mod tests {
                         include_less_specifics: false,
                         include_more_specifics: false,
                         mui: None,
+                        include_history: IncludeHistory::None,
                     },
                     guard,
                 );
@@ -170,6 +171,7 @@ mod tests {
                                 include_less_specifics: false,
                                 include_more_specifics: false,
                                 mui: None,
+                                include_history: IncludeHistory::None,
                             },
                             guard,
                         );
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index df9477db..406da8ee 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -65,6 +65,7 @@ mod tests {
                     include_less_specifics: false,
                     include_more_specifics: true,
                     mui: None,
+                    include_history: IncludeHistory::None,
                 },
                 guard,
             );
@@ -147,6 +148,7 @@ mod tests {
                     include_less_specifics: false,
                     include_more_specifics: true,
                     mui: None,
+                    include_history: IncludeHistory::None,
                 },
                 guard,
             );
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index b4c966fa..2b677ab3 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -190,6 +190,7 @@ mod tests {
                     include_less_specifics: false,
                     include_more_specifics: true,
                     mui: None,
+                    include_history: IncludeHistory::None,
                 },
                 guard,
             );
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 1d300e37..6b45222d 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -48,6 +48,7 @@ mod tests {
                 include_less_specifics: true,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -81,6 +82,7 @@ mod tests {
                 include_less_specifics: true,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -348,6 +350,7 @@ mod tests {
                     include_less_specifics: false,
                     include_more_specifics: false,
                     mui: None,
+                    include_history: IncludeHistory::None,
                 },
                 guard,
             );
@@ -363,6 +366,7 @@ mod tests {
                 include_less_specifics: true,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -443,6 +447,7 @@ mod tests {
                             include_less_specifics: false,
                             include_more_specifics: false,
                             mui: None,
+                            include_history: IncludeHistory::None,
                         },
                         guard,
                     );
@@ -511,6 +516,7 @@ mod tests {
                                 include_less_specifics: false,
                                 include_more_specifics: false,
                                 mui: Some(mui),
+                                include_history: IncludeHistory::None,
                             },
                             guard,
                         );
@@ -557,6 +563,7 @@ mod tests {
                 include_less_specifics: false,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -578,6 +585,7 @@ mod tests {
                 include_less_specifics: false,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -656,6 +664,7 @@ mod tests {
                 include_less_specifics: false,
                 include_more_specifics: true,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -694,6 +703,7 @@ mod tests {
                 include_less_specifics: false,
                 include_more_specifics: true,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -734,6 +744,7 @@ mod tests {
                 include_less_specifics: false,
                 include_more_specifics: true,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -784,6 +795,7 @@ mod tests {
                 include_less_specifics: false,
                 include_more_specifics: true,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -833,6 +845,7 @@ mod tests {
                 include_less_specifics: true,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -857,6 +870,7 @@ mod tests {
                 include_less_specifics: true,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
diff --git a/tests/treebitmap_v6.rs b/tests/treebitmap_v6.rs
index 1cca898d..3f68a3c2 100644
--- a/tests/treebitmap_v6.rs
+++ b/tests/treebitmap_v6.rs
@@ -45,6 +45,7 @@ mod tests {
                 include_less_specifics: true,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -85,6 +86,7 @@ mod tests {
                 include_less_specifics: true,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -120,6 +122,7 @@ mod tests {
                 include_less_specifics: true,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -296,6 +299,7 @@ mod tests {
                     include_less_specifics: false,
                     include_more_specifics: false,
                     mui: None,
+                    include_history: IncludeHistory::None,
                 },
                 guard,
             );
@@ -568,6 +572,7 @@ mod tests {
                     include_less_specifics: false,
                     include_more_specifics: false,
                     mui: None,
+                    include_history: IncludeHistory::None,
                 },
                 guard,
             );
@@ -586,6 +591,7 @@ mod tests {
                 include_less_specifics: true,
                 include_more_specifics: false,
                 mui: None,
+                include_history: IncludeHistory::None,
             },
             guard,
         );
@@ -671,6 +677,7 @@ mod tests {
                             include_less_specifics: false,
                             include_more_specifics: false,
                             mui: None,
+                            include_history: IncludeHistory::None,
                         },
                         guard,
                     );

From d87af6552f20be84d5cbe0943fdbb4e7e398c568 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 24 Dec 2024 17:06:05 +0100
Subject: [PATCH 035/147] move withdrawn_mui_bmin once more to TreeBitMap

---
 proc_macros/src/lib.rs              |  37 +++++--
 src/local_array/in_memory/query.rs  | 119 ++++++++++++++++++++++-
 src/local_array/in_memory/tree.rs   |  20 +++-
 src/local_array/iterators.rs        |  48 ++++-----
 src/local_array/persist/lsm_tree.rs |   2 +-
 src/local_array/query.rs            | 145 +++-------------------------
 src/local_array/rib/rib.rs          |  35 +++----
 7 files changed, 213 insertions(+), 193 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 13c4a080..077d9add 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -957,7 +957,10 @@ pub fn create_store(
                     let (left, right) = match search_pfx.addr() {
                         std::net::IpAddr::V4(addr) => {
                             (
-                                Some(self.v4.less_specific_prefix_iter(
+                                Some(self
+                                    .v4
+                                    .in_memory_tree
+                                    .less_specific_prefix_iter(
                                         PrefixId::<IPv4>::new(
                                             addr.into(),
                                             search_pfx.len(),
@@ -974,7 +977,10 @@ pub fn create_store(
                         std::net::IpAddr::V6(addr) => {
                             (
                                 None,
-                                Some(self.v6.less_specific_prefix_iter(
+                                Some(self
+                                    .v6
+                                    .in_memory_tree
+                                    .less_specific_prefix_iter(
                                         PrefixId::<IPv6>::new(
                                             addr.into(),
                                             search_pfx.len(),
@@ -1050,7 +1056,10 @@ pub fn create_store(
                             |m| { self.v6.mui_is_withdrawn(m, guard) }) {
                                 (None, None)
                         } else {
-                            (Some(self.v4.more_specific_prefix_iter_from(
+                            (Some(self
+                                .v4
+                                .in_memory_tree
+                                .more_specific_prefix_iter_from(
                                     PrefixId::<IPv4>::new(
                                         addr.into(),
                                         search_pfx.len(),
@@ -1071,7 +1080,10 @@ pub fn create_store(
                         } else {
                             (
                                 None,
-                                Some(self.v6.more_specific_prefix_iter_from(
+                                Some(self
+                                    .v6
+                                    .in_memory_tree
+                                    .more_specific_prefix_iter_from(
                                         PrefixId::<IPv6>::new(
                                             addr.into(),
                                             search_pfx.len(),
@@ -1100,7 +1112,10 @@ pub fn create_store(
                     None
                 } else {
                     Some(
-                        self.v4.more_specific_prefix_iter_from(
+                        self
+                            .v4
+                            .in_memory_tree
+                            .more_specific_prefix_iter_from(
                                 PrefixId::<IPv4>::new(
                                     0,
                                     0,
@@ -1125,7 +1140,9 @@ pub fn create_store(
                     None
                 } else {
                     Some(
-                        self.v6.more_specific_prefix_iter_from(
+                        self.v6
+                            .in_memory_tree
+                            .more_specific_prefix_iter_from(
                                 PrefixId::<IPv6>::new(
                                     0,
                                     0,
@@ -1221,10 +1238,10 @@ pub fn create_store(
             pub fn prefixes_iter(
                 &'a self,
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v4.prefixes_iter()
+                self.v4.in_memory_tree.prefixes_iter()
                     .map(|p| PrefixRecord::from(p))
                     .chain(
-                        self.v6.prefixes_iter()
+                        self.v6.in_memory_tree.prefixes_iter()
                         .map(|p| PrefixRecord::from(p))
                     )
             }
@@ -1272,7 +1289,7 @@ pub fn create_store(
             pub fn prefixes_iter_v4(
                 &'a self,
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v4.prefixes_iter()
+                self.v4.in_memory_tree.prefixes_iter()
                     .map(|p| PrefixRecord::from(p))
             }
 
@@ -1319,7 +1336,7 @@ pub fn create_store(
             pub fn prefixes_iter_v6(
                 &'a self,
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v6.prefixes_iter()
+                self.v6.in_memory_tree.prefixes_iter()
                     .map(|p| PrefixRecord::from(p))
             }
 
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index 1f8734f3..23deb2f6 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -1,9 +1,11 @@
+use crossbeam_epoch::Guard;
 use log::trace;
 
 use crate::af::AddressFamily;
+use crate::prelude::multi::RouteStatus;
 use crate::rib::query::TreeQueryResult;
 
-use crate::Meta;
+use crate::{Meta, QueryResult};
 
 use crate::local_array::in_memory::node::{SizedStrideRef, TreeBitMapNode};
 use crate::{MatchOptions, MatchType};
@@ -21,6 +23,121 @@ where
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
 {
+    pub(crate) fn match_prefix<'a>(
+        &'a self,
+        search_pfx: PrefixId<AF>,
+        options: &MatchOptions,
+        guard: &'a Guard,
+    ) -> QueryResult<M> {
+        // `non_recursive_retrieve_prefix` returns an exact match only, so no
+        // longest matching prefix!
+        let withdrawn_muis_bmin = self.withdrawn_muis_bmin(guard);
+        let mut stored_prefix =
+            self.non_recursive_retrieve_prefix(search_pfx).0.map(|pfx| {
+                (
+                    pfx.prefix,
+                    if !options.include_withdrawn {
+                        // Filter out all the withdrawn records, both
+                        // with globally withdrawn muis, and with local
+                        // statuses
+                        // set to Withdrawn.
+                        pfx.record_map
+                            .get_filtered_records(
+                                options.mui,
+                                withdrawn_muis_bmin,
+                            )
+                            .into_iter()
+                            .collect()
+                    } else {
+                        // Do no filter out any records, but do rewrite
+                        // the local statuses of the records with muis
+                        // that appear in the specified bitmap index.
+                        pfx.record_map.as_records_with_rewritten_status(
+                            withdrawn_muis_bmin,
+                            RouteStatus::Withdrawn,
+                        )
+                    },
+                )
+            });
+
+        // Check if we have an actual exact match, if not then fetch the
+        // first lesser-specific with the greatest length, that's the Longest
+        // matching prefix, but only if the user requested a longest match or
+        // empty match.
+        let match_type = match (&options.match_type, &stored_prefix) {
+            // we found an exact match, we don't need to do anything.
+            (_, Some((_pfx, meta))) if !meta.is_empty() => {
+                MatchType::ExactMatch
+            }
+            // we didn't find an exact match, but the user requested it
+            // so we need to find the longest matching prefix.
+            (MatchType::LongestMatch | MatchType::EmptyMatch, _) => {
+                stored_prefix = self
+                    .less_specific_prefix_iter(
+                        search_pfx,
+                        options.mui,
+                        options.include_withdrawn,
+                        guard,
+                    )
+                    .max_by(|p0, p1| p0.0.get_len().cmp(&p1.0.get_len()));
+                if stored_prefix.is_some() {
+                    MatchType::LongestMatch
+                } else {
+                    MatchType::EmptyMatch
+                }
+            }
+            // We got an empty match, but the user requested an exact match,
+            // even so, we're going to look for more and/or less specifics if
+            // the user asked for it.
+            (MatchType::ExactMatch, _) => MatchType::EmptyMatch,
+        };
+
+        QueryResult {
+            prefix: stored_prefix.as_ref().map(|p| p.0.into_pub()),
+            prefix_meta: stored_prefix
+                .as_ref()
+                .map(|pfx| pfx.1.clone())
+                .unwrap_or_default(),
+            less_specifics: if options.include_less_specifics {
+                Some(
+                    self.less_specific_prefix_iter(
+                        if let Some(ref pfx) = stored_prefix {
+                            pfx.0
+                        } else {
+                            search_pfx
+                        },
+                        options.mui,
+                        options.include_withdrawn,
+                        guard,
+                    )
+                    .collect(),
+                )
+            } else {
+                None
+            },
+            more_specifics: if options.include_more_specifics {
+                Some(
+                    self.more_specific_prefix_iter_from(
+                        if let Some(pfx) = stored_prefix {
+                            pfx.0
+                        } else {
+                            search_pfx
+                        },
+                        options.mui,
+                        options.include_withdrawn,
+                        guard,
+                    )
+                    .collect(),
+                )
+                // The user requested more specifics, but there aren't any, so
+                // we need to return an empty vec, not a None.
+            } else {
+                None
+            },
+            match_type,
+        }
+    }
+
     // This function assembles all entries in the `pfx_vec` of all child nodes
     // of the `start_node` into one vec, starting from itself and then
     // recursively assembling adding all `pfx_vec`s of its children.
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index ba255515..69e84395 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -187,11 +187,14 @@ use crate::local_array::bit_span::BitSpan;
 use crate::local_array::in_memory::atomic_types::StoredNode;
 use crate::prefix_record::Meta;
 use crate::prelude::multi::{NodeSet, PrefixId};
-use crossbeam_epoch::Guard;
+use crossbeam_epoch::{Atomic, Guard};
 use crossbeam_utils::Backoff;
 use log::{debug, error, log_enabled, trace};
+use roaring::RoaringBitmap;
 
-use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
+use std::sync::atomic::{
+    AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering,
+};
 use std::{fmt::Debug, marker::PhantomData};
 
 use super::atomic_types::{
@@ -225,6 +228,7 @@ pub struct TreeBitMap<
 > {
     pub(crate) node_buckets: NB,
     pub(crate) prefix_buckets: PB,
+    pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     counters: Counters,
     _af: PhantomData<AF>,
     _m: PhantomData<M>,
@@ -241,6 +245,7 @@ impl<
         let tree_bitmap = Self {
             node_buckets: NodeBuckets::init(),
             prefix_buckets: PB::init(),
+            withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
             _af: PhantomData,
             _m: PhantomData,
@@ -583,6 +588,17 @@ impl<
         }
     }
 
+    pub(crate) fn withdrawn_muis_bmin<'a>(
+        &'a self,
+        guard: &'a Guard,
+    ) -> &'a RoaringBitmap {
+        unsafe {
+            self.withdrawn_muis_bmin
+                .load(Ordering::Acquire, guard)
+                .deref()
+        }
+    }
+
     // Create a new node in the store with payload `next_node`.
     //
     // Next node will be ignored if a node with the same `id` already exists,
diff --git a/src/local_array/iterators.rs b/src/local_array/iterators.rs
index 6ce002a0..034b7490 100644
--- a/src/local_array/iterators.rs
+++ b/src/local_array/iterators.rs
@@ -21,7 +21,7 @@
 
 use super::in_memory::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
 use super::in_memory::node::{SizedStrideRef, StrideNodeId};
-use super::in_memory::tree::{Stride3, Stride4, Stride5};
+use super::in_memory::tree::{Stride3, Stride4, Stride5, TreeBitMap};
 use super::types::PrefixId;
 use crate::local_array::types::RouteStatus;
 use crate::prefix_record::PublicRecord;
@@ -304,10 +304,10 @@ pub(crate) struct MoreSpecificPrefixIter<
     M: Meta,
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
-    const PREFIX_SIZE: usize,
-    const KEY_SIZE: usize,
+    // const PREFIX_SIZE: usize,
+    // const KEY_SIZE: usize,
 > {
-    store: &'a Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
+    store: &'a TreeBitMap<AF, M, NB, PB>,
     cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
     cur_pfx_iter: SizedPrefixIter<AF>,
     start_bit_span: BitSpan,
@@ -328,10 +328,9 @@ impl<
         M: Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > Iterator
-    for MoreSpecificPrefixIter<'a, AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+        // const PREFIX_SIZE: usize,
+        // const KEY_SIZE: usize,
+    > Iterator for MoreSpecificPrefixIter<'a, AF, M, NB, PB>
 {
     type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
 
@@ -352,7 +351,6 @@ impl<
                 if let Some(mui) = self.mui {
                     if let Some(p) = self
                         .store
-                        .in_memory_tree
                         .non_recursive_retrieve_prefix(
                             next_pfx.unwrap_or_else(|| {
                                 panic!(
@@ -389,7 +387,6 @@ impl<
                 } else {
                     return self
                         .store
-                        .in_memory_tree
                         .non_recursive_retrieve_prefix(
                             next_pfx.unwrap_or_else(|| {
                                 panic!(
@@ -448,9 +445,9 @@ impl<
 
             if let Some(next_ptr) = next_ptr {
                 let node = if self.mui.is_none() {
-                    self.store.in_memory_tree.retrieve_node(next_ptr)
+                    self.store.retrieve_node(next_ptr)
                 } else {
-                    self.store.in_memory_tree.retrieve_node_for_mui(
+                    self.store.retrieve_node_for_mui(
                         next_ptr,
                         self.mui.unwrap(),
                         // self.guard,
@@ -732,9 +729,9 @@ impl<
         M: crate::prefix_record::Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+        // const PREFIX_SIZE: usize,
+        // const KEY_SIZE: usize,
+    > TreeBitMap<AF, M, NB, PB>
 {
     // Iterator over all more-specific prefixes, starting from the given
     // prefix at the given level and cursor.
@@ -753,7 +750,7 @@ impl<
         } else {
             // calculate the node start_prefix_id lives in.
             let (start_node_id, start_bit_span) =
-                self.in_memory_tree.get_node_id_for_prefix(&start_prefix_id);
+                self.get_node_id_for_prefix(&start_prefix_id);
             trace!("start node {}", start_node_id);
 
             trace!(
@@ -776,10 +773,9 @@ impl<
             let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
 
             let node = if let Some(mui) = mui {
-                self.in_memory_tree
-                    .retrieve_node_for_mui(start_node_id, mui)
+                self.retrieve_node_for_mui(start_node_id, mui)
             } else {
-                self.in_memory_tree.retrieve_node(start_node_id)
+                self.retrieve_node(start_node_id)
             };
 
             if let Some(node) = node {
@@ -873,14 +869,11 @@ impl<
             None
         } else {
             let cur_len = start_prefix_id.get_len() - 1;
-            let cur_bucket = self
-                .in_memory_tree
-                .prefix_buckets
-                .get_root_prefix_set(cur_len);
+            let cur_bucket = self.prefix_buckets.get_root_prefix_set(cur_len);
             let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
 
             Some(LessSpecificPrefixIter {
-                prefixes: &self.in_memory_tree.prefix_buckets,
+                prefixes: &self.prefix_buckets,
                 cur_len,
                 cur_bucket,
                 cur_level: 0,
@@ -899,11 +892,8 @@ impl<
         &'a self,
     ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
         PrefixIter {
-            prefixes: &self.in_memory_tree.prefix_buckets,
-            cur_bucket: self
-                .in_memory_tree
-                .prefix_buckets
-                .get_root_prefix_set(0),
+            prefixes: &self.prefix_buckets,
+            cur_bucket: self.prefix_buckets.get_root_prefix_set(0),
             cur_len: 0,
             cur_level: 0,
             cursor: 0,
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index ae5a87f5..6fc3128c 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -236,7 +236,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     ),
                 }
             }
-            // Only the searc prefix itself has historical records attacched
+            // Only the search prefix itself has historical records attacched
             // to it, other prefixes (less|more specifics), have no records
             // attached. Not useful with the MemoryOnly strategy (historical
             // records are neve kept in memory).
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index d221c1cf..529fef89 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -14,7 +14,7 @@ use crate::{Meta, QueryResult};
 use crate::{MatchOptions, MatchType};
 
 use super::errors::PrefixStoreError;
-use super::types::{PrefixId, RouteStatus};
+use super::types::PrefixId;
 
 //------------ Prefix Matching ----------------------------------------------
 
@@ -36,12 +36,13 @@ where
         let result =
             self.in_memory_tree.non_recursive_retrieve_prefix(prefix_id);
         let prefix = result.0;
-        let more_specifics_vec = self.more_specific_prefix_iter_from(
-            prefix_id,
-            mui,
-            include_withdrawn,
-            guard,
-        );
+        let more_specifics_vec =
+            self.in_memory_tree.more_specific_prefix_iter_from(
+                prefix_id,
+                mui,
+                include_withdrawn,
+                guard,
+            );
 
         QueryResult {
             prefix: if let Some(pfx) = prefix {
@@ -57,7 +58,7 @@ where
                 .map(|r| {
                     r.record_map.get_filtered_records(
                         mui,
-                        self.withdrawn_muis_bmin(guard),
+                        self.in_memory_tree.withdrawn_muis_bmin(guard),
                     )
                 })
                 .unwrap_or_default(),
@@ -80,7 +81,7 @@ where
         let prefix = result.0;
         let less_specifics_vec = result.1.map(
             |(prefix_id, _level, _cur_set, _parents, _index)| {
-                self.less_specific_prefix_iter(
+                self.in_memory_tree.less_specific_prefix_iter(
                     prefix_id,
                     mui,
                     include_withdrawn,
@@ -103,7 +104,7 @@ where
                 .map(|r| {
                     r.record_map.get_filtered_records(
                         mui,
-                        self.withdrawn_muis_bmin(guard),
+                        self.in_memory_tree.withdrawn_muis_bmin(guard),
                     )
                 })
                 .unwrap_or_default(),
@@ -123,12 +124,12 @@ where
         impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a,
         PrefixStoreError,
     > {
-        let bmin = self.withdrawn_muis_bmin(guard);
+        let bmin = self.in_memory_tree.withdrawn_muis_bmin(guard);
 
         if mui.is_some() && bmin.contains(mui.unwrap()) {
             Err(PrefixStoreError::PrefixNotFound)
         } else {
-            Ok(self.more_specific_prefix_iter_from(
+            Ok(self.in_memory_tree.more_specific_prefix_iter_from(
                 prefix_id,
                 mui,
                 include_withdrawn,
@@ -156,125 +157,7 @@ where
                     QueryResult::empty()
                 }
             }
-            _ => self.match_prefix_in_memory(search_pfx, options, guard),
-        }
-    }
-
-    fn match_prefix_in_memory(
-        &'a self,
-        search_pfx: PrefixId<AF>,
-        options: &MatchOptions,
-        guard: &'a Guard,
-    ) -> QueryResult<M> {
-        // `non_recursive_retrieve_prefix` returns an exact match only, so no
-        // longest matching prefix!
-        let withdrawn_muis_bmin = self.withdrawn_muis_bmin(guard);
-        let mut stored_prefix = self
-            .in_memory_tree
-            .non_recursive_retrieve_prefix(search_pfx)
-            .0
-            .map(|pfx| {
-                (
-                    pfx.prefix,
-                    if !options.include_withdrawn {
-                        // Filter out all the withdrawn records, both
-                        // with globally withdrawn muis, and with local
-                        // statuses
-                        // set to Withdrawn.
-                        pfx.record_map
-                            .get_filtered_records(
-                                options.mui,
-                                withdrawn_muis_bmin,
-                            )
-                            .into_iter()
-                            .collect()
-                    } else {
-                        // Do no filter out any records, but do rewrite
-                        // the local statuses of the records with muis
-                        // that appear in the specified bitmap index.
-                        pfx.record_map.as_records_with_rewritten_status(
-                            withdrawn_muis_bmin,
-                            RouteStatus::Withdrawn,
-                        )
-                    },
-                )
-            });
-
-        // Check if we have an actual exact match, if not then fetch the
-        // first lesser-specific with the greatest length, that's the Longest
-        // matching prefix, but only if the user requested a longest match or
-        // empty match.
-        let match_type = match (&options.match_type, &stored_prefix) {
-            // we found an exact match, we don't need to do anything.
-            (_, Some((_pfx, meta))) if !meta.is_empty() => {
-                MatchType::ExactMatch
-            }
-            // we didn't find an exact match, but the user requested it
-            // so we need to find the longest matching prefix.
-            (MatchType::LongestMatch | MatchType::EmptyMatch, _) => {
-                stored_prefix = self
-                    .less_specific_prefix_iter(
-                        search_pfx,
-                        options.mui,
-                        options.include_withdrawn,
-                        guard,
-                    )
-                    .max_by(|p0, p1| p0.0.get_len().cmp(&p1.0.get_len()));
-                if stored_prefix.is_some() {
-                    MatchType::LongestMatch
-                } else {
-                    MatchType::EmptyMatch
-                }
-            }
-            // We got an empty match, but the user requested an exact match,
-            // even so, we're going to look for more and/or less specifics if
-            // the user asked for it.
-            (MatchType::ExactMatch, _) => MatchType::EmptyMatch,
-        };
-
-        QueryResult {
-            prefix: stored_prefix.as_ref().map(|p| p.0.into_pub()),
-            prefix_meta: stored_prefix
-                .as_ref()
-                .map(|pfx| pfx.1.clone())
-                .unwrap_or_default(),
-            less_specifics: if options.include_less_specifics {
-                Some(
-                    self.less_specific_prefix_iter(
-                        if let Some(ref pfx) = stored_prefix {
-                            pfx.0
-                        } else {
-                            search_pfx
-                        },
-                        options.mui,
-                        options.include_withdrawn,
-                        guard,
-                    )
-                    .collect(),
-                )
-            } else {
-                None
-            },
-            more_specifics: if options.include_more_specifics {
-                Some(
-                    self.more_specific_prefix_iter_from(
-                        if let Some(pfx) = stored_prefix {
-                            pfx.0
-                        } else {
-                            search_pfx
-                        },
-                        options.mui,
-                        options.include_withdrawn,
-                        guard,
-                    )
-                    .collect(),
-                )
-                // The user requested more specifics, but there aren't any, so
-                // we need to return an empty vec, not a None.
-            } else {
-                None
-            },
-            match_type,
+            _ => self.in_memory_tree.match_prefix(search_pfx, options, guard),
         }
     }
 
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 9cedebee..394e7855 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -221,7 +221,7 @@ pub struct Rib<
     pub(in crate::local_array) persist_tree:
         Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
     // Global Roaring BitMap INdex that stores MUIs.
-    pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
+    // pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     pub counters: Counters,
 }
 
@@ -259,7 +259,7 @@ impl<
             config,
             in_memory_tree: TreeBitMap::<AF, M, NB, PB>::new()?,
             persist_tree,
-            withdrawn_muis_bmin: RoaringBitmap::new().into(),
+            // withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
         };
 
@@ -374,17 +374,6 @@ impl<
         self.in_memory_tree.get_nodes_count()
     }
 
-    pub(crate) fn withdrawn_muis_bmin(
-        &'a self,
-        guard: &'a Guard,
-    ) -> &'a RoaringBitmap {
-        unsafe {
-            self.withdrawn_muis_bmin
-                .load(Ordering::Acquire, guard)
-                .deref()
-        }
-    }
-
     // Change the status of the record for the specified (prefix, mui)
     // combination  to Withdrawn.
     pub fn mark_mui_as_withdrawn_for_prefix(
@@ -432,14 +421,17 @@ impl<
         mui: u32,
         guard: &Guard,
     ) -> Result<(), PrefixStoreError> {
-        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
+        let current = self
+            .in_memory_tree
+            .withdrawn_muis_bmin
+            .load(Ordering::Acquire, guard);
 
         let mut new = unsafe { current.as_ref() }.unwrap().clone();
         new.insert(mui);
 
         #[allow(clippy::assigning_clones)]
         loop {
-            match self.withdrawn_muis_bmin.compare_exchange(
+            match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
                 current,
                 Owned::new(new),
                 Ordering::AcqRel,
@@ -462,14 +454,17 @@ impl<
         mui: u32,
         guard: &Guard,
     ) -> Result<(), PrefixStoreError> {
-        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
+        let current = self
+            .in_memory_tree
+            .withdrawn_muis_bmin
+            .load(Ordering::Acquire, guard);
 
         let mut new = unsafe { current.as_ref() }.unwrap().clone();
         new.remove(mui);
 
         #[allow(clippy::assigning_clones)]
         loop {
-            match self.withdrawn_muis_bmin.compare_exchange(
+            match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
                 current,
                 Owned::new(new),
                 Ordering::AcqRel,
@@ -490,7 +485,8 @@ impl<
     // functions.
     pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
         unsafe {
-            self.withdrawn_muis_bmin
+            self.in_memory_tree
+                .withdrawn_muis_bmin
                 .load(Ordering::Acquire, guard)
                 .as_ref()
         }
@@ -503,7 +499,8 @@ impl<
     // functions.
     pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
         !unsafe {
-            self.withdrawn_muis_bmin
+            self.in_memory_tree
+                .withdrawn_muis_bmin
                 .load(Ordering::Acquire, guard)
                 .as_ref()
         }

From cae714cd318b246bfe3c3cee4f23dd97dc554def Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 24 Dec 2024 18:31:56 +0100
Subject: [PATCH 036/147] move iterators to in_mem

---
 src/local_array/{ => in_memory}/iterators.rs | 10 ++++------
 src/local_array/in_memory/mod.rs             |  3 +++
 src/local_array/in_memory/node.rs            |  2 +-
 3 files changed, 8 insertions(+), 7 deletions(-)
 rename src/local_array/{ => in_memory}/iterators.rs (99%)

diff --git a/src/local_array/iterators.rs b/src/local_array/in_memory/iterators.rs
similarity index 99%
rename from src/local_array/iterators.rs
rename to src/local_array/in_memory/iterators.rs
index 034b7490..80204104 100644
--- a/src/local_array/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -19,13 +19,12 @@
 // contention, since every lookup has to go through the levels near the root
 // in the TreeBitMap.
 
-use super::in_memory::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
-use super::in_memory::node::{SizedStrideRef, StrideNodeId};
-use super::in_memory::tree::{Stride3, Stride4, Stride5, TreeBitMap};
-use super::types::PrefixId;
+use super::super::types::PrefixId;
+use super::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
+use super::node::{SizedStrideRef, StrideNodeId};
+use super::tree::{Stride3, Stride4, Stride5, TreeBitMap};
 use crate::local_array::types::RouteStatus;
 use crate::prefix_record::PublicRecord;
-use crate::rib;
 use crate::{
     af::AddressFamily,
     local_array::{
@@ -36,7 +35,6 @@ use crate::{
     },
     prefix_record::Meta,
 };
-use rib::Rib;
 
 use crossbeam_epoch::Guard;
 use inetnum::addr::Prefix;
diff --git a/src/local_array/in_memory/mod.rs b/src/local_array/in_memory/mod.rs
index 9b6ac496..ad345222 100644
--- a/src/local_array/in_memory/mod.rs
+++ b/src/local_array/in_memory/mod.rs
@@ -6,6 +6,9 @@ pub(crate) mod tree;
 
 // #[deprecated]
 // mod deprecated_query;
+
+pub mod iterators;
+
 mod query;
 
 #[macro_use]
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 0aedb6e5..dd3b446f 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -8,7 +8,7 @@ use log::trace;
 use parking_lot_core::SpinWait;
 
 use super::super::bit_span::BitSpan;
-use super::super::iterators::{SizedNodeMoreSpecificIter, SizedPrefixIter};
+use super::iterators::{SizedNodeMoreSpecificIter, SizedPrefixIter};
 use super::tree::{AtomicBitmap, AtomicStride2, AtomicStride3, AtomicStride4, AtomicStride5, CasResult, Stride, Stride3, Stride4, Stride5};
 
 // pub use crate::in_memory_tree::*;

From ff67868d6b75509aedd5d3f01cecd8c0ecc699ac Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 24 Dec 2024 21:36:55 +0100
Subject: [PATCH 037/147] move iterators

---
 src/local_array/mod.rs     | 1 -
 src/local_array/rib/rib.rs | 5 ++---
 src/prelude/mod.rs         | 1 +
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/local_array/mod.rs b/src/local_array/mod.rs
index 41b51128..2d511450 100644
--- a/src/local_array/mod.rs
+++ b/src/local_array/mod.rs
@@ -4,7 +4,6 @@ mod tests;
 
 pub(crate) mod bit_span;
 pub mod errors;
-pub mod iterators;
 pub mod query;
 pub mod rib;
 pub mod types;
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 394e7855..b3ac7ee9 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -5,9 +5,8 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use inetnum::addr::Prefix;
 use log::info;
 
-use crossbeam_epoch::{self as epoch, Atomic};
+use crossbeam_epoch::{self as epoch};
 use epoch::{Guard, Owned};
-use roaring::RoaringBitmap;
 
 use crate::local_array::in_memory::tree::TreeBitMap;
 use crate::local_array::types::PrefixId;
@@ -23,7 +22,7 @@ use crate::local_array::in_memory::atomic_types::{
 
 // Make sure to also import the other methods for the Rib, so the proc macro
 // create_store can use them.
-pub use crate::local_array::iterators;
+pub use crate::local_array::in_memory::iterators;
 pub use crate::local_array::query;
 
 use crate::{IPv4, IPv6, Meta};
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index df6c1ede..68c42f5e 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -19,6 +19,7 @@ pub mod multi {
     pub use crate::local_array::in_memory::atomic_types::{
         NodeBuckets, NodeSet, PrefixBuckets, PrefixSet,
     };
+    pub use crate::local_array::in_memory::iterators;
     pub use crate::local_array::in_memory::node::StrideNodeId;
     pub use crate::local_array::types::{PrefixId, RouteStatus};
     pub use crate::prefix_record::PublicRecord as Record;

From a75e5c77903895ddfbaacd2f50300149169e7450 Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Fri, 27 Dec 2024 10:25:15 +0100
Subject: [PATCH 038/147] add docstring

---
 src/local_array/rib/rib.rs | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index b3ac7ee9..dc40dc5c 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -31,13 +31,20 @@ use crate::AddressFamily;
 
 //------------ StoreConfig ---------------------------------------------------
 
-/// Some description
+/// Defines where records are stored, in-memory and/or persisted (to disk),
+/// and, whether new records for a unique (prefix, mui) pair are overwritten
+/// or persisted.
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum PersistStrategy {
-    /// The current records are kept in history
+    /// Current records are stored both in-memory and persisted. Additionally
+    /// historical records are persisted.
     WriteAhead,
+    /// Current records are stored in-memory, historical records are pesisted.
     PersistHistory,
+    /// Current records are stored in-memory, historical records are discarded
+    /// when nwer records appear.
     MemoryOnly,
+    /// Both current and historical records are persisted.
     PersistOnly,
 }
 

From 11a2f2c79d96dcf2bebffa00ed06b1f1ec77645e Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 27 Dec 2024 12:59:20 +0100
Subject: [PATCH 039/147] contains method

---
 proc_macros/src/lib.rs            | 29 +++++++++++
 src/local_array/in_memory/tree.rs | 83 +++++++++++++++++++++++++++++++
 src/local_array/rib/rib.rs        |  8 +++
 3 files changed, 120 insertions(+)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 077d9add..a3415aec 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -709,6 +709,35 @@ pub fn create_store(
                 }
             }
 
+
+            /// Return whether the requested prefix was ever created in the
+            /// RIB. Specifying a multi unique id will return whether the
+            /// (prefix, multi_uniq_id) tuple was ever created.
+            ///
+            /// The result ignores the status of the prefix, e.g. whether it
+            /// was withdrawn, or whether the global withdrawn for the multi
+            /// unique id is withdrawn.
+            pub fn contains(
+                &'a self,
+                prefix: &Prefix,
+                mui: Option<u32>
+            ) -> bool {
+                match prefix.addr() {
+                    std::net::IpAddr::V4(addr) => {
+                        self.v4.contains(
+                            PrefixId::<IPv4>::from(*prefix),
+                            mui
+                        )
+                    },
+                    std::net::IpAddr::V6(addr) => {
+                        self.v6.contains(
+                            PrefixId::<IPv6>::from(*prefix),
+                            mui
+                        )
+                    }
+                }
+            }
+
             /// Return the record that belongs to the pre-calculated and
             /// stored best path for a given prefix.
             ///
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 69e84395..8e0f6e61 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -454,6 +454,89 @@ impl<
         }
     }
 
+    pub fn prefix_exists_for_mui(
+        &self,
+        prefix_id: PrefixId<AF>,
+        mui: u32,
+    ) -> bool {
+        // if prefix_id.get_len() == 0 {
+        //     return self.update_default_route_prefix_meta(mui);
+        // }
+
+        let mut stride_end: u8 = 0;
+        let mut cur_i = self.get_root_node_id();
+        let mut level: u8 = 0;
+
+        loop {
+            let stride = self.get_stride_sizes()[level as usize];
+            stride_end += stride;
+            let nibble_len = if prefix_id.get_len() < stride_end {
+                stride + prefix_id.get_len() - stride_end
+            } else {
+                stride
+            };
+
+            let nibble = AF::get_nibble(
+                prefix_id.get_net(),
+                stride_end - stride,
+                nibble_len,
+            );
+            let stride_start = stride_end - stride;
+
+            let node_result = match self.retrieve_node_for_mui(cur_i, mui) {
+                Some(node) => match node {
+                    SizedStrideRef::Stride3(n) => {
+                        let (child_node, found) = n.prefix_exists_in_stride(
+                            prefix_id,
+                            nibble,
+                            nibble_len,
+                            stride_start,
+                        );
+                        if found {
+                            return true;
+                        } else {
+                            child_node
+                        }
+                    }
+                    SizedStrideRef::Stride4(n) => {
+                        let (child_node, found) = n.prefix_exists_in_stride(
+                            prefix_id,
+                            nibble,
+                            nibble_len,
+                            stride_start,
+                        );
+                        if found {
+                            return true;
+                        } else {
+                            child_node
+                        }
+                    }
+                    SizedStrideRef::Stride5(n) => {
+                        let (child_node, found) = n.prefix_exists_in_stride(
+                            prefix_id,
+                            nibble,
+                            nibble_len,
+                            stride_start,
+                        );
+                        if found {
+                            return true;
+                        } else {
+                            child_node
+                        }
+                    }
+                },
+                None => {
+                    return false;
+                }
+            };
+
+            if let Some(next_id) = node_result {
+                cur_i = next_id;
+                level += 1;
+            }
+        }
+    }
+
     pub(crate) fn upsert_prefix(
         &self,
         prefix: PrefixId<AF>,
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index dc40dc5c..a05ea4e5 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -376,6 +376,14 @@ impl<
         }
     }
 
+    pub fn contains(&self, prefix: PrefixId<AF>, mui: Option<u32>) -> bool {
+        if let Some(mui) = mui {
+            self.in_memory_tree.prefix_exists_for_mui(prefix, mui)
+        } else {
+            self.in_memory_tree.prefix_exists(prefix)
+        }
+    }
+
     pub fn get_nodes_count(&self) -> usize {
         self.in_memory_tree.get_nodes_count()
     }

From 354baa4058b9924053e6a5c99ef3108ea0059a14 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 27 Dec 2024 17:56:39 +0100
Subject: [PATCH 040/147] better QueryResult separation

---
 proc_macros/src/lib.rs              |   4 +-
 src/local_array/in_memory/query.rs  |  14 +-
 src/local_array/persist/lsm_tree.rs |  60 ++++-----
 src/local_array/query.rs            | 199 ++++++++++++++++++++++++++--
 src/local_array/rib/rib.rs          |  14 --
 src/local_array/types.rs            |   8 +-
 6 files changed, 232 insertions(+), 67 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index a3415aec..da3e001a 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -694,7 +694,7 @@ pub fn create_store(
                             ),
                             options,
                             guard
-                        )
+                        ).into()
                     },
                     std::net::IpAddr::V6(addr) => {
                         self.v6.match_prefix(
@@ -704,7 +704,7 @@ pub fn create_store(
                             ),
                             options,
                             guard
-                        )
+                        ).into()
                     }
                 }
             }
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index 23deb2f6..c636f8f6 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -3,9 +3,9 @@ use log::trace;
 
 use crate::af::AddressFamily;
 use crate::prelude::multi::RouteStatus;
-use crate::rib::query::TreeQueryResult;
+use crate::rib::query::{FamilyQueryResult, TreeQueryResult};
 
-use crate::{Meta, QueryResult};
+use crate::{Meta, PublicRecord, QueryResult};
 
 use crate::local_array::in_memory::node::{SizedStrideRef, TreeBitMapNode};
 use crate::{MatchOptions, MatchType};
@@ -28,7 +28,7 @@ where
         search_pfx: PrefixId<AF>,
         options: &MatchOptions,
         guard: &'a Guard,
-    ) -> QueryResult<M> {
+    ) -> FamilyQueryResult<AF, M> {
         // `non_recursive_retrieve_prefix` returns an exact match only, so no
         // longest matching prefix!
         let withdrawn_muis_bmin = self.withdrawn_muis_bmin(guard);
@@ -92,8 +92,8 @@ where
             (MatchType::ExactMatch, _) => MatchType::EmptyMatch,
         };
 
-        QueryResult {
-            prefix: stored_prefix.as_ref().map(|p| p.0.into_pub()),
+        FamilyQueryResult {
+            prefix: stored_prefix.as_ref().map(|sp| sp.0), //.as_ref().map(|p| p.0.into_pub()),
             prefix_meta: stored_prefix
                 .as_ref()
                 .map(|pfx| pfx.1.clone())
@@ -110,7 +110,7 @@ where
                         options.include_withdrawn,
                         guard,
                     )
-                    .collect(),
+                    .collect::<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>>(),
                 )
             } else {
                 None
@@ -127,7 +127,7 @@ where
                         options.include_withdrawn,
                         guard,
                     )
-                    .collect(),
+                    .collect::<Vec<_>>(),
                 )
                 // The user requested more specifics, but there aren't any, so
                 // we need to return an empty vec, not a None.
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 6fc3128c..ca88b60f 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -8,7 +8,7 @@ use lsm_tree::AbstractTree;
 
 use crate::local_array::types::{PrefixId, RouteStatus};
 use crate::prefix_record::PublicPrefixRecord;
-use crate::rib::query::TreeQueryResult;
+use crate::rib::query::{FamilyQueryResult, TreeQueryResult};
 use crate::rib::Counters;
 use crate::{
     AddressFamily, IncludeHistory, MatchOptions, Meta, PublicRecord,
@@ -76,7 +76,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     pub fn get_records_for_key<M: Meta + From<Vec<u8>>>(
         &self,
         key: &[u8],
-    ) -> Vec<(inetnum::addr::Prefix, PublicRecord<M>)> {
+    ) -> Vec<(PrefixId<AF>, PublicRecord<M>)> {
         (*self.tree.prefix(key))
             .into_iter()
             .map(|kv| {
@@ -85,7 +85,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     Self::parse_key(kv.0.as_ref());
 
                 (
-                    PrefixId::<AF>::from(pfx).into_pub(),
+                    PrefixId::<AF>::from(pfx),
                     PublicRecord::new(
                         mui,
                         ltime,
@@ -101,7 +101,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         prefix: Option<PrefixId<AF>>,
         mui: Option<u32>,
-    ) -> (Option<Prefix>, Vec<PublicRecord<M>>) {
+    ) -> (Option<PrefixId<AF>>, Vec<PublicRecord<M>>) {
         match prefix {
             Some(pfx) => {
                 let prefix_b = if let Some(mui) = mui {
@@ -110,7 +110,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     &pfx.as_bytes::<PREFIX_SIZE>()
                 };
                 (
-                    prefix.map(|p| p.into_pub()),
+                    prefix,
                     (*self.tree.prefix(prefix_b))
                         .into_iter()
                         .last()
@@ -136,17 +136,17 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         prefixes: Option<Vec<PrefixId<AF>>>,
         mui: Option<u32>,
-    ) -> Option<RecordSet<M>> {
+    ) -> Option<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>> {
         prefixes.map(|pfxs| {
             pfxs.iter()
-                .flat_map(|pfx| {
+                .map(|pfx| {
                     let prefix_b = if let Some(mui) = mui {
                         &Self::prefix_mui_persistence_key(*pfx, mui)
                     } else {
                         &pfx.as_bytes::<PREFIX_SIZE>()
                     };
-                    Some(PublicPrefixRecord::from((
-                        pfx.into_pub(),
+                    (
+                        *pfx,
                         (*self.tree.prefix(prefix_b))
                             .into_iter()
                             .last()
@@ -162,21 +162,20 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                                 )]
                             })
                             .unwrap_or_default(),
-                    )))
+                    )
                 })
-                .collect::<RecordSet<M>>()
+                .collect::<Vec<_>>()
         })
+        // .collect::<(PrefixId<AF>, Vec<PublicRecord<M>>)>()
     }
 
     fn enrich_prefix<M: Meta>(
         &self,
         prefix: Option<PrefixId<AF>>,
         mui: Option<u32>,
-    ) -> (Option<Prefix>, Vec<PublicRecord<M>>) {
+    ) -> (Option<PrefixId<AF>>, Vec<PublicRecord<M>>) {
         match prefix {
-            Some(pfx) => {
-                (Some(pfx.into_pub()), self.get_records_for_prefix(pfx, mui))
-            }
+            Some(pfx) => (Some(pfx), self.get_records_for_prefix(pfx, mui)),
             None => (None, vec![]),
         }
     }
@@ -185,29 +184,22 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         prefixes: Option<Vec<PrefixId<AF>>>,
         mui: Option<u32>,
-    ) -> Option<RecordSet<M>> {
-        prefixes.map(|ls| {
-            ls.iter()
+    ) -> Option<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>> {
+        prefixes.map(|recs| {
+            recs.iter()
                 .flat_map(move |pfx| {
-                    Some(PublicPrefixRecord::from((
-                        *pfx,
-                        self.get_records_for_prefix(*pfx, mui),
-                    )))
+                    Some((*pfx, self.get_records_for_prefix(*pfx, mui)))
                 })
-                .collect::<RecordSet<M>>()
+                .collect()
         })
     }
 
     fn sparse_record_set<M: Meta>(
         &self,
         prefixes: Option<Vec<PrefixId<AF>>>,
-    ) -> Option<RecordSet<M>> {
-        prefixes.map(|ls| {
-            ls.iter()
-                .flat_map(|pfx| {
-                    Some(PublicPrefixRecord::from((*pfx, vec![])))
-                })
-                .collect::<RecordSet<M>>()
+    ) -> Option<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>> {
+        prefixes.map(|recs| {
+            recs.iter().flat_map(|pfx| Some((*pfx, vec![]))).collect()
         })
     }
 
@@ -215,14 +207,14 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         search_pfxs: TreeQueryResult<AF>,
         options: &MatchOptions,
-    ) -> QueryResult<M> {
+    ) -> FamilyQueryResult<AF, M> {
         match options.include_history {
             // All the records for all the prefixes
             IncludeHistory::All => {
                 let (prefix, prefix_meta) =
                     self.enrich_prefix(search_pfxs.prefix, options.mui);
 
-                QueryResult {
+                FamilyQueryResult {
                     prefix,
                     prefix_meta,
                     match_type: search_pfxs.match_type,
@@ -244,7 +236,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                 let (prefix, prefix_meta) =
                     self.enrich_prefix(search_pfxs.prefix, options.mui);
 
-                QueryResult {
+                FamilyQueryResult {
                     prefix,
                     prefix_meta,
                     match_type: search_pfxs.match_type,
@@ -262,7 +254,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     options.mui,
                 );
 
-                QueryResult {
+                FamilyQueryResult {
                     prefix,
                     prefix_meta,
                     match_type: search_pfxs.match_type,
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 529fef89..f2f0b9f4 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -9,7 +9,7 @@ use crate::prefix_record::PublicRecord;
 use crate::rib::{PersistStrategy, Rib};
 use inetnum::addr::Prefix;
 
-use crate::{Meta, QueryResult};
+use crate::{IncludeHistory, Meta, QueryResult};
 
 use crate::{MatchOptions, MatchType};
 
@@ -145,19 +145,174 @@ where
         guard: &'a Guard,
     ) -> QueryResult<M> {
         match self.config.persist_strategy() {
+            // All the records are persisted, they have never been committed
+            // to memory. However the in-memory-tree is still used to indicate
+            // which (prefix, mui) tuples have been created.
             PersistStrategy::PersistOnly => {
+                if options.match_type == MatchType::ExactMatch
+                    && !self.contains(search_pfx, options.mui)
+                {
+                    return QueryResult::empty();
+                }
+
                 if let Some(persist_tree) = &self.persist_tree {
-                    persist_tree.match_prefix(
-                        self.in_memory_tree.match_prefix_by_tree_traversal(
-                            search_pfx, options,
-                        ),
-                        options,
-                    )
+                    persist_tree
+                        .match_prefix(
+                            self.in_memory_tree
+                                .match_prefix_by_tree_traversal(
+                                    search_pfx, options,
+                                ),
+                            options,
+                        )
+                        .into()
                 } else {
                     QueryResult::empty()
                 }
             }
-            _ => self.in_memory_tree.match_prefix(search_pfx, options, guard),
+            // Everything is in memory only, so look there. There can be no
+            // historical records for this variant, so we just return the
+            // stuff found in memeory. A request for  historical records
+            // willbe ignored.
+            PersistStrategy::MemoryOnly => self
+                .in_memory_tree
+                .match_prefix(search_pfx, options, guard)
+                .into(),
+            // We have the current records in memory, additionally they may be
+            // encriched with historical data from the persisted data.
+            PersistStrategy::WriteAhead => {
+                let mut res = self
+                    .in_memory_tree
+                    .match_prefix(search_pfx, options, guard);
+                if let Some(persist_tree) = &self.persist_tree {
+                    match options.include_history {
+                        IncludeHistory::All => {
+                            if self.contains(search_pfx, options.mui) {
+                                res.prefix_meta.extend(
+                                    persist_tree.get_records_for_prefix(
+                                        search_pfx,
+                                        options.mui,
+                                    ),
+                                );
+                            }
+
+                            if let Some(ls) = &mut res.less_specifics {
+                                for rec in ls {
+                                    if self.contains(rec.0, options.mui) {
+                                        // write ahead has the current record,
+                                        // as well as the historical ones, so
+                                        // we just override the recs from the
+                                        // in-memory tree.
+                                        rec.1 = persist_tree
+                                            .get_records_for_prefix(
+                                                rec.0,
+                                                options.mui,
+                                            );
+                                    }
+                                }
+                            };
+
+                            if let Some(rs) = &mut res.more_specifics {
+                                for rec in rs {
+                                    if self.contains(rec.0, options.mui) {
+                                        rec.1 = persist_tree
+                                            .get_records_for_prefix(
+                                                rec.0,
+                                                options.mui,
+                                            );
+                                    }
+                                }
+                            };
+
+                            res.into()
+                        }
+                        IncludeHistory::SearchPrefix => {
+                            if self.contains(search_pfx, options.mui) {
+                                res.prefix_meta.extend(
+                                    persist_tree.get_records_for_prefix(
+                                        search_pfx,
+                                        options.mui,
+                                    ),
+                                );
+                            }
+
+                            res.into()
+                        }
+                        IncludeHistory::None => res.into(),
+                    }
+                } else {
+                    res.into()
+                }
+            }
+            // All current info is in memory so look there,
+            // `in_memory_tree.match_prefix` will enrich with historical data
+            // for the prefix(es) if need be.
+            PersistStrategy::PersistHistory => {
+                let mut res = self
+                    .in_memory_tree
+                    .match_prefix(search_pfx, options, guard);
+
+                if let Some(persist_tree) = &self.persist_tree {
+                    match options.include_history {
+                        IncludeHistory::All => {
+                            res.prefix_meta.extend(
+                                persist_tree.get_records_for_prefix(
+                                    search_pfx,
+                                    options.mui,
+                                ),
+                            );
+
+                            if let Some(ls) = &mut res.less_specifics {
+                                for rec in ls {
+                                    if self.contains(rec.0, options.mui) {
+                                        // the persisted tree only has the
+                                        // historical records, so we extend
+                                        // the vec. It should already hold the
+                                        // current record.
+                                        rec.1.extend(
+                                            persist_tree
+                                                .get_records_for_prefix(
+                                                    rec.0,
+                                                    options.mui,
+                                                ),
+                                        );
+                                    }
+                                }
+                            };
+
+                            if let Some(rs) = &mut res.more_specifics {
+                                for rec in rs {
+                                    if self.contains(rec.0, options.mui) {
+                                        rec.1.extend(
+                                            persist_tree
+                                                .get_records_for_prefix(
+                                                    rec.0,
+                                                    options.mui,
+                                                ),
+                                        );
+                                    }
+                                }
+                            };
+
+                            res.into()
+                        }
+                        IncludeHistory::SearchPrefix => {
+                            if self.contains(search_pfx, options.mui) {
+                                res.prefix_meta.extend(
+                                    persist_tree.get_records_for_prefix(
+                                        search_pfx,
+                                        options.mui,
+                                    ),
+                                );
+                            }
+
+                            res.into()
+                        }
+                        IncludeHistory::None => res.into(),
+                    }
+                } else {
+                    res.into()
+                }
+            }
         }
     }
 
@@ -210,9 +365,35 @@ where
     }
 }
 
-pub struct TreeQueryResult<AF: AddressFamily> {
+pub(crate) struct TreeQueryResult<AF: AddressFamily> {
     pub match_type: MatchType,
     pub prefix: Option<PrefixId<AF>>,
     pub less_specifics: Option<Vec<PrefixId<AF>>>,
     pub more_specifics: Option<Vec<PrefixId<AF>>>,
 }
+
+pub(crate) struct FamilyQueryResult<AF: AddressFamily, M: Meta> {
+    pub match_type: MatchType,
+    pub prefix: Option<PrefixId<AF>>,
+    pub prefix_meta: Vec<PublicRecord<M>>,
+    pub less_specifics: Option<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>>,
+    pub more_specifics: Option<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>>,
+}
+
+impl<AF: AddressFamily, M: Meta> From<FamilyQueryResult<AF, M>>
+    for QueryResult<M>
+{
+    fn from(value: FamilyQueryResult<AF, M>) -> Self {
+        QueryResult {
+            match_type: value.match_type,
+            prefix: value.prefix.map(|p| p.into()),
+            prefix_meta: value.prefix_meta,
+            less_specifics: value
+                .less_specifics
+                .map(|ls| ls.into_iter().collect()),
+            more_specifics: value
+                .more_specifics
+                .map(|ms| ms.into_iter().collect()),
+        }
+    }
+}
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index a05ea4e5..ea47d1c2 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -232,7 +232,6 @@ pub struct Rib<
 }
 
 impl<
-        'a,
         AF: AddressFamily,
         M: crate::prefix_record::Meta,
         NB: NodeBuckets<AF>,
@@ -522,19 +521,6 @@ impl<
         .contains(mui)
     }
 
-    // Helper to filter out records that are not-active (Inactive or
-    // Withdrawn), or whose mui appears in the global withdrawn index.
-    // pub(crate) fn get_filtered_records(
-    //     &self,
-    //     pfx: &StoredPrefix<AF, M>,
-    //     mui: Option<u32>,
-    //     guard: &Guard,
-    // ) -> Vec<PublicRecord<M>> {
-    //     let bmin = self.withdrawn_muis_bmin(guard);
-
-    //     pfx.record_map.get_filtered_records(mui, bmin)
-    // }
-
     pub fn get_prefixes_count(&self) -> UpsertCounters {
         UpsertCounters {
             in_memory_count: self.in_memory_tree.get_prefixes_count(),
diff --git a/src/local_array/types.rs b/src/local_array/types.rs
index 13f3b91b..5b93eccf 100644
--- a/src/local_array/types.rs
+++ b/src/local_array/types.rs
@@ -1,4 +1,4 @@
-use crate::AddressFamily;
+use crate::{AddressFamily, Meta};
 
 use super::errors::PrefixStoreError;
 
@@ -102,6 +102,12 @@ impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
     }
 }
 
+impl<AF: AddressFamily> Into<inetnum::addr::Prefix> for PrefixId<AF> {
+    fn into(self) -> inetnum::addr::Prefix {
+        self.into_pub()
+    }
+}
+
 impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<PrefixId<AF>>
     for [u8; PREFIX_SIZE]
 {

From f7410976273159090e37ac307fca38d21da38954 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 27 Dec 2024 18:50:12 +0100
Subject: [PATCH 041/147] remove PersistStatus in MultiMapValue. it is the
 wrong place

---
 src/local_array/in_memory/atomic_types.rs | 41 ++--------
 src/local_array/in_memory/query.rs        |  4 +-
 src/local_array/in_memory/tree.rs         | 91 +++++++++++------------
 src/local_array/persist/lsm_tree.rs       | 14 ++--
 src/local_array/query.rs                  | 29 ++++----
 src/local_array/rib/mod.rs                |  1 +
 src/local_array/rib/rib.rs                | 21 +-----
 src/local_array/types.rs                  |  8 +-
 8 files changed, 82 insertions(+), 127 deletions(-)

diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index d96d78d8..aba43663 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -258,25 +258,11 @@ impl<AF: AddressFamily, M: crate::prefix_record::Meta> StoredPrefix<AF, M> {
     }
 }
 
-#[derive(Copy, Clone, Debug)]
-pub(crate) struct PersistStatus(bool);
-
-impl PersistStatus {
-    pub(crate) fn persisted() -> Self {
-        Self(true)
-    }
-
-    pub(crate) fn not_persisted() -> Self {
-        Self(false)
-    }
-}
-
 #[derive(Clone, Debug)]
 pub(crate) struct MultiMapValue<M> {
     meta: M,
     ltime: u64,
     route_status: RouteStatus,
-    persist_status: PersistStatus,
 }
 
 impl<M: Meta> MultiMapValue<M> {
@@ -295,10 +281,6 @@ impl<M: Meta> MultiMapValue<M> {
     pub(crate) fn set_route_status(&mut self, status: RouteStatus) {
         self.route_status = status;
     }
-
-    pub(crate) fn has_persisted_data(&self) -> bool {
-        self.persist_status.0
-    }
 }
 
 impl<M: Meta> std::fmt::Display for MultiMapValue<M> {
@@ -313,13 +295,12 @@ impl<M: Meta> std::fmt::Display for MultiMapValue<M> {
     }
 }
 
-impl<M: Meta> From<(PublicRecord<M>, PersistStatus)> for MultiMapValue<M> {
-    fn from(value: (PublicRecord<M>, PersistStatus)) -> Self {
+impl<M: Meta> From<PublicRecord<M>> for MultiMapValue<M> {
+    fn from(value: PublicRecord<M>) -> Self {
         Self {
-            ltime: value.0.ltime,
-            route_status: value.0.status,
-            meta: value.0.meta,
-            persist_status: value.1,
+            ltime: value.ltime,
+            route_status: value.status,
+            meta: value.meta,
         }
     }
 }
@@ -508,13 +489,10 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     // record.multi_uniq_id. Returns the number of entries in the HashMap
     // after updating it, if it's more than 1. Returns None if this is the
     // first entry.
+    #[allow(clippy::type_complexity)]
     pub(crate) fn upsert_record(
         &self,
-        // prefix: PrefixId<AF>,
         new_rec: PublicRecord<M>,
-        persist_status: PersistStatus,
-        // persistence: &Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
-        // strategy: PersistStrategy,
     ) -> Result<(Option<(MultiMapValue<M>, usize)>, usize), PrefixStoreError>
     {
         let (mut record_map, retry_count) = self.guard_with_retry(0);
@@ -523,15 +501,12 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         match record_map.contains_key(&key) {
             true => {
                 let old_rec = record_map
-                    .insert(
-                        key,
-                        MultiMapValue::from((new_rec, persist_status)),
-                    )
+                    .insert(key, MultiMapValue::from(new_rec))
                     .map(|r| (r, record_map.len()));
                 Ok((old_rec, retry_count))
             }
             false => {
-                let new_rec = MultiMapValue::from((new_rec, persist_status));
+                let new_rec = MultiMapValue::from(new_rec);
                 let old_rec = record_map.insert(key, new_rec);
                 assert!(old_rec.is_none());
                 Ok((None, retry_count))
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index c636f8f6..4196a20d 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -5,7 +5,7 @@ use crate::af::AddressFamily;
 use crate::prelude::multi::RouteStatus;
 use crate::rib::query::{FamilyQueryResult, TreeQueryResult};
 
-use crate::{Meta, PublicRecord, QueryResult};
+use crate::{Meta, PublicRecord};
 
 use crate::local_array::in_memory::node::{SizedStrideRef, TreeBitMapNode};
 use crate::{MatchOptions, MatchType};
@@ -93,7 +93,7 @@ where
         };
 
         FamilyQueryResult {
-            prefix: stored_prefix.as_ref().map(|sp| sp.0), //.as_ref().map(|p| p.0.into_pub()),
+            prefix: stored_prefix.as_ref().map(|sp| sp.0),
             prefix_meta: stored_prefix
                 .as_ref()
                 .map(|pfx| pfx.1.clone())
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 8e0f6e61..f3a26786 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -198,8 +198,7 @@ use std::sync::atomic::{
 use std::{fmt::Debug, marker::PhantomData};
 
 use super::atomic_types::{
-    MultiMapValue, NodeBuckets, PersistStatus, PrefixBuckets, PrefixSet,
-    StoredPrefix,
+    MultiMapValue, NodeBuckets, PrefixBuckets, PrefixSet, StoredPrefix,
 };
 use crate::af::AddressFamily;
 use crate::rib::{Counters, UpsertReport};
@@ -541,7 +540,6 @@ impl<
         &self,
         prefix: PrefixId<AF>,
         record: PublicRecord<M>,
-        persist_status: PersistStatus,
         update_path_selections: Option<M::TBI>,
         guard: &Guard,
     ) -> Result<(UpsertReport, Option<MultiMapValue<M>>), PrefixStoreError>
@@ -549,38 +547,36 @@ impl<
         let mut prefix_is_new = true;
         let mut mui_is_new = true;
 
-        let (mui_count, cas_count) = match self
-            .non_recursive_retrieve_prefix_mut(prefix)
-        {
-            // There's no StoredPrefix at this location yet. Create a new
-            // PrefixRecord and try to store it in the empty slot.
-            (stored_prefix, false) => {
-                if log_enabled!(log::Level::Debug) {
-                    debug!(
-                        "{} store: Create new prefix record",
-                        std::thread::current()
-                            .name()
-                            .unwrap_or("unnamed-thread")
-                    );
-                }
+        let (mui_count, cas_count) =
+            match self.non_recursive_retrieve_prefix_mut(prefix) {
+                // There's no StoredPrefix at this location yet. Create a new
+                // PrefixRecord and try to store it in the empty slot.
+                (stored_prefix, false) => {
+                    if log_enabled!(log::Level::Debug) {
+                        debug!(
+                            "{} store: Create new prefix record",
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread")
+                        );
+                    }
 
-                let (mui_count, retry_count) = stored_prefix
-                    .record_map
-                    .upsert_record(record, persist_status)?;
+                    let (mui_count, retry_count) =
+                        stored_prefix.record_map.upsert_record(record)?;
 
-                // See if someone beat us to creating the record.
-                if mui_count.is_some() {
-                    mui_is_new = false;
-                    prefix_is_new = false;
-                }
+                    // See if someone beat us to creating the record.
+                    if mui_count.is_some() {
+                        mui_is_new = false;
+                        prefix_is_new = false;
+                    }
 
-                (mui_count, retry_count)
-            }
-            // There already is a StoredPrefix with a record at this
-            // location.
-            (stored_prefix, true) => {
-                if log_enabled!(log::Level::Debug) {
-                    debug!(
+                    (mui_count, retry_count)
+                }
+                // There already is a StoredPrefix with a record at this
+                // location.
+                (stored_prefix, true) => {
+                    if log_enabled!(log::Level::Debug) {
+                        debug!(
                         "{} store: Found existing prefix record for {}/{}",
                         std::thread::current()
                             .name()
@@ -588,26 +584,25 @@ impl<
                         prefix.get_net(),
                         prefix.get_len()
                     );
-                }
-                prefix_is_new = false;
+                    }
+                    prefix_is_new = false;
 
-                // Update the already existing record_map with our
-                // caller's record.
-                stored_prefix.set_ps_outdated(guard)?;
+                    // Update the already existing record_map with our
+                    // caller's record.
+                    stored_prefix.set_ps_outdated(guard)?;
 
-                let (mui_count, retry_count) = stored_prefix
-                    .record_map
-                    .upsert_record(record, persist_status)?;
-                mui_is_new = mui_count.is_none();
+                    let (mui_count, retry_count) =
+                        stored_prefix.record_map.upsert_record(record)?;
+                    mui_is_new = mui_count.is_none();
 
-                if let Some(tbi) = update_path_selections {
-                    stored_prefix
-                        .calculate_and_store_best_backup(&tbi, guard)?;
-                }
+                    if let Some(tbi) = update_path_selections {
+                        stored_prefix
+                            .calculate_and_store_best_backup(&tbi, guard)?;
+                    }
 
-                (mui_count, retry_count)
-            }
-        };
+                    (mui_count, retry_count)
+                }
+            };
 
         let count = mui_count.as_ref().map(|m| m.1).unwrap_or(1);
         Ok((
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index ca88b60f..4ed84aa5 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -3,16 +3,13 @@
 use std::marker::PhantomData;
 use std::path::Path;
 
-use inetnum::addr::Prefix;
 use lsm_tree::AbstractTree;
 
 use crate::local_array::types::{PrefixId, RouteStatus};
-use crate::prefix_record::PublicPrefixRecord;
-use crate::rib::query::{FamilyQueryResult, TreeQueryResult};
+use crate::rib::query::{FamilyQueryResult, FamilyRecord, TreeQueryResult};
 use crate::rib::Counters;
 use crate::{
     AddressFamily, IncludeHistory, MatchOptions, Meta, PublicRecord,
-    QueryResult, RecordSet,
 };
 
 pub struct PersistTree<
@@ -73,7 +70,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             .collect::<Vec<_>>()
     }
 
-    pub fn get_records_for_key<M: Meta + From<Vec<u8>>>(
+    pub fn _get_records_for_key<M: Meta + From<Vec<u8>>>(
         &self,
         key: &[u8],
     ) -> Vec<(PrefixId<AF>, PublicRecord<M>)> {
@@ -136,7 +133,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         prefixes: Option<Vec<PrefixId<AF>>>,
         mui: Option<u32>,
-    ) -> Option<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>> {
+    ) -> Option<FamilyRecord<AF, M>> {
         prefixes.map(|pfxs| {
             pfxs.iter()
                 .map(|pfx| {
@@ -166,7 +163,6 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                 })
                 .collect::<Vec<_>>()
         })
-        // .collect::<(PrefixId<AF>, Vec<PublicRecord<M>>)>()
     }
 
     fn enrich_prefix<M: Meta>(
@@ -184,7 +180,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         prefixes: Option<Vec<PrefixId<AF>>>,
         mui: Option<u32>,
-    ) -> Option<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>> {
+    ) -> Option<FamilyRecord<AF, M>> {
         prefixes.map(|recs| {
             recs.iter()
                 .flat_map(move |pfx| {
@@ -197,7 +193,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     fn sparse_record_set<M: Meta>(
         &self,
         prefixes: Option<Vec<PrefixId<AF>>>,
-    ) -> Option<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>> {
+    ) -> Option<FamilyRecord<AF, M>> {
         prefixes.map(|recs| {
             recs.iter().flat_map(|pfx| Some((*pfx, vec![]))).collect()
         })
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index f2f0b9f4..b2e752a8 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -145,6 +145,14 @@ where
         guard: &'a Guard,
     ) -> QueryResult<M> {
         match self.config.persist_strategy() {
+            // Everything is in memory only, so look there. There can be no
+            // historical records for this variant, so we just return the
+            // stuff found in memeory. A request for  historical records
+            // willbe ignored.
+            PersistStrategy::MemoryOnly => self
+                .in_memory_tree
+                .match_prefix(search_pfx, options, guard)
+                .into(),
             // All the records are persisted, they have never been committed
             // to memory. However the in-memory-tree is still used to indicate
             // which (prefix, mui) tuples have been created.
@@ -169,14 +177,6 @@ where
                     QueryResult::empty()
                 }
             }
-            // Everything is in memory only, so look there. There can be no
-            // historical records for this variant, so we just return the
-            // stuff found in memeory. A request for  historical records
-            // willbe ignored.
-            PersistStrategy::MemoryOnly => self
-                .in_memory_tree
-                .match_prefix(search_pfx, options, guard)
-                .into(),
             // We have the current records in memory, additionally they may be
             // encriched with historical data from the persisted data.
             PersistStrategy::WriteAhead => {
@@ -243,9 +243,9 @@ where
                     res.into()
                 }
             }
-            // All current info is in memory so look there,
-            // `in_memory_tree.match_prefix` will enrich with historical data
-            // for the prefix(es) if need be.
+            // All current info is in memory so look there. If the user has
+            // requested historical records, we will ask the persist_tree to
+            // add those to the intermedidate result of the in-memory query.
             PersistStrategy::PersistHistory => {
                 let mut res = self
                     .in_memory_tree
@@ -372,12 +372,15 @@ pub(crate) struct TreeQueryResult<AF: AddressFamily> {
     pub more_specifics: Option<Vec<PrefixId<AF>>>,
 }
 
+pub(crate) type FamilyRecord<AF, M> =
+    Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>;
+
 pub(crate) struct FamilyQueryResult<AF: AddressFamily, M: Meta> {
     pub match_type: MatchType,
     pub prefix: Option<PrefixId<AF>>,
     pub prefix_meta: Vec<PublicRecord<M>>,
-    pub less_specifics: Option<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>>,
-    pub more_specifics: Option<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>>,
+    pub less_specifics: Option<FamilyRecord<AF, M>>,
+    pub more_specifics: Option<FamilyRecord<AF, M>>,
 }
 
 impl<AF: AddressFamily, M: Meta> From<FamilyQueryResult<AF, M>>
diff --git a/src/local_array/rib/mod.rs b/src/local_array/rib/mod.rs
index 3f8d12eb..3c39e96a 100644
--- a/src/local_array/rib/mod.rs
+++ b/src/local_array/rib/mod.rs
@@ -1,3 +1,4 @@
+#[allow(clippy::module_inception)]
 pub mod rib;
 
 pub(crate) mod default_store;
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index ea47d1c2..c9b50714 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -16,9 +16,7 @@ use crate::{
 };
 
 use crate::local_array::in_memory::atomic_types::NodeBuckets;
-use crate::local_array::in_memory::atomic_types::{
-    PersistStatus, PrefixBuckets,
-};
+use crate::local_array::in_memory::atomic_types::PrefixBuckets;
 
 // Make sure to also import the other methods for the Rib, so the proc macro
 // create_store can use them.
@@ -317,7 +315,6 @@ impl<
                         .upsert_prefix(
                             prefix,
                             record,
-                            PersistStatus::persisted(),
                             update_path_selections,
                             guard,
                         )
@@ -328,13 +325,7 @@ impl<
             }
             PersistStrategy::PersistHistory => self
                 .in_memory_tree
-                .upsert_prefix(
-                    prefix,
-                    record,
-                    PersistStatus::not_persisted(),
-                    update_path_selections,
-                    guard,
-                )
+                .upsert_prefix(prefix, record, update_path_selections, guard)
                 .map(|(report, old_rec)| {
                     if let Some(rec) = old_rec {
                         if let Some(persist_tree) = &self.persist_tree {
@@ -348,13 +339,7 @@ impl<
                 }),
             PersistStrategy::MemoryOnly => self
                 .in_memory_tree
-                .upsert_prefix(
-                    prefix,
-                    record,
-                    PersistStatus::not_persisted(),
-                    update_path_selections,
-                    guard,
-                )
+                .upsert_prefix(prefix, record, update_path_selections, guard)
                 .map(|(report, _)| report),
             PersistStrategy::PersistOnly => {
                 if let Some(persist_tree) = &self.persist_tree {
diff --git a/src/local_array/types.rs b/src/local_array/types.rs
index 5b93eccf..fa38e509 100644
--- a/src/local_array/types.rs
+++ b/src/local_array/types.rs
@@ -1,4 +1,4 @@
-use crate::{AddressFamily, Meta};
+use crate::AddressFamily;
 
 use super::errors::PrefixStoreError;
 
@@ -102,9 +102,9 @@ impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
     }
 }
 
-impl<AF: AddressFamily> Into<inetnum::addr::Prefix> for PrefixId<AF> {
-    fn into(self) -> inetnum::addr::Prefix {
-        self.into_pub()
+impl<AF: AddressFamily> From<PrefixId<AF>> for inetnum::addr::Prefix {
+    fn from(value: PrefixId<AF>) -> Self {
+        value.into_pub()
     }
 }
 

From c447cb8c8bd059bfe6bbecd8f5dcdaf4037f513f Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 7 Jan 2025 17:50:48 +0100
Subject: [PATCH 042/147] fix full scan iterators for multiple persist
 scenarios

---
 proc_macros/src/lib.rs                 |   9 +-
 src/local_array/in_memory/iterators.rs |   2 +-
 src/local_array/in_memory/query.rs     |   3 +-
 src/local_array/persist/lsm_tree.rs    | 197 ++++++++++++++++++++++---
 src/local_array/query.rs               |  26 ++--
 src/local_array/rib/rib.rs             |  85 ++++++++++-
 tests/concurrency.rs                   |  60 ++++++--
 7 files changed, 327 insertions(+), 55 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index da3e001a..8e5c7d85 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -1267,10 +1267,10 @@ pub fn create_store(
             pub fn prefixes_iter(
                 &'a self,
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v4.in_memory_tree.prefixes_iter()
+                self.v4.prefixes_iter()
                     .map(|p| PrefixRecord::from(p))
                     .chain(
-                        self.v6.in_memory_tree.prefixes_iter()
+                        self.v6.prefixes_iter()
                         .map(|p| PrefixRecord::from(p))
                     )
             }
@@ -1376,7 +1376,8 @@ pub fn create_store(
             pub fn mark_mui_as_withdrawn_for_prefix(
                 &self,
                 prefix: &Prefix,
-                mui: u32
+                mui: u32,
+                ltime: u64
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
                 match prefix.addr() {
@@ -1384,12 +1385,14 @@ pub fn create_store(
                         self.v4.mark_mui_as_withdrawn_for_prefix(
                             PrefixId::<IPv4>::from(*prefix),
                             mui,
+                            ltime
                         )
                     }
                     std::net::IpAddr::V6(addr) => {
                         self.v6.mark_mui_as_withdrawn_for_prefix(
                             PrefixId::<IPv6>::from(*prefix),
                             mui,
+                            ltime
                         )
                     }
                 }
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 80204104..63968f5e 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -885,7 +885,7 @@ impl<
         .flatten()
     }
 
-    // Iterator over all the prefixes in the storage.
+    // Iterator over all the prefixes in the in_memory store.
     pub fn prefixes_iter(
         &'a self,
     ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index 4196a20d..5e9e03f5 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -700,8 +700,7 @@ where
         // any of the match_types (as specified by the user, not the return
         // type) may end up here.
 
-        let match_type = MatchType::EmptyMatch;
-        if let Some(prefix) = match_prefix_idx {
+        let match_type = if let Some(prefix) = match_prefix_idx {
             if prefix.get_len() == search_pfx.get_len() {
                 MatchType::ExactMatch
             } else {
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 4ed84aa5..63aa1651 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -3,7 +3,9 @@
 use std::marker::PhantomData;
 use std::path::Path;
 
-use lsm_tree::AbstractTree;
+use inetnum::addr::Prefix;
+use log::trace;
+use lsm_tree::{AbstractTree, KvPair};
 
 use crate::local_array::types::{PrefixId, RouteStatus};
 use crate::rib::query::{FamilyQueryResult, FamilyRecord, TreeQueryResult};
@@ -44,27 +46,74 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         self.tree.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
     }
 
+    pub fn remove(&self, key: [u8; KEY_SIZE]) {
+        self.tree.remove(key, 0);
+        // the last byte of the prefix holds the length of the prefix.
+        self.counters.dec_prefixes_count(key[PREFIX_SIZE]);
+    }
+
     pub fn get_records_for_prefix<M: Meta>(
         &self,
         prefix: PrefixId<AF>,
         mui: Option<u32>,
     ) -> Vec<PublicRecord<M>> {
-        let prefix_b = if let Some(mui) = mui {
-            &Self::prefix_mui_persistence_key(prefix, mui)
+        if let Some(mui) = mui {
+            let prefix_b = Self::prefix_mui_persistence_key(prefix, mui);
+
+            (*self.tree.prefix(prefix_b))
+                .into_iter()
+                .map(|kv| {
+                    let kv = kv.unwrap();
+                    let (_, mui, ltime, status) =
+                        Self::parse_key(kv.0.as_ref());
+                    PublicRecord::new(
+                        mui,
+                        ltime,
+                        status.try_into().unwrap(),
+                        kv.1.as_ref().to_vec().into(),
+                    )
+                })
+                .collect::<Vec<_>>()
         } else {
-            &prefix.as_bytes::<PREFIX_SIZE>()
-        };
+            let prefix_b = &prefix.as_bytes::<PREFIX_SIZE>();
+
+            (*self.tree.prefix(prefix_b))
+                .into_iter()
+                .map(|kv| {
+                    let kv = kv.unwrap();
+                    let (_, mui, ltime, status) =
+                        Self::parse_key(kv.0.as_ref());
+                    PublicRecord::new(
+                        mui,
+                        ltime,
+                        status.try_into().unwrap(),
+                        kv.1.as_ref().to_vec().into(),
+                    )
+                })
+                .collect::<Vec<_>>()
+        }
+    }
+
+    pub(crate) fn get_records_with_keys_for_prefix_mui<M: Meta>(
+        &self,
+        prefix: PrefixId<AF>,
+        mui: u32,
+    ) -> Vec<(Vec<u8>, PublicRecord<M>)> {
+        let key_b = Self::prefix_mui_persistence_key(prefix, mui);
 
-        (*self.tree.prefix(prefix_b))
+        (*self.tree.prefix(key_b))
             .into_iter()
             .map(|kv| {
                 let kv = kv.unwrap();
                 let (_, mui, ltime, status) = Self::parse_key(kv.0.as_ref());
-                PublicRecord::new(
-                    mui,
-                    ltime,
-                    status.try_into().unwrap(),
-                    kv.1.as_ref().to_vec().into(),
+                (
+                    kv.0.to_vec(),
+                    PublicRecord::new(
+                        mui,
+                        ltime,
+                        status.try_into().unwrap(),
+                        kv.1.as_ref().to_vec().into(),
+                    ),
                 )
             })
             .collect::<Vec<_>>()
@@ -104,7 +153,8 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                 let prefix_b = if let Some(mui) = mui {
                     &Self::prefix_mui_persistence_key(pfx, mui)
                 } else {
-                    &pfx.as_bytes::<PREFIX_SIZE>()
+                    trace!("prefix only");
+                    &pfx.as_bytes::<PREFIX_SIZE>().to_vec()
                 };
                 (
                     prefix,
@@ -140,7 +190,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     let prefix_b = if let Some(mui) = mui {
                         &Self::prefix_mui_persistence_key(*pfx, mui)
                     } else {
-                        &pfx.as_bytes::<PREFIX_SIZE>()
+                        &pfx.as_bytes::<PREFIX_SIZE>().to_vec()
                     };
                     (
                         *pfx,
@@ -245,10 +295,8 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             // Only the most recent record of the search prefix is returned
             // with the prefixes. This is used for the PersistOnly strategy.
             IncludeHistory::None => {
-                let (prefix, prefix_meta) = self.enrich_prefix_most_recent(
-                    search_pfxs.prefix,
-                    options.mui,
-                );
+                let (prefix, prefix_meta) =
+                    self.enrich_prefix(search_pfxs.prefix, options.mui);
 
                 FamilyQueryResult {
                     prefix,
@@ -334,8 +382,8 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     pub fn prefix_mui_persistence_key(
         prefix_id: PrefixId<AF>,
         mui: u32,
-    ) -> [u8; PREFIX_SIZE] {
-        let mut key = [0; PREFIX_SIZE];
+    ) -> Vec<u8> {
+        let mut key = vec![0; PREFIX_SIZE + 4];
         // prefix 5 or 17 bytes
         *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
 
@@ -391,6 +439,17 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             record.meta.as_ref(),
         );
     }
+
+    pub(crate) fn prefixes_iter<'a, M: Meta + 'a>(
+        &'a self,
+    ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
+        PersistedPrefixIter::<AF, M, PREFIX_SIZE, KEY_SIZE> {
+            tree_iter: self.tree.iter(),
+            cur_rec: None,
+            _af: PhantomData,
+            _m: PhantomData,
+        }
+    }
 }
 
 impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
@@ -400,3 +459,103 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         todo!()
     }
 }
+
+pub(crate) struct PersistedPrefixIter<
+    AF: AddressFamily,
+    M: Meta,
+    const PREFIX_SIZE: usize,
+    const KEY_SIZE: usize,
+> {
+    cur_rec: Option<([u8; PREFIX_SIZE], Vec<PublicRecord<M>>)>,
+    tree_iter:
+        Box<dyn DoubleEndedIterator<Item = Result<KvPair, lsm_tree::Error>>>,
+    _af: PhantomData<AF>,
+    _m: PhantomData<M>,
+}
+
+impl<
+        AF: AddressFamily,
+        M: Meta,
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > Iterator for PersistedPrefixIter<AF, M, PREFIX_SIZE, KEY_SIZE>
+{
+    type Item = (Prefix, Vec<PublicRecord<M>>);
+    fn next(&mut self) -> Option<Self::Item> {
+        let rec;
+
+        // Do we already have a record in our iter struct?
+        if let Some(_cur_rec) = &mut self.cur_rec {
+            rec = std::mem::take(&mut self.cur_rec);
+        } else {
+            // No, advance to the next record in the persist tree.
+            let next_rec = self.tree_iter.next();
+
+            match next_rec {
+                // The persist tree is completely done, iterator's done.
+                None => {
+                    return None;
+                }
+                Some(Ok((k, v))) => {
+                    let p_k =
+                        PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::parse_key(
+                            k.as_ref(),
+                        );
+                    rec = Some((
+                        p_k.0,
+                        vec![PublicRecord::<M> {
+                            multi_uniq_id: p_k.1,
+                            ltime: p_k.2,
+                            status: p_k.3.try_into().unwrap(),
+                            meta: v.to_vec().into(),
+                        }],
+                    ));
+                }
+                Some(Err(_)) => {
+                    // This is NOT GOOD. Both that it happens, and that we are
+                    // silently ignoring it.
+                    self.cur_rec = None;
+                    return None;
+                }
+            }
+        };
+
+        if let Some(mut r_rec) = rec {
+            for (k, v) in self.tree_iter.by_ref().flatten() {
+                let (pfx, mui, ltime, status) =
+                    PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::parse_key(
+                        k.as_ref(),
+                    );
+
+                if pfx == r_rec.0 {
+                    r_rec.1.push(PublicRecord {
+                        meta: v.to_vec().into(),
+                        multi_uniq_id: mui,
+                        ltime,
+                        status: status.try_into().unwrap(),
+                    });
+                } else {
+                    self.cur_rec = Some((
+                        pfx,
+                        vec![PublicRecord {
+                            meta: v.to_vec().into(),
+                            multi_uniq_id: mui,
+                            ltime,
+                            status: status.try_into().unwrap(),
+                        }],
+                    ));
+                    break;
+                }
+            }
+
+            Some((
+                Prefix::from(PrefixId::<AF>::from(
+                    *r_rec.0.first_chunk::<PREFIX_SIZE>().unwrap(),
+                )),
+                r_rec.1,
+            ))
+        } else {
+            None
+        }
+    }
+}
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index b2e752a8..784f6b6f 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -164,15 +164,22 @@ where
                 }
 
                 if let Some(persist_tree) = &self.persist_tree {
-                    persist_tree
-                        .match_prefix(
-                            self.in_memory_tree
-                                .match_prefix_by_tree_traversal(
-                                    search_pfx, options,
-                                ),
-                            options,
-                        )
-                        .into()
+                    let exists = self
+                        .in_memory_tree
+                        .match_prefix_by_tree_traversal(search_pfx, options);
+                    println!("IN_MEM {:#?}", exists);
+                    println!(
+                        "{}",
+                        self.in_memory_tree // .prefixes_iter()
+                                            // .collect::<Vec<_>>()
+                    );
+                    println!(
+                        "{:#?}",
+                        self.in_memory_tree
+                            .prefixes_iter()
+                            .collect::<Vec<_>>()
+                    );
+                    persist_tree.match_prefix(exists, options).into()
                 } else {
                     QueryResult::empty()
                 }
@@ -365,6 +372,7 @@ where
     }
 }
 
+#[derive(Debug)]
 pub(crate) struct TreeQueryResult<AF: AddressFamily> {
     pub match_type: MatchType,
     pub prefix: Option<PrefixId<AF>>,
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index c9b50714..a05c54de 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -10,6 +10,7 @@ use epoch::{Guard, Owned};
 
 use crate::local_array::in_memory::tree::TreeBitMap;
 use crate::local_array::types::PrefixId;
+use crate::prelude::multi::RouteStatus;
 use crate::stats::CreatedNodes;
 use crate::{
     local_array::errors::PrefixStoreError, prefix_record::PublicRecord,
@@ -90,6 +91,10 @@ impl Counters {
         self.prefixes[len as usize].fetch_add(1, Ordering::Relaxed);
     }
 
+    pub fn dec_prefixes_count(&self, len: u8) {
+        self.prefixes[len as usize].fetch_sub(1, Ordering::Relaxed);
+    }
+
     pub fn get_prefix_stats(&self) -> Vec<CreatedNodes> {
         self.prefixes
             .iter()
@@ -378,17 +383,60 @@ impl<
         &self,
         prefix: PrefixId<AF>,
         mui: u32,
+        ltime: u64,
     ) -> Result<(), PrefixStoreError> {
-        let (stored_prefix, exists) = self
-            .in_memory_tree
-            .non_recursive_retrieve_prefix_mut(prefix);
-
-        if !exists {
-            return Err(PrefixStoreError::StoreNotReadyError);
+        match self.persist_strategy() {
+            PersistStrategy::WriteAhead | PersistStrategy::MemoryOnly => {
+                let (stored_prefix, exists) = self
+                    .in_memory_tree
+                    .non_recursive_retrieve_prefix_mut(prefix);
+
+                if !exists {
+                    return Err(PrefixStoreError::StoreNotReadyError);
+                }
+                stored_prefix.record_map.mark_as_withdrawn_for_mui(mui);
+            }
+            PersistStrategy::PersistOnly => {
+                let p_tree = self.persist_tree.as_ref().unwrap();
+                let stored_prefixes = p_tree
+                    .get_records_with_keys_for_prefix_mui::<M>(prefix, mui);
+
+                for s in stored_prefixes {
+                    let key: [u8; KEY_SIZE] = PersistTree::<
+                        AF,
+                        PREFIX_SIZE,
+                        KEY_SIZE,
+                    >::persistence_key(
+                        prefix,
+                        mui,
+                        ltime,
+                        RouteStatus::Withdrawn,
+                    );
+                    p_tree.insert(key, s.1.meta.as_ref());
+                    p_tree.remove(*s.0.first_chunk::<KEY_SIZE>().unwrap());
+                }
+            }
+            PersistStrategy::PersistHistory => {
+                let p_tree = self.persist_tree.as_ref().unwrap();
+                let stored_prefixes =
+                    p_tree.get_records_for_prefix::<M>(prefix, Some(mui));
+
+                for s in stored_prefixes {
+                    let key: [u8; KEY_SIZE] = PersistTree::<
+                        AF,
+                        PREFIX_SIZE,
+                        KEY_SIZE,
+                    >::persistence_key(
+                        prefix,
+                        mui,
+                        ltime,
+                        RouteStatus::Withdrawn,
+                    );
+                    p_tree.insert(key, s.meta.as_ref());
+                }
+            }
         }
 
-        stored_prefix.record_map.mark_as_withdrawn_for_mui(mui);
-
         Ok(())
     }
 
@@ -530,6 +578,18 @@ impl<
         }
     }
 
+    pub fn prefixes_iter(
+        &self,
+    ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + '_ {
+        let pt_iter = self
+            .persist_tree
+            .as_ref()
+            .map(|t| t.prefixes_iter())
+            .into_iter()
+            .flatten();
+        self.in_memory_tree.prefixes_iter().chain(pt_iter)
+    }
+
     //-------- Persistence ---------------------------------------------------
 
     pub fn persist_strategy(&self) -> PersistStrategy {
@@ -599,3 +659,12 @@ impl<
         write!(f, "Rib<IPv6, {}>", std::any::type_name::<M>())
     }
 }
+
+// pub(crate) trait StorePrefixIter<M: Meta> {
+//     fn prefixes_iter<'a>(
+//         &'a self,
+//         // iter: Option<impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)>>,
+//     ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a
+//     where
+//         M: 'a;
+// }
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index 8d760745..b0f1fa79 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -2,8 +2,8 @@ use std::{str::FromStr, sync::atomic::Ordering};
 
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
-    prelude::multi::RouteStatus, test_types::BeBytesAsn, IncludeHistory,
-    MatchOptions, MultiThreadedStore, PublicRecord as Record,
+    prelude::multi::RouteStatus, rib::StoreConfig, test_types::BeBytesAsn,
+    IncludeHistory, MatchOptions, MultiThreadedStore, PublicRecord as Record,
 };
 
 mod common {
@@ -18,7 +18,29 @@ mod common {
 }
 
 #[test]
-fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
+fn test_concurrent_updates_1_multiple_persist_scenarios(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let tree_bitmap = MultiThreadedStore::<BeBytesAsn>::try_default()?;
+
+    test_concurrent_updates_1(std::sync::Arc::new(tree_bitmap))?;
+
+    let store_config = StoreConfig {
+        persist_strategy: rotonda_store::rib::PersistStrategy::PersistOnly,
+        persist_path: "/tmp/rotonda/".into(),
+    };
+
+    let tree_bitmap = std::sync::Arc::new(
+        MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
+    );
+
+    test_concurrent_updates_1(tree_bitmap)?;
+
+    Ok(())
+}
+
+fn test_concurrent_updates_1(
+    store: std::sync::Arc<MultiThreadedStore<BeBytesAsn>>,
+) -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
     let pfx_vec_1 = vec![
@@ -48,8 +70,14 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
         pfxs: Vec<Prefix>,
     }
 
-    let tree_bitmap =
-        std::sync::Arc::new(MultiThreadedStore::<BeBytesAsn>::try_default()?);
+    // let store_config = StoreConfig {
+    //     persist_strategy: rotonda_store::rib::PersistStrategy::PersistOnly,
+    //     persist_path: "/tmp/rotonda/".into(),
+    // };
+
+    // let store = std::sync::Arc::new(
+    //     MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
+    // );
 
     let mui_data_1 = MuiData {
         mui: 1,
@@ -74,7 +102,7 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
     let _: Vec<_> = vec![mui_data_1, mui_data_2, mui_data_3]
         .into_iter()
         .map(|data: MuiData| {
-            let tree_bitmap = tree_bitmap.clone();
+            let tree_bitmap = store.clone();
             let cur_ltime = cur_ltime.clone();
 
             std::thread::Builder::new()
@@ -108,11 +136,16 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
         .map(|t| t.join())
         .collect();
 
-    println!("{:#?}", tree_bitmap.prefixes_iter().collect::<Vec<_>>());
+    println!("COUNT {}", store.prefixes_count());
 
-    let all_pfxs_iter = tree_bitmap.prefixes_iter().collect::<Vec<_>>();
+    let all_pfxs_iter = store.prefixes_iter().collect::<Vec<_>>();
 
     let pfx = Prefix::from_str("185.34.0.0/16").unwrap();
+
+    assert!(store.contains(&pfx, None));
+    assert!(store.contains(&pfx, Some(1)));
+    assert!(store.contains(&pfx, Some(2)));
+
     assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
         .iter()
@@ -303,7 +336,7 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
         .clone()
         .into_iter()
         .map(|pfx: Prefix| {
-            let tree_bitmap = tree_bitmap.clone();
+            let tree_bitmap = store.clone();
             let cur_ltime = cur_ltime.clone();
 
             std::thread::Builder::new()
@@ -313,7 +346,7 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
 
                     let _ = cur_ltime.fetch_add(1, Ordering::Release);
                     tree_bitmap
-                        .mark_mui_as_withdrawn_for_prefix(&pfx, 2)
+                        .mark_mui_as_withdrawn_for_prefix(&pfx, 2, 10)
                         .unwrap();
 
                     println!("--thread withdraw 2 done.");
@@ -323,7 +356,7 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
         .map(|t| t.join())
         .collect();
 
-    println!("{:#?}", tree_bitmap.prefixes_iter().collect::<Vec<_>>());
+    println!("{:#?}", store.prefixes_iter().collect::<Vec<_>>());
 
     let match_options = MatchOptions {
         match_type: rotonda_store::MatchType::ExactMatch,
@@ -336,8 +369,9 @@ fn test_concurrent_updates_1() -> Result<(), Box<dyn std::error::Error>> {
 
     for pfx in pfx_vec_2 {
         let guard = rotonda_store::epoch::pin();
-        let res = tree_bitmap.match_prefix(&pfx, &match_options, &guard);
+        let res = store.match_prefix(&pfx, &match_options, &guard);
         assert_eq!(res.prefix, Some(pfx));
+        println!("PFX {:?}", res);
         assert_eq!(
             res.prefix_meta
                 .iter()
@@ -582,7 +616,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
 
                     let _ = cur_ltime.fetch_add(1, Ordering::Release);
                     tree_bitmap
-                        .mark_mui_as_withdrawn_for_prefix(&pfx, 2)
+                        .mark_mui_as_withdrawn_for_prefix(&pfx, 2, 11)
                         .unwrap();
 
                     println!("--thread withdraw 2 done.");

From b3b8f4080fa6ffb1fa4c76cf4244b86bca085144 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 8 Jan 2025 18:04:25 +0100
Subject: [PATCH 043/147] fix iterators some more

---
 proc_macros/src/lib.rs                    |  34 +++--
 src/local_array/in_memory/atomic_types.rs |  90 ++----------
 src/local_array/in_memory/iterators.rs    |   6 -
 src/local_array/persist/lsm_tree.rs       | 132 ++++++++++++------
 src/local_array/query.rs                  |  80 ++++++++---
 src/local_array/rib/rib.rs                | 138 +++++++++++++++----
 src/macros.rs                             |  75 +++++++++-
 src/mod.rs                                |   1 -
 tests/best-path.rs                        |   9 +-
 tests/concurrency.rs                      |  20 +++
 tests/treebitmap.rs                       | 159 +++++++++++++---------
 11 files changed, 488 insertions(+), 256 deletions(-)
 delete mode 100644 src/mod.rs

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 8e5c7d85..ac5e15c5 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -988,8 +988,8 @@ pub fn create_store(
                             (
                                 Some(self
                                     .v4
-                                    .in_memory_tree
-                                    .less_specific_prefix_iter(
+                                    // .in_memory_tree
+                                    .less_specifics_iter_from(
                                         PrefixId::<IPv4>::new(
                                             addr.into(),
                                             search_pfx.len(),
@@ -1008,8 +1008,8 @@ pub fn create_store(
                                 None,
                                 Some(self
                                     .v6
-                                    .in_memory_tree
-                                    .less_specific_prefix_iter(
+                                    // .in_memory_tree
+                                    .less_specifics_iter_from(
                                         PrefixId::<IPv6>::new(
                                             addr.into(),
                                             search_pfx.len(),
@@ -1087,8 +1087,9 @@ pub fn create_store(
                         } else {
                             (Some(self
                                 .v4
-                                .in_memory_tree
-                                .more_specific_prefix_iter_from(
+                                // .in_memory_tree
+                                // .more_specific_prefix_iter_from(
+                                .more_specifics_iter_from(
                                     PrefixId::<IPv4>::new(
                                         addr.into(),
                                         search_pfx.len(),
@@ -1111,8 +1112,8 @@ pub fn create_store(
                                 None,
                                 Some(self
                                     .v6
-                                    .in_memory_tree
-                                    .more_specific_prefix_iter_from(
+                                    // .in_memory_tree
+                                    .more_specifics_iter_from(
                                         PrefixId::<IPv6>::new(
                                             addr.into(),
                                             search_pfx.len(),
@@ -1143,8 +1144,8 @@ pub fn create_store(
                     Some(
                         self
                             .v4
-                            .in_memory_tree
-                            .more_specific_prefix_iter_from(
+                            // .in_memory_tree
+                            .more_specifics_iter_from(
                                 PrefixId::<IPv4>::new(
                                     0,
                                     0,
@@ -1170,8 +1171,8 @@ pub fn create_store(
                 } else {
                     Some(
                         self.v6
-                            .in_memory_tree
-                            .more_specific_prefix_iter_from(
+                            // .in_memory_tree
+                            .more_specifics_iter_from(
                                 PrefixId::<IPv6>::new(
                                     0,
                                     0,
@@ -1318,7 +1319,7 @@ pub fn create_store(
             pub fn prefixes_iter_v4(
                 &'a self,
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v4.in_memory_tree.prefixes_iter()
+                self.v4.prefixes_iter()
                     .map(|p| PrefixRecord::from(p))
             }
 
@@ -1365,7 +1366,7 @@ pub fn create_store(
             pub fn prefixes_iter_v6(
                 &'a self,
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v6.in_memory_tree.prefixes_iter()
+                self.v6.prefixes_iter()
                     .map(|p| PrefixRecord::from(p))
             }
 
@@ -1405,7 +1406,8 @@ pub fn create_store(
             pub fn mark_mui_as_active_for_prefix(
                 &self,
                 prefix: &Prefix,
-                mui: u32
+                mui: u32,
+                ltime: u64
             ) -> Result<(), PrefixStoreError> {
                 let guard = &epoch::pin();
                 match prefix.addr() {
@@ -1413,12 +1415,14 @@ pub fn create_store(
                         self.v4.mark_mui_as_active_for_prefix(
                             PrefixId::<IPv4>::from(*prefix),
                             mui,
+                            ltime
                         )
                     }
                     std::net::IpAddr::V6(addr) => {
                         self.v6.mark_mui_as_active_for_prefix(
                             PrefixId::<IPv6>::from(*prefix),
                             mui,
+                            ltime
                         )
                     }
                 }
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index aba43663..a0e2d865 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -270,6 +270,10 @@ impl<M: Meta> MultiMapValue<M> {
         self.ltime
     }
 
+    pub(crate) fn set_logical_time(&mut self, ltime: u64) {
+        self.ltime = ltime;
+    }
+
     pub(crate) fn meta(&self) -> &M {
         &self.meta
     }
@@ -468,20 +472,22 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     }
 
     // Change the local status of the record for this mui to Withdrawn.
-    pub fn mark_as_withdrawn_for_mui(&self, mui: u32) {
+    pub fn mark_as_withdrawn_for_mui(&self, mui: u32, ltime: u64) {
         let c_map = Arc::clone(&self.0);
         let mut record_map = c_map.lock().unwrap();
         if let Some(rec) = record_map.get_mut(&mui) {
             rec.set_route_status(RouteStatus::Withdrawn);
+            rec.set_logical_time(ltime);
         }
     }
 
     // Change the local status of the record for this mui to Active.
-    pub fn mark_as_active_for_mui(&self, mui: u32) {
+    pub fn mark_as_active_for_mui(&self, mui: u32, ltime: u64) {
         let record_map = Arc::clone(&self.0);
         let mut r_map = record_map.lock().unwrap();
         if let Some(rec) = r_map.get_mut(&mui) {
             rec.set_route_status(RouteStatus::Active);
+            rec.set_logical_time(ltime);
         }
     }
 
@@ -512,86 +518,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
                 Ok((None, retry_count))
             }
         }
-
-        // Ok((record_map.len(), retry_count))
-
-        // match (strategy, record_map.get_mut(&key)) {
-        //     // New record for (prefix, mui) in memory.
-
-        //     // We store the record in memory only.
-        //     (
-        //         PersistStrategy::PersistHistory | PersistStrategy::MemoryOnly,
-        //         None,
-        //     ) => {
-        //         record_map.insert(
-        //             key,
-        //             MultiMapValue::from((
-        //                 new_rec,
-        //                 PersistStatus::not_persisted(),
-        //             )),
-        //         );
-
-        //         Ok((None, retry_count))
-        //     }
-        //     // We only persist the record.
-        //     (PersistStrategy::PersistOnly, None) => {
-        //         if let Some(persistence) = persistence {
-        //             persistence.persist_record(prefix, key, &new_rec);
-        //             Ok((None, retry_count))
-        //         } else {
-        //             Err(PrefixStoreError::PersistFailed)
-        //         }
-        //     }
-        //     // We store both in memory and persist it.
-        //     (PersistStrategy::WriteAhead, None) => {
-        //         if let Some(persistence) = persistence {
-        //             persistence.persist_record(prefix, key, &new_rec);
-        //             let mmv = MultiMapValue::from((
-        //                 new_rec,
-        //                 PersistStatus::persisted(),
-        //             ));
-        //             record_map.insert(key, mmv);
-
-        //             Ok((None, retry_count))
-        //         } else {
-        //             Err(PrefixStoreError::PersistFailed)
-        //         }
-        //     }
-
-        //     // Existing record for (prefix, mui) in memory.
-
-        //     // We store the record in memory only, and discard the old record.
-        //     (PersistStrategy::MemoryOnly, Some(exist_rec)) => {
-        //         *exist_rec = MultiMapValue::from((
-        //             new_rec,
-        //             PersistStatus::not_persisted(),
-        //         ));
-
-        //         Ok((Some(record_map.len()), retry_count))
-        //     }
-        //     // We only persist record, so how come there's one in memory?
-        //     // Should not happen.
-        //     (PersistStrategy::PersistOnly, Some(_)) => {
-        //         panic!("Encountered illegally stored record");
-        //     }
-        //     // We store the new record in memory and persist the old record.
-        //     (
-        //         PersistStrategy::PersistHistory | PersistStrategy::WriteAhead,
-        //         Some(exist_rec),
-        //     ) => {
-        //         if let Some(persistence) = persistence {
-        //             persistence.persist_record(prefix, key, &new_rec);
-        //             *exist_rec = MultiMapValue::from((
-        //                 new_rec,
-        //                 PersistStatus::persisted(),
-        //             ));
-
-        //             Ok((Some(record_map.len()), retry_count))
-        //         } else {
-        //             Err(PrefixStoreError::PersistFailed)
-        //         }
-        //     }
-        // }
     }
 }
 
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 63968f5e..a00eba45 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -302,8 +302,6 @@ pub(crate) struct MoreSpecificPrefixIter<
     M: Meta,
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
-    // const PREFIX_SIZE: usize,
-    // const KEY_SIZE: usize,
 > {
     store: &'a TreeBitMap<AF, M, NB, PB>,
     cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
@@ -326,8 +324,6 @@ impl<
         M: Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-        // const PREFIX_SIZE: usize,
-        // const KEY_SIZE: usize,
     > Iterator for MoreSpecificPrefixIter<'a, AF, M, NB, PB>
 {
     type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
@@ -727,8 +723,6 @@ impl<
         M: crate::prefix_record::Meta,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-        // const PREFIX_SIZE: usize,
-        // const KEY_SIZE: usize,
     > TreeBitMap<AF, M, NB, PB>
 {
     // Iterator over all more-specific prefixes, starting from the given
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 63aa1651..f980fa8e 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -6,6 +6,7 @@ use std::path::Path;
 use inetnum::addr::Prefix;
 use log::trace;
 use lsm_tree::{AbstractTree, KvPair};
+use roaring::RoaringBitmap;
 
 use crate::local_array::types::{PrefixId, RouteStatus};
 use crate::rib::query::{FamilyQueryResult, FamilyRecord, TreeQueryResult};
@@ -42,7 +43,11 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         }
     }
 
-    pub fn insert(&self, key: [u8; KEY_SIZE], value: &[u8]) -> (u32, u32) {
+    pub(crate) fn insert(
+        &self,
+        key: [u8; KEY_SIZE],
+        value: &[u8],
+    ) -> (u32, u32) {
         self.tree.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
     }
 
@@ -94,6 +99,30 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         }
     }
 
+    pub fn get_most_recent_record_for_prefix_mui<M: Meta>(
+        &self,
+        prefix: PrefixId<AF>,
+        mui: u32,
+    ) -> Option<PublicRecord<M>> {
+        let key_b = Self::prefix_mui_persistence_key(prefix, mui);
+
+        (*self.tree.prefix(key_b))
+            .into_iter()
+            .map(|kv| {
+                let kv = kv.unwrap();
+                (Self::parse_key(kv.0.as_ref()), kv.1)
+            })
+            .max_by(|(a, _), (b, _)| a.2.cmp(&b.2))
+            .map(|((_pfx, mui, ltime, status), m)| {
+                PublicRecord::new(
+                    mui,
+                    ltime,
+                    status.try_into().unwrap(),
+                    m.as_ref().to_vec().into(),
+                )
+            })
+    }
+
     pub(crate) fn get_records_with_keys_for_prefix_mui<M: Meta>(
         &self,
         prefix: PrefixId<AF>,
@@ -143,42 +172,6 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             .collect::<Vec<_>>()
     }
 
-    fn enrich_prefix_most_recent<M: Meta>(
-        &self,
-        prefix: Option<PrefixId<AF>>,
-        mui: Option<u32>,
-    ) -> (Option<PrefixId<AF>>, Vec<PublicRecord<M>>) {
-        match prefix {
-            Some(pfx) => {
-                let prefix_b = if let Some(mui) = mui {
-                    &Self::prefix_mui_persistence_key(pfx, mui)
-                } else {
-                    trace!("prefix only");
-                    &pfx.as_bytes::<PREFIX_SIZE>().to_vec()
-                };
-                (
-                    prefix,
-                    (*self.tree.prefix(prefix_b))
-                        .into_iter()
-                        .last()
-                        .map(|kv| {
-                            let kv = kv.unwrap();
-                            let (_, mui, ltime, status) =
-                                Self::parse_key(kv.0.as_ref());
-                            vec![PublicRecord::new(
-                                mui,
-                                ltime,
-                                status.try_into().unwrap(),
-                                kv.1.as_ref().to_vec().into(),
-                            )]
-                        })
-                        .unwrap_or_default(),
-                )
-            }
-            None => (None, vec![]),
-        }
-    }
-
     fn enrich_prefixes_most_recent<M: Meta>(
         &self,
         prefixes: Option<Vec<PrefixId<AF>>>,
@@ -219,9 +212,25 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         prefix: Option<PrefixId<AF>>,
         mui: Option<u32>,
+        include_withdrawn: bool,
+        bmin: &RoaringBitmap,
     ) -> (Option<PrefixId<AF>>, Vec<PublicRecord<M>>) {
         match prefix {
-            Some(pfx) => (Some(pfx), self.get_records_for_prefix(pfx, mui)),
+            Some(pfx) => (
+                Some(pfx),
+                self.get_records_for_prefix(pfx, mui)
+                    .into_iter()
+                    .filter_map(|mut r| {
+                        if bmin.contains(r.multi_uniq_id) {
+                            if !include_withdrawn {
+                                return None;
+                            }
+                            r.status = RouteStatus::Withdrawn;
+                        }
+                        Some(r)
+                    })
+                    .collect(),
+            ),
             None => (None, vec![]),
         }
     }
@@ -230,11 +239,27 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         prefixes: Option<Vec<PrefixId<AF>>>,
         mui: Option<u32>,
+        include_withdrawn: bool,
+        bmin: &RoaringBitmap,
     ) -> Option<FamilyRecord<AF, M>> {
         prefixes.map(|recs| {
             recs.iter()
                 .flat_map(move |pfx| {
-                    Some((*pfx, self.get_records_for_prefix(*pfx, mui)))
+                    Some((
+                        *pfx,
+                        self.get_records_for_prefix(*pfx, mui)
+                            .into_iter()
+                            .filter_map(|mut r| {
+                                if bmin.contains(r.multi_uniq_id) {
+                                    if !include_withdrawn {
+                                        return None;
+                                    }
+                                    r.status = RouteStatus::Withdrawn;
+                                }
+                                Some(r)
+                            })
+                            .collect(),
+                    ))
                 })
                 .collect()
         })
@@ -253,12 +278,17 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         search_pfxs: TreeQueryResult<AF>,
         options: &MatchOptions,
+        bmin: &RoaringBitmap,
     ) -> FamilyQueryResult<AF, M> {
         match options.include_history {
             // All the records for all the prefixes
             IncludeHistory::All => {
-                let (prefix, prefix_meta) =
-                    self.enrich_prefix(search_pfxs.prefix, options.mui);
+                let (prefix, prefix_meta) = self.enrich_prefix(
+                    search_pfxs.prefix,
+                    options.mui,
+                    options.include_withdrawn,
+                    bmin,
+                );
 
                 FamilyQueryResult {
                     prefix,
@@ -267,10 +297,14 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     less_specifics: self.enrich_prefixes(
                         search_pfxs.less_specifics,
                         options.mui,
+                        options.include_withdrawn,
+                        bmin,
                     ),
                     more_specifics: self.enrich_prefixes(
                         search_pfxs.more_specifics,
                         options.mui,
+                        options.include_withdrawn,
+                        bmin,
                     ),
                 }
             }
@@ -279,8 +313,12 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             // attached. Not useful with the MemoryOnly strategy (historical
             // records are neve kept in memory).
             IncludeHistory::SearchPrefix => {
-                let (prefix, prefix_meta) =
-                    self.enrich_prefix(search_pfxs.prefix, options.mui);
+                let (prefix, prefix_meta) = self.enrich_prefix(
+                    search_pfxs.prefix,
+                    options.mui,
+                    options.include_withdrawn,
+                    bmin,
+                );
 
                 FamilyQueryResult {
                     prefix,
@@ -295,8 +333,12 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             // Only the most recent record of the search prefix is returned
             // with the prefixes. This is used for the PersistOnly strategy.
             IncludeHistory::None => {
-                let (prefix, prefix_meta) =
-                    self.enrich_prefix(search_pfxs.prefix, options.mui);
+                let (prefix, prefix_meta) = self.enrich_prefix(
+                    search_pfxs.prefix,
+                    options.mui,
+                    options.include_withdrawn,
+                    bmin,
+                );
 
                 FamilyQueryResult {
                     prefix,
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 784f6b6f..80ebac88 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -120,21 +120,51 @@ where
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> Result<
-        impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a,
-        PrefixStoreError,
-    > {
-        let bmin = self.in_memory_tree.withdrawn_muis_bmin(guard);
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        match self.config.persist_strategy() {
+            PersistStrategy::MemoryOnly
+            | PersistStrategy::WriteAhead
+            | PersistStrategy::PersistHistory => {
+                // if mui.is_some_and(|m| self.mui_is_withdrawn(m, guard)) {
+                //     println!("more_specifics_iter_from");
+                //     unimplemented!()
+                // } else {
+                self.in_memory_tree.more_specific_prefix_iter_from(
+                    prefix_id,
+                    mui,
+                    include_withdrawn,
+                    guard,
+                )
+                // }
+            }
+            PersistStrategy::PersistOnly => unimplemented!(),
+        }
+    }
 
-        if mui.is_some() && bmin.contains(mui.unwrap()) {
-            Err(PrefixStoreError::PrefixNotFound)
-        } else {
-            Ok(self.in_memory_tree.more_specific_prefix_iter_from(
-                prefix_id,
-                mui,
-                include_withdrawn,
-                guard,
-            ))
+    pub fn less_specifics_iter_from(
+        &'a self,
+        prefix_id: PrefixId<AF>,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        match self.config.persist_strategy() {
+            PersistStrategy::MemoryOnly
+            | PersistStrategy::WriteAhead
+            | PersistStrategy::PersistHistory => {
+                // if mui.is_some_and(|m| self.mui_is_withdrawn(m, guard)) {
+                //     println!("less_specifics_iter_from");
+                //     unimplemented!()
+                // } else {
+                self.in_memory_tree.less_specific_prefix_iter(
+                    prefix_id,
+                    mui,
+                    include_withdrawn,
+                    guard,
+                )
+                // }
+            }
+            PersistStrategy::PersistOnly => unimplemented!(),
         }
     }
 
@@ -157,6 +187,19 @@ where
             // to memory. However the in-memory-tree is still used to indicate
             // which (prefix, mui) tuples have been created.
             PersistStrategy::PersistOnly => {
+                // If no withdawn should be included and a specific mui was
+                // requested, then return early, if the mui lives in the
+                // global withdrawn muis.
+                if !options.include_withdrawn {
+                    if let Some(mui) = options.mui {
+                        let withdrawn_muis_bmin =
+                            self.in_memory_tree.withdrawn_muis_bmin(guard);
+                        if withdrawn_muis_bmin.contains(mui) {
+                            return QueryResult::empty();
+                        }
+                    }
+                }
+
                 if options.match_type == MatchType::ExactMatch
                     && !self.contains(search_pfx, options.mui)
                 {
@@ -168,18 +211,15 @@ where
                         .in_memory_tree
                         .match_prefix_by_tree_traversal(search_pfx, options);
                     println!("IN_MEM {:#?}", exists);
-                    println!(
-                        "{}",
-                        self.in_memory_tree // .prefixes_iter()
-                                            // .collect::<Vec<_>>()
-                    );
+                    println!("{}", self.in_memory_tree);
                     println!(
                         "{:#?}",
                         self.in_memory_tree
                             .prefixes_iter()
                             .collect::<Vec<_>>()
                     );
-                    persist_tree.match_prefix(exists, options).into()
+                    let bmin = self.in_memory_tree.withdrawn_muis_bmin(guard);
+                    persist_tree.match_prefix(exists, options, bmin).into()
                 } else {
                     QueryResult::empty()
                 }
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index a05c54de..09700ff3 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -392,9 +392,9 @@ impl<
                     .non_recursive_retrieve_prefix_mut(prefix);
 
                 if !exists {
-                    return Err(PrefixStoreError::StoreNotReadyError);
+                    return Err(PrefixStoreError::PrefixNotFound);
                 }
-                stored_prefix.record_map.mark_as_withdrawn_for_mui(mui);
+                stored_prefix.record_map.mark_as_withdrawn_for_mui(mui, ltime);
             }
             PersistStrategy::PersistOnly => {
                 let p_tree = self.persist_tree.as_ref().unwrap();
@@ -413,15 +413,34 @@ impl<
                         RouteStatus::Withdrawn,
                     );
                     p_tree.insert(key, s.1.meta.as_ref());
+
+                    // remove the entry for the same (prefix, mui), but with
+                    // an older logical time
                     p_tree.remove(*s.0.first_chunk::<KEY_SIZE>().unwrap());
                 }
             }
             PersistStrategy::PersistHistory => {
-                let p_tree = self.persist_tree.as_ref().unwrap();
-                let stored_prefixes =
-                    p_tree.get_records_for_prefix::<M>(prefix, Some(mui));
+                // First do the in-memory part
+                let (stored_prefix, exists) = self
+                    .in_memory_tree
+                    .non_recursive_retrieve_prefix_mut(prefix);
+
+                if !exists {
+                    return Err(PrefixStoreError::StoreNotReadyError);
+                }
+                stored_prefix.record_map.mark_as_withdrawn_for_mui(mui, ltime);
+
+                // Use the record from the in-memory RIB to persist.
+                if let Some(_record) =
+                    stored_prefix.record_map.get_record_for_active_mui(mui)
+                {
+                    let p_tree =
+                        if let Some(p_tree) = self.persist_tree.as_ref() {
+                            p_tree
+                        } else {
+                            return Err(PrefixStoreError::StoreNotReadyError);
+                        };
 
-                for s in stored_prefixes {
                     let key: [u8; KEY_SIZE] = PersistTree::<
                         AF,
                         PREFIX_SIZE,
@@ -432,7 +451,11 @@ impl<
                         ltime,
                         RouteStatus::Withdrawn,
                     );
-                    p_tree.insert(key, s.meta.as_ref());
+                    // Here we are keeping persisted history, so no removal of
+                    // old (prefix, mui) records.
+                    // We are inserting an empty record, since this is a
+                    // withdrawal.
+                    p_tree.insert(key, &[]);
                 }
             }
         }
@@ -446,16 +469,94 @@ impl<
         &self,
         prefix: PrefixId<AF>,
         mui: u32,
+        ltime: u64,
     ) -> Result<(), PrefixStoreError> {
-        let (stored_prefix, exists) = self
-            .in_memory_tree
-            .non_recursive_retrieve_prefix_mut(prefix);
+        match self.persist_strategy() {
+            PersistStrategy::WriteAhead | PersistStrategy::MemoryOnly => {
+                let (stored_prefix, exists) = self
+                    .in_memory_tree
+                    .non_recursive_retrieve_prefix_mut(prefix);
 
-        if !exists {
-            return Err(PrefixStoreError::StoreNotReadyError);
-        }
+                if !exists {
+                    return Err(PrefixStoreError::PrefixNotFound);
+                }
+                stored_prefix.record_map.mark_as_active_for_mui(mui, ltime);
+            }
+            PersistStrategy::PersistOnly => {
+                let p_tree = self.persist_tree.as_ref().unwrap();
+                // let stored_prefixes = p_tree
+                //     .get_records_with_keys_for_prefix_mui::<M>(prefix, mui)
+
+                if let Some(record) = p_tree
+                    .get_most_recent_record_for_prefix_mui::<M>(prefix, mui)
+                {
+                    // for s in stored_prefixes {
+                    let new_key: [u8; KEY_SIZE] = PersistTree::<
+                        AF,
+                        PREFIX_SIZE,
+                        KEY_SIZE,
+                    >::persistence_key(
+                        prefix,
+                        mui,
+                        ltime,
+                        RouteStatus::Active,
+                    );
+                    p_tree.insert(new_key, record.meta.as_ref());
 
-        stored_prefix.record_map.mark_as_active_for_mui(mui);
+                    // remove the entry for the same (prefix, mui), but with
+                    // an older logical time
+                    let old_key = PersistTree::<
+                        AF, 
+                        PREFIX_SIZE,
+                        KEY_SIZE>::persistence_key(
+                            prefix, 
+                            mui, 
+                            record.ltime, 
+                            record.status
+                    );
+                    p_tree.remove(old_key);
+                }
+            }
+            PersistStrategy::PersistHistory => {
+                // First do the in-memory part
+                let (stored_prefix, exists) = self
+                    .in_memory_tree
+                    .non_recursive_retrieve_prefix_mut(prefix);
+
+                if !exists {
+                    return Err(PrefixStoreError::StoreNotReadyError);
+                }
+                stored_prefix.record_map.mark_as_active_for_mui(mui, ltime);
+
+                // Use the record from the in-memory RIB to persist.
+                if let Some(_record) =
+                    stored_prefix.record_map.get_record_for_active_mui(mui)
+                {
+                    let p_tree =
+                        if let Some(p_tree) = self.persist_tree.as_ref() {
+                            p_tree
+                        } else {
+                            return Err(PrefixStoreError::StoreNotReadyError);
+                        };
+
+                    let key: [u8; KEY_SIZE] = PersistTree::<
+                        AF,
+                        PREFIX_SIZE,
+                        KEY_SIZE,
+                    >::persistence_key(
+                        prefix,
+                        mui,
+                        ltime,
+                        RouteStatus::Active,
+                    );
+                    // Here we are keeping persisted history, so no removal of
+                    // old (prefix, mui) records.
+                    // We are inserting an empty record, since this is a
+                    // withdrawal.
+                    p_tree.insert(key, &[]);
+                }
+            }
+        }
 
         Ok(())
     }
@@ -659,12 +760,3 @@ impl<
         write!(f, "Rib<IPv6, {}>", std::any::type_name::<M>())
     }
 }
-
-// pub(crate) trait StorePrefixIter<M: Meta> {
-//     fn prefixes_iter<'a>(
-//         &'a self,
-//         // iter: Option<impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)>>,
-//     ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a
-//     where
-//         M: 'a;
-// }
diff --git a/src/macros.rs b/src/macros.rs
index e487e551..d1efc4e9 100644
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -4,7 +4,7 @@
 // implementations for left|right shift, counting ones etc.
 #[doc(hidden)]
 macro_rules! impl_primitive_stride {
-    ( $( $len: expr; $bits: expr; $pfxsize:ty; $ptrsize: ty ), * ) => {
+    ( $( $len: expr; $bits: expr; $pfxsize: ty; $ptrsize: ty ), * ) => {
             $(
                 impl Stride for $pfxsize {
                     type PtrSize = $ptrsize;
@@ -42,3 +42,76 @@ macro_rules! impl_primitive_stride {
             )*
     };
 }
+
+#[macro_export]
+#[doc(hidden)]
+macro_rules! all_strategies {
+    ( $( $fn_name: ident; $test_name: ident; $ty: ty ), * ) => {
+
+        $(
+            #[test]
+            fn $fn_name() -> Result<(), Box<dyn std::error::Error>> {
+                //------- Default (MemoryOnly)
+
+                println!("default strategy starting...");
+                let tree_bitmap =
+                    MultiThreadedStore::<$ty>::try_default()?;
+
+                $test_name(tree_bitmap)?;
+
+                //------- PersistOnly
+
+                println!("PersistOnly strategy starting...");
+                let store_config = StoreConfig {
+                    persist_strategy:
+                        rotonda_store::rib::PersistStrategy::PersistOnly,
+                    persist_path: "/tmp/rotonda/".into(),
+                };
+
+                let tree_bitmap = MultiThreadedStore::<
+                    $ty,
+                >::new_with_config(
+                    store_config
+                )?;
+
+                $test_name(tree_bitmap)?;
+
+                //------- PersistHistory
+
+                println!("PersistHistory strategy starting...");
+                let store_config = StoreConfig {
+                    persist_strategy:
+                        rotonda_store::rib::PersistStrategy::PersistHistory,
+                    persist_path: "/tmp/rotonda/".into(),
+                };
+
+                let tree_bitmap = MultiThreadedStore::<
+                    $ty,
+                >::new_with_config(
+                    store_config
+                )?;
+
+                $test_name(tree_bitmap)?;
+
+                //------- WriteAhead
+
+                println!("WriteAhead strategy starting...");
+                let store_config = StoreConfig {
+                    persist_strategy:
+                        rotonda_store::rib::PersistStrategy::WriteAhead,
+                    persist_path: "/tmp/rotonda/".into(),
+                };
+
+                let tree_bitmap = MultiThreadedStore::<
+                    $ty,
+                >::new_with_config(
+                    store_config
+                )?;
+
+                $test_name(tree_bitmap)?;
+
+                Ok(())
+            }
+        )*
+    };
+}
diff --git a/src/mod.rs b/src/mod.rs
deleted file mode 100644
index 0b47c732..00000000
--- a/src/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod test_types;
diff --git a/tests/best-path.rs b/tests/best-path.rs
index 85b091df..4a99cff7 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -63,8 +63,15 @@ mod common {
     }
 }
 
+// rotonda_store::all_strategies![
+//     best_path;
+//     test_best_path_1;
+//     Ipv4Route
+// ];
+
 #[test]
-fn test_best_path_1() -> Result<(), Box<dyn std::error::Error>> {
+fn test_best_path_1(// tree_bitmap: MultiThreadedStore<Ipv4Route>,
+) -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
     let store_config = StoreConfig {
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index b0f1fa79..f2b7f76e 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -20,10 +20,16 @@ mod common {
 #[test]
 fn test_concurrent_updates_1_multiple_persist_scenarios(
 ) -> Result<(), Box<dyn std::error::Error>> {
+    //------- Default (MemoryOnly)
+
+    println!("default strategy starting...");
     let tree_bitmap = MultiThreadedStore::<BeBytesAsn>::try_default()?;
 
     test_concurrent_updates_1(std::sync::Arc::new(tree_bitmap))?;
 
+    //------- PersistOnly
+
+    println!("PersistOnly strategy starting...");
     let store_config = StoreConfig {
         persist_strategy: rotonda_store::rib::PersistStrategy::PersistOnly,
         persist_path: "/tmp/rotonda/".into(),
@@ -35,6 +41,20 @@ fn test_concurrent_updates_1_multiple_persist_scenarios(
 
     test_concurrent_updates_1(tree_bitmap)?;
 
+    //------- PersistHistory
+
+    println!("PersistHistory strategy starting...");
+    let store_config = StoreConfig {
+        persist_strategy: rotonda_store::rib::PersistStrategy::PersistHistory,
+        persist_path: "/tmp/rotonda/".into(),
+    };
+
+    let tree_bitmap = std::sync::Arc::new(
+        MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
+    );
+
+    test_concurrent_updates_1(tree_bitmap)?;
+
     Ok(())
 }
 
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 6b45222d..02dd4d8a 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -20,9 +20,16 @@ mod tests {
         prelude::*,
     };
 
-    #[test]
-    fn test_insert_extremes_ipv4() -> Result<(), Box<dyn std::error::Error>> {
-        let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
+    rotonda_store::all_strategies![
+        test_treebitmap;
+        test_insert_extremes_ipv4;
+        NoMeta
+    ];
+
+    // #[test]
+    fn test_insert_extremes_ipv4(
+        trie: MultiThreadedStore<NoMeta>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
         let min_pfx = Prefix::new_relaxed(
             std::net::Ipv4Addr::new(0, 0, 0, 0).into(),
             1,
@@ -91,11 +98,18 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn test_tree_ipv4() -> Result<(), Box<dyn std::error::Error>> {
+    rotonda_store::all_strategies![
+        tree_ipv4;
+        test_tree_ipv4;
+        PrefixAs
+    ];
+
+    fn test_tree_ipv4(
+        tree_bitmap: MultiThreadedStore<PrefixAs>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+        // let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             // Prefix::new_relaxed(0b0000_0000_0000_0000_0000_0000_0000_000 0_u32.into_ipaddr(), 0),
             Prefix::new_relaxed(
@@ -404,64 +418,84 @@ mod tests {
 
     #[test]
     fn test_ranges_ipv4() -> Result<(), Box<dyn std::error::Error>> {
-        for i_net in 0..255 {
-            let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
-
-            let pfx_vec: Vec<Prefix> = (1..32)
-                .collect::<Vec<u8>>()
-                .into_iter()
-                .map(|i_len| {
-                    Prefix::new_relaxed(
-                        std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                        i_len,
-                    )
-                    .unwrap()
-                })
-                .collect();
-
-            let mut i_len_s = 0;
-            for pfx in pfx_vec {
-                i_len_s += 1;
-                tree_bitmap.insert(
-                    &pfx,
-                    Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
-                    None,
-                )?;
-
-                let res_pfx = Prefix::new_relaxed(
-                    std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                    i_len_s,
-                );
-
-                let guard = &epoch::pin();
-                for s_len in i_len_s..32 {
-                    let pfx = Prefix::new_relaxed(
-                        std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                        s_len,
-                    )?;
-                    let res = tree_bitmap.match_prefix(
+        for persist_strategy in [
+            PersistStrategy::MemoryOnly,
+            PersistStrategy::PersistOnly,
+            PersistStrategy::WriteAhead,
+            PersistStrategy::PersistHistory,
+        ] {
+            for i_net in 0..255 {
+                let config = StoreConfig {
+                    persist_strategy,
+                    persist_path: "/tmp/rotonda".into(),
+                };
+                let tree_bitmap =
+                    MultiThreadedStore::<NoMeta>::new_with_config(config)?;
+
+                let pfx_vec: Vec<Prefix> = (1..32)
+                    .collect::<Vec<u8>>()
+                    .into_iter()
+                    .map(|i_len| {
+                        Prefix::new_relaxed(
+                            std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
+                            i_len,
+                        )
+                        .unwrap()
+                    })
+                    .collect();
+
+                let mut i_len_s = 0;
+                for pfx in pfx_vec {
+                    i_len_s += 1;
+                    tree_bitmap.insert(
                         &pfx,
-                        &MatchOptions {
-                            match_type: MatchType::LongestMatch,
-                            include_withdrawn: false,
-                            include_less_specifics: false,
-                            include_more_specifics: false,
-                            mui: None,
-                            include_history: IncludeHistory::None,
-                        },
-                        guard,
+                        Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
+                        None,
+                    )?;
+
+                    let res_pfx = Prefix::new_relaxed(
+                        std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
+                        i_len_s,
                     );
-                    println!("{:?}", pfx);
 
-                    assert_eq!(res.prefix.unwrap(), res_pfx?);
+                    let guard = &epoch::pin();
+                    for s_len in i_len_s..32 {
+                        let pfx = Prefix::new_relaxed(
+                            std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
+                            s_len,
+                        )?;
+                        let res = tree_bitmap.match_prefix(
+                            &pfx,
+                            &MatchOptions {
+                                match_type: MatchType::LongestMatch,
+                                include_withdrawn: false,
+                                include_less_specifics: false,
+                                include_more_specifics: false,
+                                mui: None,
+                                include_history: IncludeHistory::None,
+                            },
+                            guard,
+                        );
+                        println!("{:?}", pfx);
+
+                        assert_eq!(res.prefix.unwrap(), res_pfx?);
+                    }
                 }
             }
         }
+
         Ok(())
     }
 
+    // rotonda_store::all_strategies![
+    //     multi_ranges;
+    //     test_multi_ranges_ipv4;
+    //     NoMeta
+    // ];
+
     #[test]
-    fn test_multi_ranges_ipv4() -> Result<(), Box<dyn std::error::Error>> {
+    fn test_multi_ranges_ipv4(// tree_bitmap: MultiThreadedStore<NoMeta>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
         let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
@@ -596,7 +630,7 @@ mod tests {
             .any(|r| r.multi_uniq_id == 1));
 
         let wd_pfx = Prefix::from_str("1.0.0.0/16")?;
-        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 2)?;
+        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 2, 1)?;
 
         println!("all records");
 
@@ -733,7 +767,7 @@ mod tests {
 
         //------------------
 
-        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 1)?;
+        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 1, 10)?;
         tree_bitmap.mark_mui_as_active_v4(1)?;
 
         let more_specifics = tree_bitmap.match_prefix(
@@ -782,10 +816,10 @@ mod tests {
         assert!(rec.is_empty());
 
         // withdraw muis 2,3,4,5 for the requested prefix
-        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 2)?;
-        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 3)?;
-        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 4)?;
-        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 5)?;
+        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 2, 11)?;
+        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 3, 12)?;
+        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 4, 13)?;
+        tree_bitmap.mark_mui_as_withdrawn_for_prefix(&wd_pfx, 5, 14)?;
 
         let more_specifics = tree_bitmap.match_prefix(
             &Prefix::from_str("1.0.0.0/16")?,
@@ -855,12 +889,13 @@ mod tests {
         assert_eq!(less_specifics.prefix_meta.len(), 5);
 
         let less_specifics = less_specifics.less_specifics.unwrap();
-        // All records for the less specific /16 are withdrawn, so this should be empty.
+        // All records for the less specific /16 are withdrawn, so this should
+        // be empty.
         assert!(less_specifics.is_empty());
 
         //--------------------
 
-        tree_bitmap.mark_mui_as_active_for_prefix(&wd_pfx, 5)?;
+        tree_bitmap.mark_mui_as_active_for_prefix(&wd_pfx, 5, 1)?;
 
         let less_specifics = tree_bitmap.match_prefix(
             &Prefix::from_str("1.0.0.0/17")?,

From 65edf347968ddae0dc489094eb96999f7d8e345d Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 10 Jan 2025 14:19:40 +0100
Subject: [PATCH 044/147] reform PrefixId and converstion traits

---
 src/af.rs                | 51 ++++++-----------------------
 src/local_array/types.rs | 70 +++++++++++++++++-----------------------
 src/prefix_record.rs     |  8 ++---
 3 files changed, 43 insertions(+), 86 deletions(-)

diff --git a/src/af.rs b/src/af.rs
index 9366b8b0..5995e2fb 100644
--- a/src/af.rs
+++ b/src/af.rs
@@ -54,14 +54,7 @@ pub trait AddressFamily:
     // finding node_ids (always zero for 0/0).
     fn checked_shr_or_zero(self, rhs: u32) -> Self;
 
-    fn as_prefix_bytes<const PREFIX_SIZE: usize>(
-        &self,
-        pfx_len: u8,
-    ) -> [u8; PREFIX_SIZE];
-
-    fn from_prefix_bytes<const PREFIX_SIZE: usize>(
-        value: [u8; PREFIX_SIZE],
-    ) -> (Self, u8);
+    fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE];
 }
 
 //-------------- Ipv4 Type --------------------------------------------------
@@ -160,23 +153,10 @@ impl AddressFamily for IPv4 {
         self.checked_shr(rhs).unwrap_or(0)
     }
 
-    fn as_prefix_bytes<const PREFIX_SIZE: usize>(
-        &self,
-        pfx_len: u8,
-    ) -> [u8; PREFIX_SIZE] {
-        let bytes = &mut [0_u8; PREFIX_SIZE];
-        *bytes.first_chunk_mut::<4>().unwrap() = self.to_le_bytes();
-        bytes[PREFIX_SIZE - 1] = pfx_len;
-        *bytes
-    }
-
-    fn from_prefix_bytes<const PREFIX_SIZE: usize>(
-        value: [u8; PREFIX_SIZE],
-    ) -> (Self, u8) {
-        (
-            u32::from_le_bytes(*value.first_chunk::<4>().unwrap()),
-            value[PREFIX_SIZE - 1],
-        )
+    fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
+        *u32::to_be_bytes(*self)
+            .first_chunk::<PREFIX_SIZE>()
+            .unwrap()
     }
 }
 
@@ -296,23 +276,10 @@ impl AddressFamily for IPv6 {
         self.checked_shr(rhs).unwrap_or(0)
     }
 
-    fn as_prefix_bytes<const PREFIX_SIZE: usize>(
-        &self,
-        pfx_len: u8,
-    ) -> [u8; PREFIX_SIZE] {
-        let res = &mut [0_u8; PREFIX_SIZE];
-        *res.first_chunk_mut::<16>().unwrap() = self.to_le_bytes();
-        res[PREFIX_SIZE - 1] = pfx_len;
-        *res
-    }
-
-    fn from_prefix_bytes<const PREFIX_SIZE: usize>(
-        value: [u8; PREFIX_SIZE],
-    ) -> (Self, u8) {
-        (
-            u128::from_le_bytes(*value.first_chunk::<16>().unwrap()),
-            value[PREFIX_SIZE - 1],
-        )
+    fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
+        *u128::to_be_bytes(*self)
+            .first_chunk::<PREFIX_SIZE>()
+            .unwrap()
     }
 }
 
diff --git a/src/local_array/types.rs b/src/local_array/types.rs
index fa38e509..0228a751 100644
--- a/src/local_array/types.rs
+++ b/src/local_array/types.rs
@@ -45,33 +45,22 @@ impl TryFrom<u8> for RouteStatus {
 //------------ PrefixId ------------------------------------------------------
 
 #[derive(Hash, Eq, PartialEq, Debug, Copy, Clone)]
-pub struct PrefixId<AF: AddressFamily>(Option<(AF, u8)>);
+pub struct PrefixId<AF: AddressFamily> {
+    net: AF,
+    len: u8,
+}
 
 impl<AF: AddressFamily> PrefixId<AF> {
     pub fn new(net: AF, len: u8) -> Self {
-        PrefixId(Some((net, len)))
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.0.is_none()
+        PrefixId { net, len }
     }
 
     pub fn get_net(&self) -> AF {
-        self.0.unwrap().0
+        self.net
     }
 
     pub fn get_len(&self) -> u8 {
-        self.0.unwrap().1
-    }
-
-    // This should never fail, since there shouldn't be a invalid prefix in
-    // this prefix id in the first place.
-    pub fn into_pub(&self) -> inetnum::addr::Prefix {
-        inetnum::addr::Prefix::new(
-            self.get_net().into_ipaddr(),
-            self.get_len(),
-        )
-        .unwrap_or_else(|p| panic!("can't convert {:?} into prefix.", p))
+        self.len
     }
 
     // Increment the length of the prefix without changing the bits part.
@@ -79,40 +68,38 @@ impl<AF: AddressFamily> PrefixId<AF> {
     // since the more specifics iterator includes the requested `base_prefix`
     // itself.
     pub fn inc_len(self) -> Self {
-        Self(self.0.map(|(net, len)| (net, len + 1)))
-    }
-
-    pub fn as_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
-        match self.0 {
-            Some(r) => r.0.as_prefix_bytes(r.1),
-            _ => [255; PREFIX_SIZE],
+        Self {
+            net: self.net,
+            len: self.len + 1,
         }
     }
-}
 
-impl<AF: AddressFamily> std::default::Default for PrefixId<AF> {
-    fn default() -> Self {
-        PrefixId(None)
+    // The lsm tree, used for persistence, stores the prefix in the key with
+    // len first, so that key range lookups can be made for more-specifics in
+    // each prefix length.
+    pub fn to_len_first_bytes<const PREFIX_SIZE: usize>(
+        &self,
+    ) -> [u8; PREFIX_SIZE] {
+        let bytes = &mut [0_u8; PREFIX_SIZE];
+        *bytes.last_chunk_mut::<4>().unwrap() = self.net.to_be_bytes();
+        bytes[0] = self.len;
+        *bytes
     }
 }
 
 impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
     fn from(value: inetnum::addr::Prefix) -> Self {
-        Self(Some((AF::from_ipaddr(value.addr()), value.len())))
+        Self {
+            net: AF::from_ipaddr(value.addr()),
+            len: value.len(),
+        }
     }
 }
 
 impl<AF: AddressFamily> From<PrefixId<AF>> for inetnum::addr::Prefix {
     fn from(value: PrefixId<AF>) -> Self {
-        value.into_pub()
-    }
-}
-
-impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<PrefixId<AF>>
-    for [u8; PREFIX_SIZE]
-{
-    fn from(value: PrefixId<AF>) -> Self {
-        value.as_bytes::<PREFIX_SIZE>()
+        println!("{} {}", value.get_net(), value.get_len());
+        Self::new(value.get_net().into_ipaddr(), value.get_len()).unwrap()
     }
 }
 
@@ -120,6 +107,9 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<[u8; PREFIX_SIZE]>
     for PrefixId<AF>
 {
     fn from(value: [u8; PREFIX_SIZE]) -> Self {
-        PrefixId(Some(AF::from_prefix_bytes(value)))
+        Self {
+            net: u32::from_be_bytes(*value.last_chunk::<4>().unwrap()).into(),
+            len: value[0],
+        }
     }
 }
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index ed612b42..7a50a8f8 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -208,7 +208,7 @@ where
 {
     fn from(record: (PrefixId<AF>, Arc<M>)) -> Self {
         Self {
-            prefix: record.0.into_pub(),
+            prefix: record.0.into(),
             meta: (*record.1).clone(),
         }
     }
@@ -288,7 +288,7 @@ where
 {
     fn from(record: (PrefixId<AF>, Vec<PublicRecord<M>>)) -> Self {
         Self {
-            prefix: record.0.into_pub(),
+            prefix: record.0.into(),
             meta: record.1,
         }
     }
@@ -438,7 +438,7 @@ impl<AF: AddressFamily, M: Meta>
         let mut v4 = vec![];
         let mut v6 = vec![];
         for pfx in iter {
-            let u_pfx = pfx.0.into_pub();
+            let u_pfx = Prefix::from(pfx.0);
             match u_pfx.addr() {
                 std::net::IpAddr::V4(_) => {
                     v4.push(PublicPrefixSingleRecord::new(
@@ -617,7 +617,7 @@ impl<AF: AddressFamily, M: Meta>
         let mut v4 = vec![];
         let mut v6 = vec![];
         for pfx in iter {
-            let u_pfx = pfx.0.into_pub();
+            let u_pfx = Prefix::from(pfx.0);
             match u_pfx.addr() {
                 std::net::IpAddr::V4(_) => {
                     v4.push(PublicPrefixRecord::new(u_pfx, pfx.1));

From e7fcfd5d1abc31c4dc9dd5698d8bb1ff7733a07f Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 10 Jan 2025 14:20:08 +0100
Subject: [PATCH 045/147] MSRV 1.82

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 1ad52638..dad9394b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,7 +20,7 @@ version = "0.4.1"
 edition = "2021"
 authors = ["NLnet Labs <routing-team@nlnetlabs.nl>"]
 license = "BSD-3-Clause"
-rust-version = "1.80"
+rust-version = "1.82"
 
 [dependencies]
 crossbeam-epoch = "^0.9"

From 6c2fd8428285952097facf2d2d0995521f8f273b Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 10 Jan 2025 14:22:33 +0100
Subject: [PATCH 046/147] reform PrefixId

---
 src/local_array/in_memory/iterators.rs | 8 ++++----
 src/local_array/in_memory/node.rs      | 8 ++++++--
 src/local_array/in_memory/query.rs     | 6 +++++-
 src/local_array/in_memory/tree.rs      | 8 --------
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index a00eba45..c064f948 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -19,12 +19,11 @@
 // contention, since every lookup has to go through the levels near the root
 // in the TreeBitMap.
 
-use super::super::types::PrefixId;
 use super::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
 use super::node::{SizedStrideRef, StrideNodeId};
 use super::tree::{Stride3, Stride4, Stride5, TreeBitMap};
 use crate::local_array::types::RouteStatus;
-use crate::prefix_record::PublicRecord;
+use crate::PublicRecord;
 use crate::{
     af::AddressFamily,
     local_array::{
@@ -32,6 +31,7 @@ use crate::{
         in_memory::node::{
             NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter,
         },
+        types::PrefixId,
     },
     prefix_record::Meta,
 };
@@ -203,7 +203,7 @@ impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
                         //     })
                         // {
                         return Some((
-                            s_pfx.get_prefix_id().into_pub(),
+                            s_pfx.get_prefix_id().into(),
                             s_pfx.record_map.as_records(),
                         ));
                         // } else {
@@ -226,7 +226,7 @@ impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
                         // {
                         self.cursor += 1;
                         return Some((
-                            s_pfx.get_prefix_id().into_pub(),
+                            s_pfx.get_prefix_id().into(),
                             s_pfx.record_map.as_records(),
                         ));
                         // }
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index dd3b446f..2f6e10dd 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -519,6 +519,7 @@ where
         start_bit: u8,
         less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
     ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
+
         let pfxbitarr = self.pfxbitarr.load();
         let ptrbitarr = self.ptrbitarr.load();
         let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
@@ -574,10 +575,13 @@ where
                 == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType as std::ops::BitAnd>::Output::zero()
         {
             // No children or at the end, return the definitive LMP we found.
-            true => (
+            true => { 
+                println!("found {:?}", found_pfx); 
+                (
                 None,      /* no more children */
                 found_pfx, /* The definitive LMP if any */
-            ),
+                )
+            },
             // There's another child, we won't return the found_pfx, since
             // we're not at the last nibble and we want an exact match only.
             false => (
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index 5e9e03f5..74b2f77d 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -714,7 +714,11 @@ where
             prefix: match_prefix_idx,
             match_type,
             less_specifics: if options.include_less_specifics {
-                less_specifics_vec
+                less_specifics_vec.map(|lsv| {
+                    lsv.into_iter()
+                        .filter(|r| r != &search_pfx)
+                        .collect::<Vec<_>>()
+                })
             } else {
                 None
             },
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index f3a26786..039b9c09 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -1048,14 +1048,6 @@ impl<
         }
     }
 
-    #[allow(dead_code)]
-    fn remove_prefix(&mut self, index: PrefixId<AF>) -> Option<M> {
-        match index.is_empty() {
-            false => self.prefix_buckets.remove(index),
-            true => None,
-        }
-    }
-
     pub fn get_prefixes_count(&self) -> usize {
         self.counters.get_prefixes_count().iter().sum()
     }

From d43265850bea480b71385aa8cc9e11b4554907b1 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 10 Jan 2025 14:23:09 +0100
Subject: [PATCH 047/147] persist_tree iterators

---
 src/local_array/persist/lsm_tree.rs | 472 ++++++++++++++++++++--------
 src/local_array/query.rs            | 103 +++---
 src/local_array/rib/rib.rs          |   3 +-
 tests/treebitmap.rs                 |  18 +-
 4 files changed, 405 insertions(+), 191 deletions(-)

diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index f980fa8e..f792f216 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -4,7 +4,6 @@ use std::marker::PhantomData;
 use std::path::Path;
 
 use inetnum::addr::Prefix;
-use log::trace;
 use lsm_tree::{AbstractTree, KvPair};
 use roaring::RoaringBitmap;
 
@@ -12,12 +11,13 @@ use crate::local_array::types::{PrefixId, RouteStatus};
 use crate::rib::query::{FamilyQueryResult, FamilyRecord, TreeQueryResult};
 use crate::rib::Counters;
 use crate::{
-    AddressFamily, IncludeHistory, MatchOptions, Meta, PublicRecord,
+    AddressFamily, IncludeHistory, MatchOptions, MatchType, Meta,
+    PublicRecord,
 };
 
 pub struct PersistTree<
     AF: AddressFamily,
-    // The size in bytes of the prefix in the peristed storage (disk), this
+    // The size in bytes of the prefix in the persisted storage (disk), this
     // amounnts to the bytes for the addres (4 for IPv4, 16 for IPv6) and 1
     // bytefor the prefix length.
     const PREFIX_SIZE: usize,
@@ -74,13 +74,13 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     PublicRecord::new(
                         mui,
                         ltime,
-                        status.try_into().unwrap(),
+                        status,
                         kv.1.as_ref().to_vec().into(),
                     )
                 })
                 .collect::<Vec<_>>()
         } else {
-            let prefix_b = &prefix.as_bytes::<PREFIX_SIZE>();
+            let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
 
             (*self.tree.prefix(prefix_b))
                 .into_iter()
@@ -91,7 +91,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     PublicRecord::new(
                         mui,
                         ltime,
-                        status.try_into().unwrap(),
+                        status,
                         kv.1.as_ref().to_vec().into(),
                     )
                 })
@@ -117,7 +117,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                 PublicRecord::new(
                     mui,
                     ltime,
-                    status.try_into().unwrap(),
+                    status,
                     m.as_ref().to_vec().into(),
                 )
             })
@@ -140,7 +140,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     PublicRecord::new(
                         mui,
                         ltime,
-                        status.try_into().unwrap(),
+                        status,
                         kv.1.as_ref().to_vec().into(),
                     ),
                 )
@@ -148,64 +148,22 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             .collect::<Vec<_>>()
     }
 
-    pub fn _get_records_for_key<M: Meta + From<Vec<u8>>>(
+    pub fn get_records_for_more_specific_prefix_in_len(
         &self,
-        key: &[u8],
-    ) -> Vec<(PrefixId<AF>, PublicRecord<M>)> {
-        (*self.tree.prefix(key))
-            .into_iter()
-            .map(|kv| {
-                let kv = kv.unwrap();
-                let (pfx, mui, ltime, status) =
-                    Self::parse_key(kv.0.as_ref());
-
-                (
-                    PrefixId::<AF>::from(pfx),
-                    PublicRecord::new(
-                        mui,
-                        ltime,
-                        status.try_into().unwrap(),
-                        kv.1.as_ref().to_vec().into(),
-                    ),
-                )
-            })
-            .collect::<Vec<_>>()
-    }
-
-    fn enrich_prefixes_most_recent<M: Meta>(
-        &self,
-        prefixes: Option<Vec<PrefixId<AF>>>,
-        mui: Option<u32>,
-    ) -> Option<FamilyRecord<AF, M>> {
-        prefixes.map(|pfxs| {
-            pfxs.iter()
-                .map(|pfx| {
-                    let prefix_b = if let Some(mui) = mui {
-                        &Self::prefix_mui_persistence_key(*pfx, mui)
-                    } else {
-                        &pfx.as_bytes::<PREFIX_SIZE>().to_vec()
-                    };
-                    (
-                        *pfx,
-                        (*self.tree.prefix(prefix_b))
-                            .into_iter()
-                            .last()
-                            .map(|kv| {
-                                let kv = kv.unwrap();
-                                let (_, mui, ltime, status) =
-                                    Self::parse_key(kv.0.as_ref());
-                                vec![PublicRecord::new(
-                                    mui,
-                                    ltime,
-                                    status.try_into().unwrap(),
-                                    kv.1.as_ref().to_vec().into(),
-                                )]
-                            })
-                            .unwrap_or_default(),
-                    )
-                })
-                .collect::<Vec<_>>()
-        })
+        prefix: PrefixId<AF>,
+        len: u8,
+    ) -> Box<
+        dyn DoubleEndedIterator<
+            Item = Result<
+                (lsm_tree::Slice, lsm_tree::Slice),
+                lsm_tree::Error,
+            >,
+        >,
+    > {
+        let start = PrefixId::new(prefix.get_net(), len);
+        let end: [u8; PREFIX_SIZE] = start.inc_len().to_len_first_bytes();
+
+        self.tree.range(start.to_len_first_bytes()..end)
     }
 
     fn enrich_prefix<M: Meta>(
@@ -221,6 +179,11 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                 self.get_records_for_prefix(pfx, mui)
                     .into_iter()
                     .filter_map(|mut r| {
+                        if !include_withdrawn
+                            && r.status == RouteStatus::Withdrawn
+                        {
+                            return None;
+                        }
                         if bmin.contains(r.multi_uniq_id) {
                             if !include_withdrawn {
                                 return None;
@@ -280,81 +243,92 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         options: &MatchOptions,
         bmin: &RoaringBitmap,
     ) -> FamilyQueryResult<AF, M> {
-        match options.include_history {
+        let (prefix, prefix_meta) = self.enrich_prefix(
+            search_pfxs.prefix,
+            options.mui,
+            options.include_withdrawn,
+            bmin,
+        );
+
+        let mut res = match options.include_history {
             // All the records for all the prefixes
-            IncludeHistory::All => {
-                let (prefix, prefix_meta) = self.enrich_prefix(
-                    search_pfxs.prefix,
+            IncludeHistory::All => FamilyQueryResult {
+                prefix,
+                prefix_meta,
+                match_type: search_pfxs.match_type,
+                less_specifics: self.enrich_prefixes(
+                    search_pfxs.less_specifics,
                     options.mui,
                     options.include_withdrawn,
                     bmin,
-                );
-
-                FamilyQueryResult {
-                    prefix,
-                    prefix_meta,
-                    match_type: search_pfxs.match_type,
-                    less_specifics: self.enrich_prefixes(
-                        search_pfxs.less_specifics,
+                ),
+                more_specifics: Some(
+                    self.more_specific_prefix_iter_from(
+                        prefix.unwrap(),
                         options.mui,
-                        options.include_withdrawn,
                         bmin,
-                    ),
-                    more_specifics: self.enrich_prefixes(
-                        search_pfxs.more_specifics,
-                        options.mui,
                         options.include_withdrawn,
-                        bmin,
-                    ),
-                }
-            }
-            // Only the search prefix itself has historical records attacched
+                    )
+                    .collect::<Vec<_>>(),
+                ),
+            },
+            // Only the search prefix itself has historical records attached
             // to it, other prefixes (less|more specifics), have no records
             // attached. Not useful with the MemoryOnly strategy (historical
             // records are neve kept in memory).
-            IncludeHistory::SearchPrefix => {
-                let (prefix, prefix_meta) = self.enrich_prefix(
-                    search_pfxs.prefix,
-                    options.mui,
-                    options.include_withdrawn,
-                    bmin,
-                );
-
-                FamilyQueryResult {
-                    prefix,
-                    prefix_meta,
-                    match_type: search_pfxs.match_type,
-                    less_specifics: self
-                        .sparse_record_set(search_pfxs.less_specifics),
-                    more_specifics: self
-                        .sparse_record_set(search_pfxs.more_specifics),
-                }
-            }
+            IncludeHistory::SearchPrefix => FamilyQueryResult {
+                prefix,
+                prefix_meta,
+                match_type: search_pfxs.match_type,
+                less_specifics: self
+                    .sparse_record_set(search_pfxs.less_specifics),
+                more_specifics: self
+                    .sparse_record_set(search_pfxs.more_specifics),
+            },
             // Only the most recent record of the search prefix is returned
             // with the prefixes. This is used for the PersistOnly strategy.
-            IncludeHistory::None => {
-                let (prefix, prefix_meta) = self.enrich_prefix(
-                    search_pfxs.prefix,
-                    options.mui,
-                    options.include_withdrawn,
-                    bmin,
-                );
-
-                FamilyQueryResult {
-                    prefix,
-                    prefix_meta,
-                    match_type: search_pfxs.match_type,
-                    less_specifics: self.enrich_prefixes_most_recent(
-                        search_pfxs.less_specifics,
+            IncludeHistory::None => FamilyQueryResult {
+                prefix,
+                prefix_meta,
+                match_type: search_pfxs.match_type,
+                less_specifics: search_pfxs.less_specifics.map(|ls| {
+                    self.less_specific_prefix_iter_from(
+                        ls,
                         options.mui,
-                    ),
-                    more_specifics: self.enrich_prefixes_most_recent(
-                        search_pfxs.more_specifics,
+                        bmin,
+                        options.include_withdrawn,
+                    )
+                    .collect::<Vec<_>>()
+                }),
+                more_specifics: prefix.map(|p| {
+                    self.more_specific_prefix_iter_from(
+                        p,
                         options.mui,
-                    ),
+                        bmin,
+                        options.include_withdrawn,
+                    )
+                    .collect::<Vec<_>>()
+                }),
+            },
+        };
+
+        res.match_type = match (options.match_type, &res) {
+            (_, res) if !res.prefix_meta.is_empty() => MatchType::ExactMatch,
+            (MatchType::LongestMatch | MatchType::EmptyMatch, _) => {
+                if res
+                    .less_specifics
+                    .as_ref()
+                    .is_some_and(|lp| !lp.is_empty())
+                {
+                    MatchType::LongestMatch
+                } else {
+                    MatchType::EmptyMatch
                 }
             }
-        }
+            (MatchType::ExactMatch, _) => MatchType::EmptyMatch,
+        };
+
+        res
     }
 
     pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
@@ -402,7 +376,8 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         let key = &mut [0_u8; KEY_SIZE];
 
         // prefix 5 or 17 bytes
-        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
+        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() =
+            prefix_id.to_len_first_bytes();
 
         // mui 4 bytes
         *key[PREFIX_SIZE..PREFIX_SIZE + 4]
@@ -427,7 +402,8 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     ) -> Vec<u8> {
         let mut key = vec![0; PREFIX_SIZE + 4];
         // prefix 5 or 17 bytes
-        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() = prefix_id.as_bytes();
+        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() =
+            prefix_id.to_len_first_bytes();
 
         // mui 4 bytes
         *key[PREFIX_SIZE..PREFIX_SIZE + 4]
@@ -438,10 +414,10 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     }
 
     #[cfg(feature = "persist")]
-    pub fn parse_key(bytes: &[u8]) -> ([u8; PREFIX_SIZE], u32, u64, u8) {
+    pub fn parse_key(bytes: &[u8]) -> (PrefixId<AF>, u32, u64, RouteStatus) {
         (
             // prefix 5 or 17 bytes
-            *bytes.first_chunk::<PREFIX_SIZE>().unwrap(),
+            PrefixId::from(*bytes.first_chunk::<PREFIX_SIZE>().unwrap()),
             // mui 4 bytes
             u32::from_le_bytes(
                 *bytes[PREFIX_SIZE..PREFIX_SIZE + 4]
@@ -455,7 +431,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     .unwrap(),
             ),
             // status 1 byte
-            bytes[PREFIX_SIZE + 12],
+            RouteStatus::try_from(bytes[PREFIX_SIZE + 12]).unwrap(),
         )
     }
 
@@ -492,6 +468,49 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             _m: PhantomData,
         }
     }
+
+    pub(crate) fn more_specific_prefix_iter_from<'a, M: Meta + 'a>(
+        &'a self,
+        search_prefix: PrefixId<AF>,
+        mui: Option<u32>,
+        global_withdrawn_bmin: &'a RoaringBitmap,
+        include_withdrawn: bool,
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        let cur_range = self.get_records_for_more_specific_prefix_in_len(
+            search_prefix,
+            search_prefix.get_len() + 1,
+        );
+
+        println!("more_specific_prefix_iter_from with mui {:?}", mui);
+        MoreSpecificPrefixIter {
+            store: self,
+            search_prefix,
+            cur_len: search_prefix.get_len() + 1,
+            mui,
+            global_withdrawn_bmin,
+            include_withdrawn,
+            cur_range,
+            next_rec: None,
+        }
+    }
+
+    pub(crate) fn less_specific_prefix_iter_from<'a, M: Meta + 'a>(
+        &'a self,
+        search_lengths: Vec<PrefixId<AF>>,
+        mui: Option<u32>,
+        global_withdrawn_bmin: &'a RoaringBitmap,
+        include_withdrawn: bool,
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        println!("more_specific_prefix_iter_from with mui {:?}", mui);
+        LessSpecificPrefixIter {
+            store: self,
+            search_lengths,
+            mui,
+            global_withdrawn_bmin,
+            include_withdrawn,
+            _m: PhantomData,
+        }
+    }
 }
 
 impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
@@ -502,13 +521,16 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     }
 }
 
+// Iterator for all items in a lsm tree partition. The iterator used for
+// this will scann through the entire tree, and there's no way to start at a
+// specified offset.
 pub(crate) struct PersistedPrefixIter<
     AF: AddressFamily,
     M: Meta,
     const PREFIX_SIZE: usize,
     const KEY_SIZE: usize,
 > {
-    cur_rec: Option<([u8; PREFIX_SIZE], Vec<PublicRecord<M>>)>,
+    cur_rec: Option<(PrefixId<AF>, Vec<PublicRecord<M>>)>,
     tree_iter:
         Box<dyn DoubleEndedIterator<Item = Result<KvPair, lsm_tree::Error>>>,
     _af: PhantomData<AF>,
@@ -528,6 +550,7 @@ impl<
 
         // Do we already have a record in our iter struct?
         if let Some(_cur_rec) = &mut self.cur_rec {
+            // yes, use it.
             rec = std::mem::take(&mut self.cur_rec);
         } else {
             // No, advance to the next record in the persist tree.
@@ -548,7 +571,7 @@ impl<
                         vec![PublicRecord::<M> {
                             multi_uniq_id: p_k.1,
                             ltime: p_k.2,
-                            status: p_k.3.try_into().unwrap(),
+                            status: p_k.3,
                             meta: v.to_vec().into(),
                         }],
                     ));
@@ -574,7 +597,7 @@ impl<
                         meta: v.to_vec().into(),
                         multi_uniq_id: mui,
                         ltime,
-                        status: status.try_into().unwrap(),
+                        status,
                     });
                 } else {
                     self.cur_rec = Some((
@@ -583,21 +606,192 @@ impl<
                             meta: v.to_vec().into(),
                             multi_uniq_id: mui,
                             ltime,
-                            status: status.try_into().unwrap(),
+                            status,
                         }],
                     ));
                     break;
                 }
             }
 
-            Some((
-                Prefix::from(PrefixId::<AF>::from(
-                    *r_rec.0.first_chunk::<PREFIX_SIZE>().unwrap(),
-                )),
-                r_rec.1,
-            ))
+            Some((r_rec.0.into(), r_rec.1))
         } else {
             None
         }
     }
 }
+
+pub(crate) struct MoreSpecificPrefixIter<
+    'a,
+    AF: AddressFamily + 'a,
+    M: Meta + 'a,
+    const PREFIX_SIZE: usize,
+    const KEY_SIZE: usize,
+> {
+    next_rec: Option<(PrefixId<AF>, Vec<PublicRecord<M>>)>,
+    store: &'a PersistTree<AF, PREFIX_SIZE, KEY_SIZE>,
+    search_prefix: PrefixId<AF>,
+    cur_len: u8,
+    cur_range: Box<
+        dyn DoubleEndedIterator<
+            Item = lsm_tree::Result<(lsm_tree::Slice, lsm_tree::Slice)>,
+        >,
+    >,
+    mui: Option<u32>,
+    global_withdrawn_bmin: &'a RoaringBitmap,
+    include_withdrawn: bool,
+}
+
+impl<
+        'a,
+        AF: AddressFamily + 'a,
+        M: Meta + 'a,
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > Iterator for MoreSpecificPrefixIter<'a, AF, M, PREFIX_SIZE, KEY_SIZE>
+{
+    type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
+    fn next(&mut self) -> Option<Self::Item> {
+        println!("get the next from persist w/ {:?}", self.mui);
+        let mut cur_pfx = None;
+        let mut recs =
+            if let Some(next_rec) = std::mem::take(&mut self.next_rec) {
+                cur_pfx = Some(next_rec.0);
+                next_rec.1
+            } else {
+                vec![]
+            };
+        loop {
+            if let Some(Ok((k, v))) = self.cur_range.next() {
+                print!("..{:?}", k);
+                let (pfx, mui, ltime, mut status) =
+                    PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::parse_key(
+                        k.as_ref(),
+                    );
+                println!(
+                    "..{:?}++{}={:?}",
+                    pfx, status, self.include_withdrawn
+                );
+
+                if !self.include_withdrawn
+                    && (status == RouteStatus::Withdrawn)
+                {
+                    println!("skip 1.");
+                    continue;
+                }
+
+                if self.global_withdrawn_bmin.contains(mui) {
+                    if !self.include_withdrawn {
+                        println!("skip 2.");
+                        continue;
+                    } else {
+                        status = RouteStatus::Withdrawn;
+                    }
+                }
+
+                if let Some(m) = self.mui {
+                    if m != mui {
+                        println!("skip 3.");
+                        continue;
+                    }
+                }
+
+                println!("X {:?} == {:?}", pfx, cur_pfx);
+                cur_pfx = if cur_pfx.is_some() {
+                    cur_pfx
+                } else {
+                    Some(pfx)
+                };
+
+                if cur_pfx.is_some_and(|c| c == pfx) {
+                    println!("Y");
+                    recs.push(PublicRecord::new(
+                        mui,
+                        ltime,
+                        status,
+                        v.as_ref().to_vec().into(),
+                    ));
+                } else {
+                    println!("Z {:?}", (pfx, &recs));
+                    self.next_rec = cur_pfx.map(|_p| {
+                        (
+                            pfx,
+                            vec![PublicRecord::new(
+                                mui,
+                                ltime,
+                                status,
+                                v.as_ref().to_vec().into(),
+                            )],
+                        )
+                    });
+                    return Some((pfx, recs));
+                }
+                println!("..{}", recs.len());
+            } else {
+                println!("B");
+                // See if there's a next prefix length to iterate over
+                if self.cur_len == AF::BITS {
+                    println!("A");
+                    return cur_pfx.map(|p| (p, recs));
+                }
+
+                self.cur_len += 1;
+                self.cur_range =
+                    self.store.get_records_for_more_specific_prefix_in_len(
+                        self.search_prefix,
+                        self.cur_len,
+                    );
+            }
+        }
+    }
+}
+
+pub(crate) struct LessSpecificPrefixIter<
+    'a,
+    AF: AddressFamily + 'a,
+    M: Meta + 'a,
+    const PREFIX_SIZE: usize,
+    const KEY_SIZE: usize,
+> {
+    store: &'a PersistTree<AF, PREFIX_SIZE, KEY_SIZE>,
+    search_lengths: Vec<PrefixId<AF>>,
+    mui: Option<u32>,
+    global_withdrawn_bmin: &'a RoaringBitmap,
+    include_withdrawn: bool,
+    _m: PhantomData<M>,
+}
+
+impl<
+        'a,
+        AF: AddressFamily + 'a,
+        M: Meta + 'a,
+        const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > Iterator for LessSpecificPrefixIter<'a, AF, M, PREFIX_SIZE, KEY_SIZE>
+{
+    type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            if let Some(lp) = self.search_lengths.pop() {
+                let recs = self
+                    .store
+                    .get_records_for_prefix(lp, self.mui)
+                    .into_iter()
+                    .filter(|r| self.mui.is_none_or(|m| m == r.multi_uniq_id))
+                    .filter(|r| {
+                        self.include_withdrawn
+                            || (!self
+                                .global_withdrawn_bmin
+                                .contains(r.multi_uniq_id)
+                                && r.status != RouteStatus::Withdrawn)
+                    })
+                    .collect::<Vec<_>>();
+
+                if !recs.is_empty() {
+                    return Some((lp, recs));
+                }
+            } else {
+                return None;
+            }
+        }
+    }
+}
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 80ebac88..b3c05fbe 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -5,8 +5,8 @@ use crate::af::AddressFamily;
 use crate::local_array::in_memory::atomic_types::{
     NodeBuckets, PrefixBuckets,
 };
-use crate::prefix_record::PublicRecord;
 use crate::rib::{PersistStrategy, Rib};
+use crate::PublicRecord;
 use inetnum::addr::Prefix;
 
 use crate::{IncludeHistory, Meta, QueryResult};
@@ -121,24 +121,42 @@ where
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-        match self.config.persist_strategy() {
-            PersistStrategy::MemoryOnly
-            | PersistStrategy::WriteAhead
-            | PersistStrategy::PersistHistory => {
-                // if mui.is_some_and(|m| self.mui_is_withdrawn(m, guard)) {
-                //     println!("more_specifics_iter_from");
-                //     unimplemented!()
-                // } else {
-                self.in_memory_tree.more_specific_prefix_iter_from(
-                    prefix_id,
-                    mui,
-                    include_withdrawn,
-                    guard,
-                )
-                // }
-            }
-            PersistStrategy::PersistOnly => unimplemented!(),
-        }
+        // If the user wanted a specific mui and not withdrawn prefixes, we
+        // may return early if the mui is globally withdrawn.
+        (if mui.is_some_and(|m| {
+            !include_withdrawn && self.mui_is_withdrawn(m, guard)
+        }) {
+            None
+        } else {
+            Some(self.in_memory_tree.more_specific_prefix_iter_from(
+                prefix_id,
+                mui,
+                include_withdrawn,
+                guard,
+            ))
+        })
+        .into_iter()
+        .flatten()
+        .chain(
+            (if mui.is_some_and(|m| {
+                !include_withdrawn && self.mui_is_withdrawn(m, guard)
+            }) {
+                None
+            } else {
+                let global_withdrawn_bmin =
+                    self.in_memory_tree.withdrawn_muis_bmin(guard);
+                self.persist_tree.as_ref().map(|persist_tree| {
+                    persist_tree.more_specific_prefix_iter_from(
+                        prefix_id,
+                        mui,
+                        global_withdrawn_bmin,
+                        include_withdrawn,
+                    )
+                })
+            })
+            .into_iter()
+            .flatten(),
+        )
     }
 
     pub fn less_specifics_iter_from(
@@ -152,17 +170,18 @@ where
             PersistStrategy::MemoryOnly
             | PersistStrategy::WriteAhead
             | PersistStrategy::PersistHistory => {
-                // if mui.is_some_and(|m| self.mui_is_withdrawn(m, guard)) {
-                //     println!("less_specifics_iter_from");
-                //     unimplemented!()
-                // } else {
-                self.in_memory_tree.less_specific_prefix_iter(
-                    prefix_id,
-                    mui,
-                    include_withdrawn,
-                    guard,
-                )
-                // }
+                (if mui.is_some_and(|m| self.mui_is_withdrawn(m, guard)) {
+                    None
+                } else {
+                    Some(self.in_memory_tree.less_specific_prefix_iter(
+                        prefix_id,
+                        mui,
+                        include_withdrawn,
+                        guard,
+                    ))
+                })
+                .into_iter()
+                .flatten()
             }
             PersistStrategy::PersistOnly => unimplemented!(),
         }
@@ -187,13 +206,13 @@ where
             // to memory. However the in-memory-tree is still used to indicate
             // which (prefix, mui) tuples have been created.
             PersistStrategy::PersistOnly => {
+                let withdrawn_muis_bmin =
+                    self.in_memory_tree.withdrawn_muis_bmin(guard);
                 // If no withdawn should be included and a specific mui was
                 // requested, then return early, if the mui lives in the
                 // global withdrawn muis.
                 if !options.include_withdrawn {
                     if let Some(mui) = options.mui {
-                        let withdrawn_muis_bmin =
-                            self.in_memory_tree.withdrawn_muis_bmin(guard);
                         if withdrawn_muis_bmin.contains(mui) {
                             return QueryResult::empty();
                         }
@@ -207,19 +226,17 @@ where
                 }
 
                 if let Some(persist_tree) = &self.persist_tree {
-                    let exists = self
+                    let tbm_result = self
                         .in_memory_tree
                         .match_prefix_by_tree_traversal(search_pfx, options);
-                    println!("IN_MEM {:#?}", exists);
-                    println!("{}", self.in_memory_tree);
-                    println!(
-                        "{:#?}",
-                        self.in_memory_tree
-                            .prefixes_iter()
-                            .collect::<Vec<_>>()
-                    );
-                    let bmin = self.in_memory_tree.withdrawn_muis_bmin(guard);
-                    persist_tree.match_prefix(exists, options, bmin).into()
+
+                    persist_tree
+                        .match_prefix(
+                            tbm_result,
+                            options,
+                            withdrawn_muis_bmin,
+                        )
+                        .into()
                 } else {
                     QueryResult::empty()
                 }
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 09700ff3..6a3ba2ba 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -397,6 +397,7 @@ impl<
                 stored_prefix.record_map.mark_as_withdrawn_for_mui(mui, ltime);
             }
             PersistStrategy::PersistOnly => {
+                println!("mark as wd in persist tree {:?} for mui {:?}", prefix, mui);
                 let p_tree = self.persist_tree.as_ref().unwrap();
                 let stored_prefixes = p_tree
                     .get_records_with_keys_for_prefix_mui::<M>(prefix, mui);
@@ -524,7 +525,7 @@ impl<
                     .non_recursive_retrieve_prefix_mut(prefix);
 
                 if !exists {
-                    return Err(PrefixStoreError::StoreNotReadyError);
+                    return Err(PrefixStoreError::PrefixNotFound);
                 }
                 stored_prefix.record_map.mark_as_active_for_mui(mui, ltime);
 
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 02dd4d8a..76717968 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -487,18 +487,19 @@ mod tests {
         Ok(())
     }
 
-    // rotonda_store::all_strategies![
-    //     multi_ranges;
-    //     test_multi_ranges_ipv4;
-    //     NoMeta
-    // ];
+    rotonda_store::all_strategies![
+        multi_ranges;
+        test_multi_ranges_ipv4;
+        NoMeta
+    ];
 
-    #[test]
-    fn test_multi_ranges_ipv4(// tree_bitmap: MultiThreadedStore<NoMeta>,
+    // #[test]
+    fn test_multi_ranges_ipv4(
+        tree_bitmap: MultiThreadedStore<NoMeta>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
+        // let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
         for mui in [1_u32, 2, 3, 4, 5] {
             println!("Multi Uniq ID {mui}");
 
@@ -786,6 +787,7 @@ mod tests {
         println!("more_specifics match w/o withdrawn #2 {}", more_specifics);
         // We withdrew mui 1 for the requested prefix itself, since mui 2 was
         // already withdrawn above, we're left with 3 records
+        println!("PREFIX META: {:#?}", more_specifics.prefix_meta);
         assert_eq!(more_specifics.prefix_meta.len(), 3);
 
         let more_specifics = more_specifics.more_specifics.unwrap();

From 290f85aa45b45db11caa127d608881673314a007 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 10 Jan 2025 16:30:39 +0100
Subject: [PATCH 048/147] skip persist_store lookups for WriteAhead strategy

---
 src/local_array/query.rs   |  4 +++-
 src/local_array/rib/rib.rs | 32 +++++++++++++++++++++-----------
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index b3c05fbe..5a995226 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -139,7 +139,8 @@ where
         .flatten()
         .chain(
             (if mui.is_some_and(|m| {
-                !include_withdrawn && self.mui_is_withdrawn(m, guard)
+                self.config.persist_strategy == PersistStrategy::WriteAhead
+                    || (!include_withdrawn && self.mui_is_withdrawn(m, guard))
             }) {
                 None
             } else {
@@ -148,6 +149,7 @@ where
                 self.persist_tree.as_ref().map(|persist_tree| {
                     persist_tree.more_specific_prefix_iter_from(
                         prefix_id,
+                        vec![],
                         mui,
                         global_withdrawn_bmin,
                         include_withdrawn,
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 6a3ba2ba..8f41b27d 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -394,10 +394,15 @@ impl<
                 if !exists {
                     return Err(PrefixStoreError::PrefixNotFound);
                 }
-                stored_prefix.record_map.mark_as_withdrawn_for_mui(mui, ltime);
+                stored_prefix
+                    .record_map
+                    .mark_as_withdrawn_for_mui(mui, ltime);
             }
             PersistStrategy::PersistOnly => {
-                println!("mark as wd in persist tree {:?} for mui {:?}", prefix, mui);
+                println!(
+                    "mark as wd in persist tree {:?} for mui {:?}",
+                    prefix, mui
+                );
                 let p_tree = self.persist_tree.as_ref().unwrap();
                 let stored_prefixes = p_tree
                     .get_records_with_keys_for_prefix_mui::<M>(prefix, mui);
@@ -429,7 +434,9 @@ impl<
                 if !exists {
                     return Err(PrefixStoreError::StoreNotReadyError);
                 }
-                stored_prefix.record_map.mark_as_withdrawn_for_mui(mui, ltime);
+                stored_prefix
+                    .record_map
+                    .mark_as_withdrawn_for_mui(mui, ltime);
 
                 // Use the record from the in-memory RIB to persist.
                 if let Some(_record) =
@@ -507,12 +514,12 @@ impl<
                     // remove the entry for the same (prefix, mui), but with
                     // an older logical time
                     let old_key = PersistTree::<
-                        AF, 
+                        AF,
                         PREFIX_SIZE,
                         KEY_SIZE>::persistence_key(
-                            prefix, 
-                            mui, 
-                            record.ltime, 
+                            prefix,
+                            mui,
+                            record.ltime,
                             record.status
                     );
                     p_tree.remove(old_key);
@@ -683,12 +690,15 @@ impl<
     pub fn prefixes_iter(
         &self,
     ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + '_ {
-        let pt_iter = self
-            .persist_tree
-            .as_ref()
-            .map(|t| t.prefixes_iter())
+        let pt_iter =
+            if self.config.persist_strategy == PersistStrategy::WriteAhead {
+                None
+            } else {
+                self.persist_tree.as_ref().map(|t| t.prefixes_iter())
+            }
             .into_iter()
             .flatten();
+
         self.in_memory_tree.prefixes_iter().chain(pt_iter)
     }
 

From 416bb9b3780cf6578eca9533f37c57297430295c Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 10 Jan 2025 16:31:19 +0100
Subject: [PATCH 049/147] use prefix lenghts from tbm for perist store
 more|less specifics

---
 src/local_array/persist/lsm_tree.rs | 68 ++++++++++++-----------------
 src/local_array/types.rs            |  1 -
 2 files changed, 27 insertions(+), 42 deletions(-)

diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index f792f216..e1e1de4f 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -262,15 +262,16 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     options.include_withdrawn,
                     bmin,
                 ),
-                more_specifics: Some(
+                more_specifics: search_pfxs.more_specifics.map(|ms| {
                     self.more_specific_prefix_iter_from(
                         prefix.unwrap(),
+                        ms.iter().map(|p| p.get_len()).collect::<Vec<_>>(),
                         options.mui,
                         bmin,
                         options.include_withdrawn,
                     )
-                    .collect::<Vec<_>>(),
-                ),
+                    .collect::<Vec<_>>()
+                }),
             },
             // Only the search prefix itself has historical records attached
             // to it, other prefixes (less|more specifics), have no records
@@ -300,9 +301,10 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                     )
                     .collect::<Vec<_>>()
                 }),
-                more_specifics: prefix.map(|p| {
+                more_specifics: search_pfxs.more_specifics.map(|ms| {
                     self.more_specific_prefix_iter_from(
-                        p,
+                        prefix.unwrap(),
+                        ms.iter().map(|p| p.get_len()).collect::<Vec<_>>(),
                         options.mui,
                         bmin,
                         options.include_withdrawn,
@@ -435,11 +437,6 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         )
     }
 
-    // #[cfg(feature = "persist")]
-    // pub fn parse_prefix(bytes: &[u8]) -> [u8; PREFIX_SIZE] {
-    //     *bytes.first_chunk::<PREFIX_SIZE>().unwrap()
-    // }
-
     #[cfg(feature = "persist")]
     pub(crate) fn persist_record<M: Meta>(
         &self,
@@ -472,20 +469,25 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     pub(crate) fn more_specific_prefix_iter_from<'a, M: Meta + 'a>(
         &'a self,
         search_prefix: PrefixId<AF>,
+        mut search_lengths: Vec<u8>,
         mui: Option<u32>,
         global_withdrawn_bmin: &'a RoaringBitmap,
         include_withdrawn: bool,
     ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-        let cur_range = self.get_records_for_more_specific_prefix_in_len(
-            search_prefix,
-            search_prefix.get_len() + 1,
-        );
+        if search_lengths.is_empty() {
+            for l in search_prefix.get_len() + 1..=AF::BITS {
+                search_lengths.push(l);
+            }
+        }
+
+        let len = search_lengths.pop().unwrap();
+        let cur_range = self
+            .get_records_for_more_specific_prefix_in_len(search_prefix, len);
 
-        println!("more_specific_prefix_iter_from with mui {:?}", mui);
         MoreSpecificPrefixIter {
             store: self,
             search_prefix,
-            cur_len: search_prefix.get_len() + 1,
+            search_lengths,
             mui,
             global_withdrawn_bmin,
             include_withdrawn,
@@ -501,7 +503,6 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         global_withdrawn_bmin: &'a RoaringBitmap,
         include_withdrawn: bool,
     ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-        println!("more_specific_prefix_iter_from with mui {:?}", mui);
         LessSpecificPrefixIter {
             store: self,
             search_lengths,
@@ -630,7 +631,7 @@ pub(crate) struct MoreSpecificPrefixIter<
     next_rec: Option<(PrefixId<AF>, Vec<PublicRecord<M>>)>,
     store: &'a PersistTree<AF, PREFIX_SIZE, KEY_SIZE>,
     search_prefix: PrefixId<AF>,
-    cur_len: u8,
+    search_lengths: Vec<u8>,
     cur_range: Box<
         dyn DoubleEndedIterator<
             Item = lsm_tree::Result<(lsm_tree::Slice, lsm_tree::Slice)>,
@@ -651,7 +652,6 @@ impl<
 {
     type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
     fn next(&mut self) -> Option<Self::Item> {
-        println!("get the next from persist w/ {:?}", self.mui);
         let mut cur_pfx = None;
         let mut recs =
             if let Some(next_rec) = std::mem::take(&mut self.next_rec) {
@@ -662,26 +662,19 @@ impl<
             };
         loop {
             if let Some(Ok((k, v))) = self.cur_range.next() {
-                print!("..{:?}", k);
                 let (pfx, mui, ltime, mut status) =
                     PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::parse_key(
                         k.as_ref(),
                     );
-                println!(
-                    "..{:?}++{}={:?}",
-                    pfx, status, self.include_withdrawn
-                );
 
                 if !self.include_withdrawn
                     && (status == RouteStatus::Withdrawn)
                 {
-                    println!("skip 1.");
                     continue;
                 }
 
                 if self.global_withdrawn_bmin.contains(mui) {
                     if !self.include_withdrawn {
-                        println!("skip 2.");
                         continue;
                     } else {
                         status = RouteStatus::Withdrawn;
@@ -690,12 +683,10 @@ impl<
 
                 if let Some(m) = self.mui {
                     if m != mui {
-                        println!("skip 3.");
                         continue;
                     }
                 }
 
-                println!("X {:?} == {:?}", pfx, cur_pfx);
                 cur_pfx = if cur_pfx.is_some() {
                     cur_pfx
                 } else {
@@ -703,7 +694,6 @@ impl<
                 };
 
                 if cur_pfx.is_some_and(|c| c == pfx) {
-                    println!("Y");
                     recs.push(PublicRecord::new(
                         mui,
                         ltime,
@@ -711,7 +701,6 @@ impl<
                         v.as_ref().to_vec().into(),
                     ));
                 } else {
-                    println!("Z {:?}", (pfx, &recs));
                     self.next_rec = cur_pfx.map(|_p| {
                         (
                             pfx,
@@ -725,21 +714,18 @@ impl<
                     });
                     return Some((pfx, recs));
                 }
-                println!("..{}", recs.len());
             } else {
-                println!("B");
                 // See if there's a next prefix length to iterate over
-                if self.cur_len == AF::BITS {
-                    println!("A");
+                if let Some(len) = self.search_lengths.pop() {
+                    self.cur_range = self
+                        .store
+                        .get_records_for_more_specific_prefix_in_len(
+                            self.search_prefix,
+                            len,
+                        );
+                } else {
                     return cur_pfx.map(|p| (p, recs));
                 }
-
-                self.cur_len += 1;
-                self.cur_range =
-                    self.store.get_records_for_more_specific_prefix_in_len(
-                        self.search_prefix,
-                        self.cur_len,
-                    );
             }
         }
     }
diff --git a/src/local_array/types.rs b/src/local_array/types.rs
index 0228a751..731b85a2 100644
--- a/src/local_array/types.rs
+++ b/src/local_array/types.rs
@@ -98,7 +98,6 @@ impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
 
 impl<AF: AddressFamily> From<PrefixId<AF>> for inetnum::addr::Prefix {
     fn from(value: PrefixId<AF>) -> Self {
-        println!("{} {}", value.get_net(), value.get_len());
         Self::new(value.get_net().into_ipaddr(), value.get_len()).unwrap()
     }
 }

From d6c370282d2299cb156771d5dbdb4495ebc66220 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 17 Jan 2025 15:32:14 +0100
Subject: [PATCH 050/147] breaking changes

---
 proc_macros/src/lib.rs                    |  34 +-
 src/bin/load_mrt.rs                       |   6 +-
 src/local_array/in_memory/atomic_types.rs |  12 +-
 src/local_array/in_memory/iterators.rs    | 268 +++++++++++++-
 src/local_array/in_memory/macros.rs       |   2 +-
 src/local_array/in_memory/node.rs         |  49 ++-
 src/local_array/in_memory/query.rs        | 170 ++++++---
 src/local_array/in_memory/tree.rs         |  14 +-
 src/local_array/persist/lsm_tree.rs       | 405 +++++++++++++++-------
 src/local_array/query.rs                  | 199 ++++++++---
 src/local_array/rib/default_store.rs      |   3 +-
 src/local_array/rib/rib.rs                |  14 +-
 src/macros.rs                             |  73 ++++
 src/meta_examples.rs                      |   2 +-
 src/test_types.rs                         |   7 +-
 tests/concurrency.rs                      | 225 ++++++++----
 16 files changed, 1137 insertions(+), 346 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index ac5e15c5..38a57851 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -888,6 +888,25 @@ pub fn create_store(
                 }
             }
 
+            pub fn more_specifics_keys_from(&'a self,
+                search_pfx: &Prefix,
+            ) -> Vec<Prefix> {
+
+                match search_pfx.addr() {
+                    std::net::IpAddr::V4(addr) => self
+                        .v4
+                        .more_specifics_keys_from(PrefixId::from(*search_pfx)
+                        ).map(|p| Prefix::from(p)).collect(),
+                    std::net::IpAddr::V6(addr) => self
+                        .v6
+                        .more_specifics_keys_from(
+                            PrefixId::<IPv6>::from(
+                                *search_pfx
+                            ),
+                        ).map(|p| Prefix::from(p)).collect()
+                }
+            }
+
             /// Return a `QuerySet` that contains all the less-specific
             /// prefixes of the `search_pfx` in the store, including the
             /// meta-data of these prefixes.
@@ -1646,12 +1665,21 @@ pub fn create_store(
 
             pub fn get_records_for_prefix(
                 &self, prefix: &Prefix,
-                mui: Option<u32>
+                mui: Option<u32>,
+                include_withdrawn: bool
             ) ->
                 Vec<Record<M>> {
                 match prefix.is_v4() {
-                    true => self.v4.get_records_for_prefix(prefix, mui),
-                    false => self.v6.get_records_for_prefix(prefix, mui)
+                    true => self.v4.get_records_for_prefix(
+                        prefix,
+                        mui,
+                        include_withdrawn
+                    ),
+                    false => self.v6.get_records_for_prefix(
+                        prefix,
+                        mui,
+                        include_withdrawn
+                    )
                 }
             }
 
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index 17b26c86..c1c2153a 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -596,7 +596,8 @@ fn main() {
         println!("\nverifying disk persistence...");
         let mut max_len = 0;
         for pfx in persisted_prefixes {
-            let values = store.unwrap().get_records_for_prefix(&pfx, None);
+            let values =
+                store.unwrap().get_records_for_prefix(&pfx, None, false);
             if values.is_empty() {
                 eprintln!("Found empty prefix on disk");
                 eprintln!("prefix: {}", pfx);
@@ -604,7 +605,8 @@ fn main() {
             }
             if values.len() > max_len {
                 max_len = values.len();
-                let recs = store.unwrap().get_records_for_prefix(&pfx, None);
+                let recs =
+                    store.unwrap().get_records_for_prefix(&pfx, None, false);
                 println!("LEN {} prefix: {}", max_len, pfx);
                 for rec in recs {
                     let pa = OwnedPathAttributes::from((
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index a0e2d865..98a34546 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -356,15 +356,18 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         record_map.len()
     }
 
-    pub fn get_record_for_active_mui(
+    pub fn get_record_for_mui(
         &self,
         mui: u32,
+        include_withdrawn: bool,
     ) -> Option<PublicRecord<M>> {
         let c_map = Arc::clone(&self.0);
         let record_map = c_map.lock().unwrap();
 
         record_map.get(&mui).and_then(|r| {
-            if r.route_status() == RouteStatus::Active {
+            if include_withdrawn {
+                Some(PublicRecord::from((mui, r)))
+            } else if r.route_status() == RouteStatus::Active {
                 Some(PublicRecord::from((mui, r)))
             } else {
                 None
@@ -407,10 +410,13 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     pub fn get_filtered_records(
         &self,
         mui: Option<u32>,
+        include_withdrawn: bool,
         bmin: &RoaringBitmap,
     ) -> Vec<PublicRecord<M>> {
         if let Some(mui) = mui {
-            self.get_record_for_active_mui(mui).into_iter().collect()
+            self.get_record_for_mui(mui, include_withdrawn)
+                .into_iter()
+                .collect()
         } else {
             self.as_active_records_not_in_bmin(bmin)
         }
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index c064f948..d4af5a50 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -278,6 +278,159 @@ impl<AF: AddressFamily> SizedPrefixIter<AF> {
     }
 }
 
+pub(crate) struct MoreSpecificPrefixOnlyIter<
+    'a,
+    AF: AddressFamily,
+    M: Meta,
+    NB: NodeBuckets<AF>,
+    PB: PrefixBuckets<AF, M>,
+    // const PREFIX_SIZE: usize,
+    // const KEY_SIZE: usize,
+> {
+    // rib: &'a Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
+    store: &'a TreeBitMap<AF, M, NB, PB>,
+    cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
+    cur_pfx_iter: SizedPrefixIter<AF>,
+    start_bit_span: BitSpan,
+    // skip_self: bool,
+    parent_and_position: Vec<SizedNodeMoreSpecificIter<AF>>,
+    // If specified, we're only iterating over records for this mui.
+    // mui: Option<u32>,
+    // This is the tree-wide index of withdrawn muis, used to rewrite the
+    // statuses of these records, or filter them out.
+    // global_withdrawn_bmin: &'a RoaringBitmap,
+    // Whether we should filter out the withdrawn records in the search result
+    // include_withdrawn: bool,
+}
+
+impl<
+        AF: AddressFamily,
+        M: Meta,
+        NB: NodeBuckets<AF>,
+        PB: PrefixBuckets<AF, M>,
+        // const PREFIX_SIZE: usize,
+        // const KEY_SIZE: usize,
+    > Iterator for MoreSpecificPrefixOnlyIter<'_, AF, M, NB, PB>
+//, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+{
+    type Item = PrefixId<AF>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        trace!("MoreSpecificsPrefixOnlyIter");
+
+        loop {
+            // first drain the current prefix iterator until empty.
+            let next_pfx = self.cur_pfx_iter.next();
+
+            if next_pfx.is_some() {
+                return next_pfx;
+            }
+
+            // Our current prefix iterator for this node is done, look for
+            // the next pfx iterator of the next child node in the current
+            // ptr iterator.
+            trace!("cur ptrbitarr {:?}", self.cur_ptr_iter);
+            trace!("start first ptr_iter");
+            let mut next_ptr = self.cur_ptr_iter.next();
+            trace!("new ptrbitarr {:?}", next_ptr);
+
+            // Our current ptr iterator is also done, maybe we have a parent
+            if next_ptr.is_none() {
+                trace!("try for parent");
+                if let Some(cur_ptr_iter) = self.parent_and_position.pop() {
+                    trace!("continue with parent");
+                    self.cur_ptr_iter = cur_ptr_iter;
+                    next_ptr = self.cur_ptr_iter.next();
+                } else {
+                    trace!("no more parents");
+                    return None;
+                }
+            }
+
+            if let Some(next_ptr) = next_ptr {
+                let node = self.store.retrieve_node(next_ptr);
+
+                match node {
+                    Some(SizedStrideRef::Stride3(next_node)) => {
+                        // copy the current iterator into the parent vec and create
+                        // a new ptr iterator for this node
+                        self.parent_and_position.push(self.cur_ptr_iter);
+                        let ptr_iter = next_node.more_specific_ptr_iter(
+                            next_ptr,
+                            BitSpan::new(0, 0),
+                        );
+                        self.cur_ptr_iter = ptr_iter.wrap();
+
+                        trace!(
+                            "next stride new iterator stride 3 {:?} start \
+                        bit_span {:?}",
+                            self.cur_ptr_iter,
+                            self.start_bit_span
+                        );
+                        self.cur_pfx_iter = next_node
+                            .more_specific_pfx_iter(
+                                next_ptr,
+                                BitSpan::new(0, 0),
+                                false,
+                            )
+                            .wrap();
+                    }
+                    Some(SizedStrideRef::Stride4(next_node)) => {
+                        // create new ptr iterator for this node.
+                        self.parent_and_position.push(self.cur_ptr_iter);
+                        let ptr_iter = next_node.more_specific_ptr_iter(
+                            next_ptr,
+                            BitSpan::new(0, 0),
+                        );
+                        self.cur_ptr_iter = ptr_iter.wrap();
+
+                        trace!(
+                            "next stride new iterator stride 4 {:?} start \
+                        bit_span {:?}",
+                            self.cur_ptr_iter,
+                            self.start_bit_span
+                        );
+                        self.cur_pfx_iter = next_node
+                            .more_specific_pfx_iter(
+                                next_ptr,
+                                BitSpan::new(0, 0),
+                                false,
+                            )
+                            .wrap();
+                    }
+                    Some(SizedStrideRef::Stride5(next_node)) => {
+                        // create new ptr iterator for this node.
+                        self.parent_and_position.push(self.cur_ptr_iter);
+                        let ptr_iter = next_node.more_specific_ptr_iter(
+                            next_ptr,
+                            BitSpan::new(0, 0),
+                        );
+                        self.cur_ptr_iter = ptr_iter.wrap();
+
+                        trace!(
+                            "next stride new iterator stride 5 {:?} start \
+                        bit_span {:?}",
+                            self.cur_ptr_iter,
+                            self.start_bit_span
+                        );
+                        self.cur_pfx_iter = next_node
+                            .more_specific_pfx_iter(
+                                next_ptr,
+                                BitSpan::new(0, 0),
+                                false,
+                            )
+                            .wrap();
+                    }
+                    None => {
+                        println!("no node here.");
+                        return None;
+                    }
+                };
+            }
+        }
+    }
+}
+
 // ----------- MoreSpecificPrefixIter ------------------------------------
 
 // A iterator over all the more-specifics for a given prefix.
@@ -372,8 +525,9 @@ impl<
                             {
                                 return Some((p.prefix, vec![rec]));
                             }
-                        } else if let Some(rec) =
-                            p.record_map.get_record_for_active_mui(mui)
+                        } else if let Some(rec) = p
+                            .record_map
+                            .get_record_for_mui(mui, self.include_withdrawn)
                         {
                             return Some((p.prefix, vec![rec]));
                         }
@@ -642,7 +796,7 @@ impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
                     } else {
                         stored_prefix
                             .record_map
-                            .get_record_for_active_mui(mui)
+                            .get_record_for_mui(mui, self.include_withdrawn)
                             .into_iter()
                             .collect()
                     }
@@ -788,6 +942,14 @@ impl<
                         );
                     }
                     SizedStrideRef::Stride4(n) => {
+                        trace!(
+                            "ALTERNATIVE {:#?}",
+                            n.add_more_specifics_at(
+                                start_bit_span.bits,
+                                start_bit_span.len,
+                                start_node_id
+                            )
+                        );
                         cur_pfx_iter = SizedPrefixIter::Stride4(
                             n.more_specific_pfx_iter(
                                 start_node_id,
@@ -839,6 +1001,106 @@ impl<
         .flatten()
     }
 
+    pub fn more_specific_prefix_only_iter_from(
+        &'a self,
+        start_prefix_id: PrefixId<AF>,
+    ) -> impl Iterator<Item = PrefixId<AF>> + 'a {
+        trace!("more specifics (prefix only) for {:?}", start_prefix_id);
+
+        // A v4 /32 or a v6 /128 doesn't have more specific prefixes 🤓.
+        if start_prefix_id.get_len() >= AF::BITS {
+            None
+        } else {
+            // calculate the node start_prefix_id lives in.
+            let (start_node_id, start_bit_span) =
+                self.get_node_id_for_prefix(&start_prefix_id);
+            trace!("start node {}", start_node_id);
+
+            trace!(
+                "start prefix id {:032b} (len {})",
+                start_prefix_id.get_net(),
+                start_prefix_id.get_len()
+            );
+            trace!(
+                "start node id   {:032b} (bits {} len {})",
+                start_node_id.get_id().0,
+                start_node_id.get_id().0,
+                start_node_id.get_len()
+            );
+            trace!(
+                "start bit span  {:032b} {}",
+                start_bit_span,
+                start_bit_span.bits
+            );
+            let cur_pfx_iter: SizedPrefixIter<AF>;
+            let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
+
+            let node = self.retrieve_node(start_node_id);
+
+            if let Some(node) = node {
+                match node {
+                    SizedStrideRef::Stride3(n) => {
+                        cur_pfx_iter = SizedPrefixIter::Stride3(
+                            n.more_specific_pfx_iter(
+                                start_node_id,
+                                start_bit_span,
+                                true,
+                            ),
+                        );
+                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
+                            n.more_specific_ptr_iter(
+                                start_node_id,
+                                start_bit_span,
+                            ),
+                        );
+                    }
+                    SizedStrideRef::Stride4(n) => {
+                        cur_pfx_iter = SizedPrefixIter::Stride4(
+                            n.more_specific_pfx_iter(
+                                start_node_id,
+                                start_bit_span,
+                                true,
+                            ),
+                        );
+                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride4(
+                            n.more_specific_ptr_iter(
+                                start_node_id,
+                                start_bit_span,
+                            ),
+                        );
+                    }
+                    SizedStrideRef::Stride5(n) => {
+                        cur_pfx_iter = SizedPrefixIter::Stride5(
+                            n.more_specific_pfx_iter(
+                                start_node_id,
+                                start_bit_span,
+                                true,
+                            ),
+                        );
+                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
+                            n.more_specific_ptr_iter(
+                                start_node_id,
+                                start_bit_span,
+                            ),
+                        );
+                    }
+                };
+
+                Some(MoreSpecificPrefixOnlyIter {
+                    store: self,
+                    cur_pfx_iter,
+                    cur_ptr_iter,
+                    start_bit_span,
+                    parent_and_position: vec![],
+                })
+            } else {
+                None
+            }
+        }
+        .into_iter()
+        .flatten()
+    }
+
     // Iterator over all less-specific prefixes, starting from the given
     // prefix at the given level and cursor.
     pub fn less_specific_prefix_iter(
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index 96c88d1e..a62be011 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -187,7 +187,7 @@ macro_rules! impl_primitive_atomic_stride {
                 const STRIDE_LEN: u8 = $len;
 
                 fn get_bit_pos(nibble: u32, len: u8) -> $pfxsize {
-                    trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
+                    // trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
                     1 << (
                             <Self as Stride>::BITS - ((1 << len) - 1) as u8
                             - nibble as u8 - 1
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 2f6e10dd..9adb4b44 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -419,6 +419,7 @@ where
     ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
         let pfxbitarr = self.pfxbitarr.load();
         let ptrbitarr = self.ptrbitarr.load();
+        println!("ptrbitarr {:064b}", ptrbitarr);
         // This is an exact match, so we're only considering the position of
         // the full nibble.
         let bit_pos = S::get_bit_pos(nibble, nibble_len);
@@ -638,7 +639,7 @@ where
         // till 4.
         // 
         // ex.:
-        // nibble: 1 , (nibble_len: 1)
+        // nibble: 1 , (nibble_len: 1)x
         // Iteration:
         // ms_nibble_len=1,n_l=0: 10, n_l=1: 11
         // ms_nibble_len=2,n_l=0: 100, n_l=1: 101, n_l=2: 110, n_l=3: 111
@@ -811,7 +812,8 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
         // Early exits
 
         // Empty bitmap
-        if self.ptrbitarr == <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType::zero() {
+        if self.ptrbitarr ==
+            <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType::zero() {
             trace!("empty ptrbitrarr. this iterator is done.");
             return None;
         }
@@ -819,9 +821,13 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
         // Previous iteration incremented the cursor beyond the stride size.
         if let Some(cursor) = self.cursor { 
             if cursor >= (1 << (S::STRIDE_LEN - self.start_bit_span.len)) {
-                trace!("cursor.bits >= (1 << (S::STRIDE_LEN - self.start_bit_span.len))");
+                trace!("cursor.bits >= (1 << (S::STRIDE_LEN - self.
+                    start_bit_span.len))");
                 trace!("cursor: {}", cursor);
-                trace!("start_bit_span: {} {}", self.start_bit_span.bits, self.start_bit_span.len);
+                trace!("start_bit_span: {} {}",
+                    self.start_bit_span.bits,
+                    self.start_bit_span.len
+                );
                 return None;
             }
         }
@@ -830,22 +836,30 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
         trace!("NodeMoreSpecificChildIter");
         trace!("base_prefix {}", self.base_prefix);
         trace!("stride_size {}", S::STRIDE_LEN);
-        trace!("start_bit_span bits {} len {} ", self.start_bit_span.bits, self.start_bit_span.len);
+        trace!("start_bit_span bits {} len {} ",
+            self.start_bit_span.bits, self.start_bit_span.len);
         trace!("cursor bits {:?}", self.cursor);
         trace!("          x1  4   8  12  16  20  24  28  32");
-        trace!("ptrbitarr {:032b}", self.ptrbitarr);
+        trace!("ptrbitarr {:032b}x", self.ptrbitarr);
 
-        let start = if let Some(bits) = self.cursor { bits } else { self.start_bit_span.bits };
+        let start = if let Some(bits) = self.cursor {
+            bits
+        } else {
+            self.start_bit_span.bits
+        };
         // We either stop if we have reached the maximum number of bits that
         // we should check for this bit_span or we stop at the end of the
         // stride (in case of a start_bit_span.bits == 0).
-        let stop = <u32>::min((1 << (S::STRIDE_LEN - self.start_bit_span.len)) + start, (1 << S::STRIDE_LEN) - 1); 
+        let stop = <u32>::min(
+            (1 << (S::STRIDE_LEN - self.start_bit_span.len)) + start,
+            (1 << S::STRIDE_LEN) - 1
+        ); 
 
         trace!("start {:?} stop {}", start, stop);
         for cursor in start..=stop {
-            // move the bit_span left with the amount of bits we're going to loop over.
-            // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
-            // becomes 0000 0000 0000 1100, then it will iterate over 
+            // move the bit_span left with the amount of bits we're going to
+            // loop over. e.g. a stride of size 4 with a nibble 0000 0000 0000
+            // 0011 becomes 0000 0000 0000 1100, then it will iterate over
             // ...1100,...1101,...1110,...1111
             let bit_pos = 
                 S::get_bit_pos(
@@ -858,8 +872,14 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
                 trace!("bingo!");
                 self.cursor = Some(cursor + 1);
 
-                trace!("next bit_span {} {} with cursor {:?}", self.start_bit_span.bits, self.start_bit_span.len, self.cursor);
-                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
+                trace!("next bit_span {} {} with cursor {:?}",
+                    self.start_bit_span.bits,
+                    self.start_bit_span.len,
+                    self.cursor
+                );
+                return Some(
+                    self.base_prefix.add_nibble(cursor, S::STRIDE_LEN)
+                );
             }
         }
         trace!("No more nodes. End of the iterator.");
@@ -885,8 +905,7 @@ impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride5> {
     }
 }
 
-
-// ----------- NodePrefixIter -----------------------------------------------
+// ----------- NodePrefixIter ------------------------------------------------
 
 // Create an iterator of all prefix ids hosted by this node.
 
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index 74b2f77d..7808a823 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -44,6 +44,7 @@ where
                         pfx.record_map
                             .get_filtered_records(
                                 options.mui,
+                                options.include_withdrawn,
                                 withdrawn_muis_bmin,
                             )
                             .into_iter()
@@ -116,6 +117,7 @@ where
                 None
             },
             more_specifics: if options.include_more_specifics {
+                println!("add more specifics");
                 Some(
                     self.more_specific_prefix_iter_from(
                         if let Some(pfx) = stored_prefix {
@@ -146,12 +148,19 @@ where
         start_node_id: StrideNodeId<AF>,
         found_pfx_vec: &mut Vec<PrefixId<AF>>,
     ) {
-        trace!("{:?}", self.retrieve_node(start_node_id));
+        trace!(
+            "get_all_more_specifics_for_node {:?}",
+            self.retrieve_node(start_node_id)
+        );
         match self.retrieve_node(start_node_id) {
             Some(SizedStrideRef::Stride3(n)) => {
                 found_pfx_vec.extend(
                     n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
                 );
+                trace!(
+                    "3 found_pfx no {:#?}",
+                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>()
+                );
 
                 for child_node in n.ptr_iter(start_node_id) {
                     self.get_all_more_specifics_for_node(
@@ -164,6 +173,10 @@ where
                 found_pfx_vec.extend(
                     n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
                 );
+                trace!(
+                    "4 found_pfx no {:#?}",
+                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>()
+                );
 
                 for child_node in n.ptr_iter(start_node_id) {
                     self.get_all_more_specifics_for_node(
@@ -176,6 +189,10 @@ where
                 found_pfx_vec.extend(
                     n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
                 );
+                trace!(
+                    "5 found_pfx no {:#?}",
+                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>()
+                );
 
                 for child_node in n.ptr_iter(start_node_id) {
                     self.get_all_more_specifics_for_node(
@@ -210,6 +227,7 @@ where
         for child_node in cnvec.iter() {
             self.get_all_more_specifics_for_node(*child_node, &mut msvec);
         }
+        trace!("MSVEC {:?}", msvec);
         Some(msvec)
     }
 
@@ -248,28 +266,28 @@ where
         // children of the root node. We, however, want the default prefix
         // which lives on the root node itself! We are *not* going to return
         // all of the prefixes in the tree as more-specifics.
-        if search_pfx.get_len() == 0 {
-            // match self.load_default_route_prefix_serial() {
-            //     0 => {
-            //         return QueryResult {
-            //             prefix: None,
-            //             prefix_meta: vec![],
-            //             match_type: MatchType::EmptyMatch,
-            //             less_specifics: None,
-            //             more_specifics: None,
-            //         };
-            //     }
-
-            //     _serial => {
-            return TreeQueryResult {
-                prefix: None,
-                match_type: MatchType::EmptyMatch,
-                less_specifics: None,
-                more_specifics: None,
-            };
-            // }
-            // }
-        }
+        // if search_pfx.get_len() == 0 {
+        // match self.load_default_route_prefix_serial() {
+        //     0 => {
+        //         return QueryResult {
+        //             prefix: None,
+        //             prefix_meta: vec![],
+        //             match_type: MatchType::EmptyMatch,
+        //             less_specifics: None,
+        //             more_specifics: None,
+        //         };
+        //     }
+
+        //     _serial => {
+        // return treequeryresult {
+        //     prefix: none,
+        //     match_type: matchtype::emptymatch,
+        //     less_specifics: none,
+        //     more_specifics: none,
+        // };
+        // }
+        // }
+        // }
 
         let mut stride_end = 0;
 
@@ -280,6 +298,8 @@ where
             _ => self.retrieve_node(root_node_id).unwrap(),
         };
 
+        trace!("RETRIEVED ROOT NODE {:#?}", node);
+
         let mut nibble;
         let mut nibble_len;
 
@@ -302,11 +322,12 @@ where
         };
 
         // The indexes of the more-specifics.
-        let mut more_specifics_vec = if options.include_more_specifics {
-            Some(Vec::<PrefixId<AF>>::new())
-        } else {
-            None
-        };
+        let mut more_specifics_vec: Vec<PrefixId<AF>> = vec![];
+        // let mut more_specifics_vec = if options.include_more_specifics {
+        //     Some(Vec::<PrefixId<AF>>::new())
+        // } else {
+        //     None
+        // };
 
         //---- Stride Processing --------------------------------------------
 
@@ -322,6 +343,7 @@ where
         // `post-processing` section.
 
         for stride in self.get_stride_sizes() {
+            println!("stride {}", stride);
             stride_end += stride;
 
             let last_stride = search_pfx.get_len() < stride_end;
@@ -386,7 +408,7 @@ where
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    more_specifics_vec = self
+                                    if let Some(v) = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -395,7 +417,10 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        );
+                                        )
+                                    {
+                                        more_specifics_vec.extend(v.iter());
+                                    }
                                 }
                                 break;
                             }
@@ -405,7 +430,7 @@ where
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    more_specifics_vec = self
+                                    if let Some(v) = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -414,7 +439,10 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        );
+                                        )
+                                    {
+                                        more_specifics_vec.extend(v);
+                                    }
                                 }
                                 break;
                             }
@@ -424,7 +452,7 @@ where
                         // node.
                         (None, Some(pfx_idx)) => {
                             if options.include_more_specifics {
-                                more_specifics_vec = self
+                                if let Some(v) = self
                                     .get_all_more_specifics_from_nibble(
                                         current_node,
                                         nibble,
@@ -433,7 +461,10 @@ where
                                             search_pfx.get_net(),
                                             stride_end - stride,
                                         ),
-                                    );
+                                    )
+                                {
+                                    more_specifics_vec.extend(v);
+                                }
                             }
                             match_prefix_idx = Some(pfx_idx);
                             break;
@@ -449,7 +480,7 @@ where
                                     // To make sure we don't process this
                                     // match arm more then once, we return
                                     // early here.
-                                    more_specifics_vec = self
+                                    if let Some(v) = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -458,7 +489,10 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        );
+                                        )
+                                    {
+                                        more_specifics_vec.extend(v);
+                                    }
 
                                     match_prefix_idx = None;
                                     break;
@@ -504,7 +538,7 @@ where
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    more_specifics_vec = self
+                                    if let Some(v) = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -513,7 +547,10 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        );
+                                        )
+                                    {
+                                        more_specifics_vec.extend(v.iter());
+                                    }
                                 }
                                 break;
                             }
@@ -523,7 +560,7 @@ where
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    more_specifics_vec = self
+                                    if let Some(v) = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -532,14 +569,17 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        );
+                                        )
+                                    {
+                                        more_specifics_vec.extend(v);
+                                    }
                                 }
                                 break;
                             }
                         }
                         (None, Some(pfx_idx)) => {
                             if options.include_more_specifics {
-                                more_specifics_vec = self
+                                if let Some(v) = self
                                     .get_all_more_specifics_from_nibble(
                                         current_node,
                                         nibble,
@@ -548,7 +588,10 @@ where
                                             search_pfx.get_net(),
                                             stride_end - stride,
                                         ),
-                                    );
+                                    )
+                                {
+                                    more_specifics_vec.extend(v);
+                                }
                             }
                             match_prefix_idx = Some(pfx_idx);
                             break;
@@ -559,7 +602,7 @@ where
                                     // To make sure we don't process this
                                     // match arm more then once, we return
                                     // early here.
-                                    more_specifics_vec = self
+                                    if let Some(v) = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -568,7 +611,10 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        );
+                                        )
+                                    {
+                                        more_specifics_vec.extend(v);
+                                    }
 
                                     match_prefix_idx = None;
                                     break;
@@ -583,6 +629,7 @@ where
                     }
                 }
                 SizedStrideRef::Stride5(current_node) => {
+                    println!("5 {:?}", options.match_type);
                     let search_fn = match options.match_type {
                         MatchType::ExactMatch => {
                             if options.include_less_specifics {
@@ -607,12 +654,13 @@ where
                         &mut less_specifics_vec,
                     ) {
                         (Some(n), Some(pfx_idx)) => {
+                            trace!("5 found node. found prefix");
                             match_prefix_idx = Some(pfx_idx);
                             node = self.retrieve_node(n).unwrap();
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    more_specifics_vec = self
+                                    if let Some(v) = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -621,17 +669,21 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        );
+                                        )
+                                    {
+                                        more_specifics_vec.extend(v);
+                                    }
                                 }
                                 break;
                             }
                         }
                         (Some(n), None) => {
+                            trace!("5 found a next node. No prefix here.");
                             node = self.retrieve_node(n).unwrap();
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    more_specifics_vec = self
+                                    if let Some(v) = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -640,14 +692,18 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        );
+                                        )
+                                    {
+                                        more_specifics_vec.extend(v);
+                                    }
                                 }
                                 break;
                             }
                         }
                         (None, Some(pfx_idx)) => {
+                            trace!("5 no next node. found prefix");
                             if options.include_more_specifics {
-                                more_specifics_vec = self
+                                if let Some(v) = self
                                     .get_all_more_specifics_from_nibble(
                                         current_node,
                                         nibble,
@@ -656,15 +712,19 @@ where
                                             search_pfx.get_net(),
                                             stride_end - stride,
                                         ),
-                                    );
+                                    )
+                                {
+                                    more_specifics_vec.extend(v);
+                                }
                             }
                             match_prefix_idx = Some(pfx_idx);
                             break;
                         }
                         (None, None) => {
+                            trace!("5 no next node. no prefix");
                             match options.match_type {
                                 MatchType::EmptyMatch => {
-                                    more_specifics_vec = self
+                                    if let Some(v) = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -673,8 +733,10 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        );
-
+                                        )
+                                    {
+                                        more_specifics_vec.extend(v);
+                                    }
                                     match_prefix_idx = None;
                                     break;
                                 }
@@ -723,7 +785,7 @@ where
                 None
             },
             more_specifics: if options.include_more_specifics {
-                more_specifics_vec
+                Some(more_specifics_vec)
             } else {
                 None
             },
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 039b9c09..6401dd58 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -1157,13 +1157,13 @@ impl<
             0
         };
         let this_level = <NB>::len_to_store_bits(id.get_id().1, level);
-        trace!("bits division {}", this_level);
-        trace!(
-            "calculated index ({} << {}) >> {}",
-            id.get_id().0,
-            last_level,
-            ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-        );
+        // trace!("bits division {}", this_level);
+        // trace!(
+        //     "calculated index ({} << {}) >> {}",
+        //     id.get_id().0,
+        //     last_level,
+        //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
+        // );
         // HASHING FUNCTION
         ((id.get_id().0 << last_level)
             >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index e1e1de4f..5e924c6c 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -4,6 +4,7 @@ use std::marker::PhantomData;
 use std::path::Path;
 
 use inetnum::addr::Prefix;
+use log::trace;
 use lsm_tree::{AbstractTree, KvPair};
 use roaring::RoaringBitmap;
 
@@ -52,7 +53,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     }
 
     pub fn remove(&self, key: [u8; KEY_SIZE]) {
-        self.tree.remove(key, 0);
+        self.tree.remove_weak(key, 0);
         // the last byte of the prefix holds the length of the prefix.
         self.counters.dec_prefixes_count(key[PREFIX_SIZE]);
     }
@@ -61,42 +62,168 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &self,
         prefix: PrefixId<AF>,
         mui: Option<u32>,
+        include_withdrawn: bool,
+        withdrawn_muis_bmin: &RoaringBitmap,
     ) -> Vec<PublicRecord<M>> {
-        if let Some(mui) = mui {
-            let prefix_b = Self::prefix_mui_persistence_key(prefix, mui);
-
-            (*self.tree.prefix(prefix_b))
-                .into_iter()
-                .map(|kv| {
-                    let kv = kv.unwrap();
-                    let (_, mui, ltime, status) =
-                        Self::parse_key(kv.0.as_ref());
-                    PublicRecord::new(
-                        mui,
-                        ltime,
-                        status,
-                        kv.1.as_ref().to_vec().into(),
-                    )
-                })
-                .collect::<Vec<_>>()
-        } else {
-            let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
-
-            (*self.tree.prefix(prefix_b))
-                .into_iter()
-                .map(|kv| {
-                    let kv = kv.unwrap();
-                    let (_, mui, ltime, status) =
-                        Self::parse_key(kv.0.as_ref());
-                    PublicRecord::new(
-                        mui,
-                        ltime,
-                        status,
-                        kv.1.as_ref().to_vec().into(),
-                    )
-                })
-                .collect::<Vec<_>>()
+        match (mui, include_withdrawn) {
+            // Specific mui, include withdrawn routes
+            (Some(mui), true) => {
+                // get the records from the persist store for the (prefix,
+                // mui) tuple only.
+                let prefix_b = Self::prefix_mui_persistence_key(prefix, mui);
+                (*self.tree.prefix(prefix_b))
+                    .into_iter()
+                    .map(|kv| {
+                        let kv = kv.unwrap();
+                        let (_, r_mui, ltime, mut status) =
+                            Self::parse_key(kv.0.as_ref());
+
+                        // If mui is in the global withdrawn muis table, then
+                        // rewrite the routestatus of the record to withdrawn.
+                        if withdrawn_muis_bmin.contains(r_mui) {
+                            status = RouteStatus::Withdrawn;
+                        }
+                        PublicRecord::new(
+                            mui,
+                            ltime,
+                            status,
+                            kv.1.as_ref().to_vec().into(),
+                        )
+                    })
+                    .collect::<Vec<_>>()
+            }
+            // Al muis, include withdrawn routes
+            (None, true) => {
+                // get all records for this prefix
+                let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
+                (*self.tree.prefix(prefix_b))
+                    .into_iter()
+                    .map(|kv| {
+                        let kv = kv.unwrap();
+                        let (_, r_mui, ltime, mut status) =
+                            Self::parse_key(kv.0.as_ref());
+
+                        // If mui is in the global withdrawn muis table, then
+                        // rewrite the routestatus of the record to withdrawn.
+                        if withdrawn_muis_bmin.contains(r_mui) {
+                            status = RouteStatus::Withdrawn;
+                        }
+                        PublicRecord::new(
+                            r_mui,
+                            ltime,
+                            status,
+                            kv.1.as_ref().to_vec().into(),
+                        )
+                    })
+                    .collect::<Vec<_>>()
+            }
+            // All muis, exclude withdrawn routes
+            (None, false) => {
+                // get all records for this prefix
+                let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
+                (*self.tree.prefix(prefix_b))
+                    .into_iter()
+                    .filter_map(|kv| {
+                        let kv = kv.unwrap();
+                        let (_, r_mui, ltime, status) =
+                            Self::parse_key(kv.0.as_ref());
+
+                        // If mui is in the global withdrawn muis table, then
+                        // skip this record
+                        if status == RouteStatus::Withdrawn
+                            || withdrawn_muis_bmin.contains(r_mui)
+                        {
+                            return None;
+                        }
+                        Some(PublicRecord::new(
+                            r_mui,
+                            ltime,
+                            status,
+                            kv.1.as_ref().to_vec().into(),
+                        ))
+                    })
+                    .collect::<Vec<_>>()
+            }
+            // Specific mui, exclude withdrawn routes
+            (Some(mui), false) => {
+                // get the records from the persist store for the (prefix,
+                // mui) tuple only.
+                let prefix_b = Self::prefix_mui_persistence_key(prefix, mui);
+                (*self.tree.prefix(prefix_b))
+                    .into_iter()
+                    .filter_map(|kv| {
+                        let kv = kv.unwrap();
+                        let (_, r_mui, ltime, status) =
+                            Self::parse_key(kv.0.as_ref());
+
+                        // If mui is in the global withdrawn muis table, then
+                        // skip this record
+                        if status == RouteStatus::Withdrawn
+                            || withdrawn_muis_bmin.contains(r_mui)
+                        {
+                            return None;
+                        }
+                        Some(PublicRecord::new(
+                            mui,
+                            ltime,
+                            status,
+                            kv.1.as_ref().to_vec().into(),
+                        ))
+                    })
+                    .collect::<Vec<_>>()
+            }
         }
+
+        // if let Some(mui) = mui {
+        //     let prefix_b = Self::prefix_mui_persistence_key(prefix, mui);
+
+        //     (*self.tree.prefix(prefix_b))
+        //         .into_iter()
+        //         .filter_map(|kv| {
+        //             let kv = kv.unwrap();
+        //             let (_, mui, ltime, mut status) =
+        //                 Self::parse_key(kv.0.as_ref());
+        //             if include_withdrawn {
+        //                 // If mui is in the global withdrawn muis table, then
+        //                 // rewrite the routestatus of the record to withdrawn.
+        //                 if withdrawn_muis_bmin.contains(mui) {
+        //                     status = RouteStatus::Withdrawn;
+        //                 }
+        //             // If the use does not want withdrawn routes then filter
+        //             // them out here.
+        //             } else if status == RouteStatus::Withdrawn {
+        //                 return None;
+        //             }
+        //             Some(PublicRecord::new(
+        //                 mui,
+        //                 ltime,
+        //                 status,
+        //                 kv.1.as_ref().to_vec().into(),
+        //             ))
+        //         })
+        //         .collect::<Vec<_>>()
+        // } else {
+        //     let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
+
+        //     (*self.tree.prefix(prefix_b))
+        //         .into_iter()
+        //         .map(|kv| {
+        //             let kv = kv.unwrap();
+        //             let (_, mui, ltime, status) =
+        //                 Self::parse_key(kv.0.as_ref());
+        //             if include_withdrawn || status != RouteStatus::Withdrawn {
+        //                 Some(PublicRecord::new(
+        //                     mui,
+        //                     ltime,
+        //                     status,
+        //                     kv.1.as_ref().to_vec().into(),
+        //                 ))
+        //             } else {
+        //                 None
+        //             }
+        //         })
+        //         .collect::<Vec<_>>()
+        // }
     }
 
     pub fn get_most_recent_record_for_prefix_mui<M: Meta>(
@@ -166,37 +293,29 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         self.tree.range(start.to_len_first_bytes()..end)
     }
 
-    fn enrich_prefix<M: Meta>(
-        &self,
-        prefix: Option<PrefixId<AF>>,
-        mui: Option<u32>,
-        include_withdrawn: bool,
-        bmin: &RoaringBitmap,
-    ) -> (Option<PrefixId<AF>>, Vec<PublicRecord<M>>) {
-        match prefix {
-            Some(pfx) => (
-                Some(pfx),
-                self.get_records_for_prefix(pfx, mui)
-                    .into_iter()
-                    .filter_map(|mut r| {
-                        if !include_withdrawn
-                            && r.status == RouteStatus::Withdrawn
-                        {
-                            return None;
-                        }
-                        if bmin.contains(r.multi_uniq_id) {
-                            if !include_withdrawn {
-                                return None;
-                            }
-                            r.status = RouteStatus::Withdrawn;
-                        }
-                        Some(r)
-                    })
-                    .collect(),
-            ),
-            None => (None, vec![]),
-        }
-    }
+    // fn enrich_prefix<M: Meta>(
+    //     &self,
+    //     prefix: PrefixId<AF>,
+    //     mui: Option<u32>,
+    //     include_withdrawn: bool,
+    //     bmin: &RoaringBitmap,
+    // ) -> Vec<PublicRecord<M>> {
+    //     self.get_records_for_prefix(prefix, mui, include_withdrawn, bmin)
+    // .into_iter()
+    // .filter_map(|mut r| {
+    //     if !include_withdrawn && r.status == RouteStatus::Withdrawn {
+    //         return None;
+    //     }
+    //     if bmin.contains(r.multi_uniq_id) {
+    //         if !include_withdrawn {
+    //             return None;
+    //         }
+    //         r.status = RouteStatus::Withdrawn;
+    //     }
+    //     Some(r)
+    // })
+    // .collect()
+    // }
 
     fn enrich_prefixes<M: Meta>(
         &self,
@@ -210,18 +329,23 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                 .flat_map(move |pfx| {
                     Some((
                         *pfx,
-                        self.get_records_for_prefix(*pfx, mui)
-                            .into_iter()
-                            .filter_map(|mut r| {
-                                if bmin.contains(r.multi_uniq_id) {
-                                    if !include_withdrawn {
-                                        return None;
-                                    }
-                                    r.status = RouteStatus::Withdrawn;
+                        self.get_records_for_prefix(
+                            *pfx,
+                            mui,
+                            include_withdrawn,
+                            bmin,
+                        )
+                        .into_iter()
+                        .filter_map(|mut r| {
+                            if bmin.contains(r.multi_uniq_id) {
+                                if !include_withdrawn {
+                                    return None;
                                 }
-                                Some(r)
-                            })
-                            .collect(),
+                                r.status = RouteStatus::Withdrawn;
+                            }
+                            Some(r)
+                        })
+                        .collect(),
                     ))
                 })
                 .collect()
@@ -243,17 +367,42 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         options: &MatchOptions,
         bmin: &RoaringBitmap,
     ) -> FamilyQueryResult<AF, M> {
-        let (prefix, prefix_meta) = self.enrich_prefix(
-            search_pfxs.prefix,
-            options.mui,
-            options.include_withdrawn,
-            bmin,
-        );
+        let (prefix, prefix_meta) = if let Some(prefix) = search_pfxs.prefix {
+            (
+                prefix,
+                self.get_records_for_prefix(
+                    prefix,
+                    options.mui,
+                    options.include_withdrawn,
+                    bmin,
+                ),
+            )
+        } else {
+            return FamilyQueryResult {
+                match_type: MatchType::EmptyMatch,
+                prefix: None,
+                prefix_meta: vec![],
+                less_specifics: if options.include_less_specifics {
+                    search_pfxs.less_specifics.map(|v| {
+                        v.into_iter().map(|p| (p, vec![])).collect::<Vec<_>>()
+                    })
+                } else {
+                    None
+                },
+                more_specifics: if options.include_more_specifics {
+                    search_pfxs.more_specifics.map(|v| {
+                        v.into_iter().map(|p| (p, vec![])).collect::<Vec<_>>()
+                    })
+                } else {
+                    None
+                },
+            };
+        };
 
         let mut res = match options.include_history {
             // All the records for all the prefixes
             IncludeHistory::All => FamilyQueryResult {
-                prefix,
+                prefix: Some(prefix),
                 prefix_meta,
                 match_type: search_pfxs.match_type,
                 less_specifics: self.enrich_prefixes(
@@ -264,7 +413,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                 ),
                 more_specifics: search_pfxs.more_specifics.map(|ms| {
                     self.more_specific_prefix_iter_from(
-                        prefix.unwrap(),
+                        prefix,
                         ms.iter().map(|p| p.get_len()).collect::<Vec<_>>(),
                         options.mui,
                         bmin,
@@ -278,7 +427,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             // attached. Not useful with the MemoryOnly strategy (historical
             // records are neve kept in memory).
             IncludeHistory::SearchPrefix => FamilyQueryResult {
-                prefix,
+                prefix: Some(prefix),
                 prefix_meta,
                 match_type: search_pfxs.match_type,
                 less_specifics: self
@@ -288,30 +437,35 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             },
             // Only the most recent record of the search prefix is returned
             // with the prefixes. This is used for the PersistOnly strategy.
-            IncludeHistory::None => FamilyQueryResult {
-                prefix,
-                prefix_meta,
-                match_type: search_pfxs.match_type,
-                less_specifics: search_pfxs.less_specifics.map(|ls| {
-                    self.less_specific_prefix_iter_from(
-                        ls,
-                        options.mui,
-                        bmin,
-                        options.include_withdrawn,
-                    )
-                    .collect::<Vec<_>>()
-                }),
-                more_specifics: search_pfxs.more_specifics.map(|ms| {
-                    self.more_specific_prefix_iter_from(
-                        prefix.unwrap(),
-                        ms.iter().map(|p| p.get_len()).collect::<Vec<_>>(),
-                        options.mui,
-                        bmin,
-                        options.include_withdrawn,
-                    )
-                    .collect::<Vec<_>>()
-                }),
-            },
+            IncludeHistory::None => {
+                println!("Include history: None");
+                FamilyQueryResult {
+                    prefix: Some(prefix),
+                    prefix_meta,
+                    match_type: search_pfxs.match_type,
+                    less_specifics: search_pfxs.less_specifics.map(|ls| {
+                        self.less_specific_prefix_iter_from(
+                            ls,
+                            options.mui,
+                            bmin,
+                            options.include_withdrawn,
+                        )
+                        .collect::<Vec<_>>()
+                    }),
+                    more_specifics: search_pfxs.more_specifics.map(|ms| {
+                        self.more_specific_prefix_iter_from(
+                            prefix,
+                            ms.iter()
+                                .map(|p| p.get_len())
+                                .collect::<Vec<_>>(),
+                            options.mui,
+                            bmin,
+                            options.include_withdrawn,
+                        )
+                        .collect::<Vec<_>>()
+                    }),
+                }
+            }
         };
 
         res.match_type = match (options.match_type, &res) {
@@ -474,11 +628,13 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         global_withdrawn_bmin: &'a RoaringBitmap,
         include_withdrawn: bool,
     ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        trace!("search more specifics in the persist store.");
         if search_lengths.is_empty() {
             for l in search_prefix.get_len() + 1..=AF::BITS {
                 search_lengths.push(l);
             }
         }
+        println!("more specific prefix lengths {:?}", search_lengths);
 
         let len = search_lengths.pop().unwrap();
         let cur_range = self
@@ -758,19 +914,22 @@ impl<
     fn next(&mut self) -> Option<Self::Item> {
         loop {
             if let Some(lp) = self.search_lengths.pop() {
-                let recs = self
-                    .store
-                    .get_records_for_prefix(lp, self.mui)
-                    .into_iter()
-                    .filter(|r| self.mui.is_none_or(|m| m == r.multi_uniq_id))
-                    .filter(|r| {
-                        self.include_withdrawn
-                            || (!self
-                                .global_withdrawn_bmin
-                                .contains(r.multi_uniq_id)
-                                && r.status != RouteStatus::Withdrawn)
-                    })
-                    .collect::<Vec<_>>();
+                let recs = self.store.get_records_for_prefix(
+                    lp,
+                    self.mui,
+                    self.include_withdrawn,
+                    self.global_withdrawn_bmin,
+                );
+                // .into_iter()
+                // .filter(|r| self.mui.is_none_or(|m| m == r.multi_uniq_id))
+                // .filter(|r| {
+                //     self.include_withdrawn
+                //         || (!self
+                //             .global_withdrawn_bmin
+                //             .contains(r.multi_uniq_id)
+                //             && r.status != RouteStatus::Withdrawn)
+                // })
+                // .collect::<Vec<_>>();
 
                 if !recs.is_empty() {
                     return Some((lp, recs));
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 5a995226..6005aaf8 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -26,6 +26,40 @@ where
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
 {
+    pub fn get_value(
+        &'a self,
+        prefix_id: PrefixId<AF>,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> Option<Vec<PublicRecord<M>>> {
+        match self.persist_strategy() {
+            PersistStrategy::WriteAhead => todo!(),
+            PersistStrategy::PersistHistory => todo!(),
+            PersistStrategy::MemoryOnly => self
+                .in_memory_tree
+                .non_recursive_retrieve_prefix(prefix_id)
+                .0
+                .map(|pfx| {
+                    pfx.record_map.get_filtered_records(
+                        mui,
+                        include_withdrawn,
+                        self.in_memory_tree.withdrawn_muis_bmin(guard),
+                    )
+                }),
+            PersistStrategy::PersistOnly => {
+                self.persist_tree.as_ref().map(|tree| {
+                    tree.get_records_for_prefix(
+                        prefix_id,
+                        mui,
+                        include_withdrawn,
+                        self.in_memory_tree.withdrawn_muis_bmin(guard),
+                    )
+                })
+            }
+        }
+    }
+
     pub fn more_specifics_from(
         &'a self,
         prefix_id: PrefixId<AF>,
@@ -33,9 +67,12 @@ where
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> QueryResult<M> {
-        let result =
-            self.in_memory_tree.non_recursive_retrieve_prefix(prefix_id);
-        let prefix = result.0;
+        let prefix = if !self.contains(prefix_id, mui) {
+            Some(Prefix::from(prefix_id))
+        } else {
+            None
+        };
+        // let prefix = result.0;
         let more_specifics_vec =
             self.in_memory_tree.more_specific_prefix_iter_from(
                 prefix_id,
@@ -45,23 +82,27 @@ where
             );
 
         QueryResult {
-            prefix: if let Some(pfx) = prefix {
-                Prefix::new(
-                    pfx.prefix.get_net().into_ipaddr(),
-                    pfx.prefix.get_len(),
-                )
-                .ok()
-            } else {
-                None
-            },
+            // prefix: if let Some(pfx) = prefix_id {
+            //     Prefix::new(
+            //         pfx.prefix.get_net().into_ipaddr(),
+            //         pfx.prefix.get_len(),
+            //     )
+            //     .ok()
+            // } else {
+            //     None
+            // },
+            prefix,
+            // prefix_meta: prefix_id.record_map.get_filtered_records(
+            //     mui,
+            //     include_withdrawn,
+            //     self.in_memory_tree.withdrawn_muis_bmin(guard),
+            // ),
             prefix_meta: prefix
-                .map(|r| {
-                    r.record_map.get_filtered_records(
-                        mui,
-                        self.in_memory_tree.withdrawn_muis_bmin(guard),
-                    )
+                .map(|_pfx| {
+                    self.get_value(prefix_id, mui, include_withdrawn, guard)
+                        .unwrap_or_default()
                 })
-                .unwrap_or_default(),
+                .unwrap_or(vec![]),
             match_type: MatchType::EmptyMatch,
             less_specifics: None,
             more_specifics: Some(more_specifics_vec.collect()),
@@ -75,45 +116,49 @@ where
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> QueryResult<M> {
-        let result =
-            self.in_memory_tree.non_recursive_retrieve_prefix(prefix_id);
-
-        let prefix = result.0;
-        let less_specifics_vec = result.1.map(
-            |(prefix_id, _level, _cur_set, _parents, _index)| {
-                self.in_memory_tree.less_specific_prefix_iter(
-                    prefix_id,
-                    mui,
-                    include_withdrawn,
-                    guard,
-                )
-            },
-        );
+        let prefix = if !self.contains(prefix_id, mui) {
+            Some(Prefix::from(prefix_id))
+        } else {
+            None
+        };
+
+        // let less_specifics_vec = result.1.map(
+        //     |(prefix_id, _level, _cur_set, _parents, _index)| {
+        let less_specifics_vec =
+            self.in_memory_tree.less_specific_prefix_iter(
+                prefix_id,
+                mui,
+                include_withdrawn,
+                guard,
+            );
 
         QueryResult {
-            prefix: if let Some(pfx) = prefix {
-                Prefix::new(
-                    pfx.prefix.get_net().into_ipaddr(),
-                    pfx.prefix.get_len(),
-                )
-                .ok()
-            } else {
-                None
-            },
-            prefix_meta: prefix
-                .map(|r| {
-                    r.record_map.get_filtered_records(
-                        mui,
-                        self.in_memory_tree.withdrawn_muis_bmin(guard),
-                    )
-                })
+            prefix,
+            // prefix_meta: prefix
+            //     .map(|r| {
+            //         r.record_map.get_filtered_records(
+            //             mui,
+            //             self.in_memory_tree.withdrawn_muis_bmin(guard),
+            //         )
+            //     })
+            //     .unwrap_or_default(),
+            prefix_meta: self
+                .get_value(prefix_id, mui, include_withdrawn, guard)
                 .unwrap_or_default(),
             match_type: MatchType::EmptyMatch,
-            less_specifics: less_specifics_vec.map(|iter| iter.collect()),
+            less_specifics: Some(less_specifics_vec.collect()),
             more_specifics: None,
         }
     }
 
+    pub fn more_specifics_keys_from(
+        &self,
+        prefix_id: PrefixId<AF>,
+    ) -> impl Iterator<Item = PrefixId<AF>> + '_ {
+        self.in_memory_tree
+            .more_specific_prefix_only_iter_from(prefix_id)
+    }
+
     pub fn more_specifics_iter_from(
         &'a self,
         prefix_id: PrefixId<AF>,
@@ -121,6 +166,7 @@ where
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        println!("more_specifics_iter_from fn");
         // If the user wanted a specific mui and not withdrawn prefixes, we
         // may return early if the mui is globally withdrawn.
         (if mui.is_some_and(|m| {
@@ -195,6 +241,7 @@ where
         options: &MatchOptions,
         guard: &'a Guard,
     ) -> QueryResult<M> {
+        println!("match_prefix rib");
         match self.config.persist_strategy() {
             // Everything is in memory only, so look there. There can be no
             // historical records for this variant, so we just return the
@@ -208,29 +255,34 @@ where
             // to memory. However the in-memory-tree is still used to indicate
             // which (prefix, mui) tuples have been created.
             PersistStrategy::PersistOnly => {
-                let withdrawn_muis_bmin =
-                    self.in_memory_tree.withdrawn_muis_bmin(guard);
                 // If no withdawn should be included and a specific mui was
                 // requested, then return early, if the mui lives in the
                 // global withdrawn muis.
                 if !options.include_withdrawn {
                     if let Some(mui) = options.mui {
+                        let withdrawn_muis_bmin =
+                            self.in_memory_tree.withdrawn_muis_bmin(guard);
                         if withdrawn_muis_bmin.contains(mui) {
                             return QueryResult::empty();
                         }
                     }
                 }
 
-                if options.match_type == MatchType::ExactMatch
-                    && !self.contains(search_pfx, options.mui)
-                {
-                    return QueryResult::empty();
-                }
+                // if options.match_type == MatchType::ExactMatch
+                //     && !self.contains(search_pfx, options.mui)
+                // {
+                //     return QueryResult::empty();
+                // }
 
                 if let Some(persist_tree) = &self.persist_tree {
+                    let withdrawn_muis_bmin =
+                        self.in_memory_tree.withdrawn_muis_bmin(guard);
+                    println!("persist store found");
+                    println!("mem record {:#?}", search_pfx);
                     let tbm_result = self
                         .in_memory_tree
                         .match_prefix_by_tree_traversal(search_pfx, options);
+                    println!("found by traversal {:#?}", tbm_result);
 
                     persist_tree
                         .match_prefix(
@@ -240,6 +292,7 @@ where
                         )
                         .into()
                 } else {
+                    println!("no persist store");
                     QueryResult::empty()
                 }
             }
@@ -257,6 +310,9 @@ where
                                     persist_tree.get_records_for_prefix(
                                         search_pfx,
                                         options.mui,
+                                        options.include_withdrawn,
+                                        self.in_memory_tree
+                                            .withdrawn_muis_bmin(guard),
                                     ),
                                 );
                             }
@@ -272,6 +328,11 @@ where
                                             .get_records_for_prefix(
                                                 rec.0,
                                                 options.mui,
+                                                options.include_withdrawn,
+                                                self.in_memory_tree
+                                                    .withdrawn_muis_bmin(
+                                                        guard,
+                                                    ),
                                             );
                                     }
                                 }
@@ -284,6 +345,11 @@ where
                                             .get_records_for_prefix(
                                                 rec.0,
                                                 options.mui,
+                                                options.include_withdrawn,
+                                                self.in_memory_tree
+                                                    .withdrawn_muis_bmin(
+                                                        guard,
+                                                    ),
                                             );
                                     }
                                 }
@@ -297,6 +363,9 @@ where
                                     persist_tree.get_records_for_prefix(
                                         search_pfx,
                                         options.mui,
+                                        options.include_withdrawn,
+                                        self.in_memory_tree
+                                            .withdrawn_muis_bmin(guard),
                                     ),
                                 );
                             }
@@ -324,6 +393,9 @@ where
                                 persist_tree.get_records_for_prefix(
                                     search_pfx,
                                     options.mui,
+                                    options.include_withdrawn,
+                                    self.in_memory_tree
+                                        .withdrawn_muis_bmin(guard),
                                 ),
                             );
 
@@ -339,6 +411,11 @@ where
                                                 .get_records_for_prefix(
                                                     rec.0,
                                                     options.mui,
+                                                    options.include_withdrawn,
+                                                    self.in_memory_tree
+                                                        .withdrawn_muis_bmin(
+                                                            guard,
+                                                        ),
                                                 ),
                                         );
                                     }
@@ -353,6 +430,11 @@ where
                                                 .get_records_for_prefix(
                                                     rec.0,
                                                     options.mui,
+                                                    options.include_withdrawn,
+                                                    self.in_memory_tree
+                                                        .withdrawn_muis_bmin(
+                                                            guard,
+                                                        ),
                                                 ),
                                         );
                                     }
@@ -367,6 +449,9 @@ where
                                     persist_tree.get_records_for_prefix(
                                         search_pfx,
                                         options.mui,
+                                        options.include_withdrawn,
+                                        self.in_memory_tree
+                                            .withdrawn_muis_bmin(guard),
                                     ),
                                 );
                             }
@@ -396,7 +481,7 @@ where
                     |mui| {
                         p_rec
                             .record_map
-                            .get_record_for_active_mui(mui)
+                            .get_record_for_mui(mui, false)
                             .ok_or(PrefixStoreError::StoreNotReadyError)
                     },
                 )
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index aa683a16..c4d9bf46 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -3,7 +3,8 @@ use crate::prelude::*;
 
 // The default stride sizes for IPv4, IPv6, resp.
 #[create_store((
-    ([5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 5, 18),
+    // ([5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 5, 18),
+    ([4, 4, 4, 4, 4, 4, 4, 4], 5, 18),
     ([4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
     4, 4, 4, 4, 4, 4, 4, 4, 4], 17, 30)
 ))]
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 8f41b27d..cc3cac5d 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -418,6 +418,7 @@ impl<
                         ltime,
                         RouteStatus::Withdrawn,
                     );
+
                     p_tree.insert(key, s.1.meta.as_ref());
 
                     // remove the entry for the same (prefix, mui), but with
@@ -440,7 +441,7 @@ impl<
 
                 // Use the record from the in-memory RIB to persist.
                 if let Some(_record) =
-                    stored_prefix.record_map.get_record_for_active_mui(mui)
+                    stored_prefix.record_map.get_record_for_mui(mui, true)
                 {
                     let p_tree =
                         if let Some(p_tree) = self.persist_tree.as_ref() {
@@ -538,7 +539,7 @@ impl<
 
                 // Use the record from the in-memory RIB to persist.
                 if let Some(_record) =
-                    stored_prefix.record_map.get_record_for_active_mui(mui)
+                    stored_prefix.record_map.get_record_for_mui(mui, true)
                 {
                     let p_tree =
                         if let Some(p_tree) = self.persist_tree.as_ref() {
@@ -712,9 +713,16 @@ impl<
         &self,
         prefix: &Prefix,
         mui: Option<u32>,
+        include_withdrawn: bool,
     ) -> Vec<PublicRecord<M>> {
         if let Some(p) = &self.persist_tree {
-            p.get_records_for_prefix(PrefixId::from(*prefix), mui)
+            let guard = epoch::pin();
+            p.get_records_for_prefix(
+                PrefixId::from(*prefix),
+                mui,
+                include_withdrawn,
+                self.in_memory_tree.withdrawn_muis_bmin(&guard),
+            )
         } else {
             vec![]
         }
diff --git a/src/macros.rs b/src/macros.rs
index d1efc4e9..f0c94061 100644
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -115,3 +115,76 @@ macro_rules! all_strategies {
         )*
     };
 }
+
+#[macro_export]
+#[doc(hidden)]
+macro_rules! all_strategies_arced {
+    ( $( $fn_name: ident; $test_name: ident; $ty: ty ), * ) => {
+
+        $(
+            #[test]
+            fn $fn_name() -> Result<(), Box<dyn std::error::Error>> {
+                //------- Default (MemoryOnly)
+
+                println!("default strategy starting...");
+                let tree_bitmap =
+                    MultiThreadedStore::<$ty>::try_default()?;
+
+                $test_name(Arc::new(tree_bitmap))?;
+
+                //------- PersistOnly
+
+                println!("PersistOnly strategy starting...");
+                let store_config = StoreConfig {
+                    persist_strategy:
+                        rotonda_store::rib::PersistStrategy::PersistOnly,
+                    persist_path: "/tmp/rotonda/".into(),
+                };
+
+                let tree_bitmap = MultiThreadedStore::<
+                    $ty,
+                >::new_with_config(
+                    store_config
+                )?;
+
+                $test_name(Arc::new(tree_bitmap))?;
+
+                //------- PersistHistory
+
+                println!("PersistHistory strategy starting...");
+                let store_config = StoreConfig {
+                    persist_strategy:
+                        rotonda_store::rib::PersistStrategy::PersistHistory,
+                    persist_path: "/tmp/rotonda/".into(),
+                };
+
+                let tree_bitmap = MultiThreadedStore::<
+                    $ty,
+                >::new_with_config(
+                    store_config
+                )?;
+
+                $test_name(Arc::new(tree_bitmap))?;
+
+                //------- WriteAhead
+
+                println!("WriteAhead strategy starting...");
+                let store_config = StoreConfig {
+                    persist_strategy:
+                        rotonda_store::rib::PersistStrategy::WriteAhead,
+                    persist_path: "/tmp/rotonda/".into(),
+                };
+
+                let tree_bitmap = MultiThreadedStore::<
+                    $ty,
+                >::new_with_config(
+                    store_config
+                )?;
+
+                $test_name(Arc::new(tree_bitmap))?;
+
+                Ok(())
+            }
+        )*
+    };
+}
diff --git a/src/meta_examples.rs b/src/meta_examples.rs
index 4cc482ba..1828fea2 100644
--- a/src/meta_examples.rs
+++ b/src/meta_examples.rs
@@ -2,7 +2,7 @@
 
 use inetnum::asn::Asn;
 
-use crate::Meta;
+use crate::{Meta, PublicRecord};
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub struct PrefixAs([u8; 4]);
diff --git a/src/test_types.rs b/src/test_types.rs
index 85d47c25..b044db3b 100644
--- a/src/test_types.rs
+++ b/src/test_types.rs
@@ -13,7 +13,12 @@ impl AsRef<[u8]> for BeBytesAsn {
 
 impl From<Vec<u8>> for BeBytesAsn {
     fn from(value: Vec<u8>) -> Self {
-        Self(*value.first_chunk::<4>().unwrap())
+        println!("value {:#?}", value);
+        if let Some(value) = value.first_chunk::<4>() {
+            Self(*value)
+        } else {
+            Self([0; 4])
+        }
     }
 }
 
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index f2b7f76e..6d2eef4f 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -1,9 +1,13 @@
-use std::{str::FromStr, sync::atomic::Ordering};
+use std::{
+    str::FromStr,
+    sync::{atomic::Ordering, Arc},
+};
 
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
-    prelude::multi::RouteStatus, rib::StoreConfig, test_types::BeBytesAsn,
-    IncludeHistory, MatchOptions, MultiThreadedStore, PublicRecord as Record,
+    meta_examples::NoMeta, prelude::multi::RouteStatus, rib::StoreConfig,
+    test_types::BeBytesAsn, IncludeHistory, MatchOptions, MultiThreadedStore,
+    PublicRecord as Record,
 };
 
 mod common {
@@ -17,49 +21,14 @@ mod common {
     }
 }
 
-#[test]
-fn test_concurrent_updates_1_multiple_persist_scenarios(
-) -> Result<(), Box<dyn std::error::Error>> {
-    //------- Default (MemoryOnly)
-
-    println!("default strategy starting...");
-    let tree_bitmap = MultiThreadedStore::<BeBytesAsn>::try_default()?;
-
-    test_concurrent_updates_1(std::sync::Arc::new(tree_bitmap))?;
-
-    //------- PersistOnly
-
-    println!("PersistOnly strategy starting...");
-    let store_config = StoreConfig {
-        persist_strategy: rotonda_store::rib::PersistStrategy::PersistOnly,
-        persist_path: "/tmp/rotonda/".into(),
-    };
-
-    let tree_bitmap = std::sync::Arc::new(
-        MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
-    );
-
-    test_concurrent_updates_1(tree_bitmap)?;
-
-    //------- PersistHistory
-
-    println!("PersistHistory strategy starting...");
-    let store_config = StoreConfig {
-        persist_strategy: rotonda_store::rib::PersistStrategy::PersistHistory,
-        persist_path: "/tmp/rotonda/".into(),
-    };
-
-    let tree_bitmap = std::sync::Arc::new(
-        MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
-    );
-
-    test_concurrent_updates_1(tree_bitmap)?;
-
-    Ok(())
-}
+rotonda_store::all_strategies![
+    test_cc_updates_1;
+    test_concurrent_updates_1;
+    BeBytesAsn
+];
 
 fn test_concurrent_updates_1(
-    store: std::sync::Arc<MultiThreadedStore<BeBytesAsn>>,
+    tree_bitmap: MultiThreadedStore<BeBytesAsn>,
 ) -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
@@ -118,12 +87,12 @@ fn test_concurrent_updates_1(
     };
 
     let cur_ltime = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
-
+    let tree_bitmap = std::sync::Arc::new(tree_bitmap);
     let _: Vec<_> = vec![mui_data_1, mui_data_2, mui_data_3]
         .into_iter()
         .map(|data: MuiData| {
-            let tree_bitmap = store.clone();
             let cur_ltime = cur_ltime.clone();
+            let tbm = tree_bitmap.clone();
 
             std::thread::Builder::new()
                 .name(data.mui.to_string())
@@ -132,7 +101,7 @@ fn test_concurrent_updates_1(
                     for pfx in data.pfxs {
                         let _ = cur_ltime.fetch_add(1, Ordering::Release);
 
-                        match tree_bitmap.insert(
+                        match tbm.insert(
                             &pfx,
                             Record::new(
                                 data.mui,
@@ -156,15 +125,15 @@ fn test_concurrent_updates_1(
         .map(|t| t.join())
         .collect();
 
-    println!("COUNT {}", store.prefixes_count());
+    println!("COUNT {:?}", tree_bitmap.prefixes_count());
 
-    let all_pfxs_iter = store.prefixes_iter().collect::<Vec<_>>();
+    let all_pfxs_iter = tree_bitmap.prefixes_iter().collect::<Vec<_>>();
 
     let pfx = Prefix::from_str("185.34.0.0/16").unwrap();
 
-    assert!(store.contains(&pfx, None));
-    assert!(store.contains(&pfx, Some(1)));
-    assert!(store.contains(&pfx, Some(2)));
+    assert!(tree_bitmap.contains(&pfx, None));
+    assert!(tree_bitmap.contains(&pfx, Some(1)));
+    assert!(tree_bitmap.contains(&pfx, Some(2)));
 
     assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
@@ -356,7 +325,7 @@ fn test_concurrent_updates_1(
         .clone()
         .into_iter()
         .map(|pfx: Prefix| {
-            let tree_bitmap = store.clone();
+            let tree_bitmap = tree_bitmap.clone();
             let cur_ltime = cur_ltime.clone();
 
             std::thread::Builder::new()
@@ -376,7 +345,10 @@ fn test_concurrent_updates_1(
         .map(|t| t.join())
         .collect();
 
-    println!("{:#?}", store.prefixes_iter().collect::<Vec<_>>());
+    println!(
+        "prefixes_iter {:#?}",
+        tree_bitmap.as_ref().prefixes_iter().collect::<Vec<_>>()
+    );
 
     let match_options = MatchOptions {
         match_type: rotonda_store::MatchType::ExactMatch,
@@ -389,7 +361,7 @@ fn test_concurrent_updates_1(
 
     for pfx in pfx_vec_2 {
         let guard = rotonda_store::epoch::pin();
-        let res = store.match_prefix(&pfx, &match_options, &guard);
+        let res = tree_bitmap.match_prefix(&pfx, &match_options, &guard);
         assert_eq!(res.prefix, Some(pfx));
         println!("PFX {:?}", res);
         assert_eq!(
@@ -404,8 +376,16 @@ fn test_concurrent_updates_1(
     Ok(())
 }
 
-#[test]
-fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
+rotonda_store::all_strategies_arced![
+    test_cc_updates_2;
+    test_concurrent_updates_2;
+    BeBytesAsn
+];
+
+// #[test]
+fn test_concurrent_updates_2(
+    tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>,
+) -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
     let pfx_vec_1 = vec![
@@ -429,19 +409,25 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
         Prefix::from_str("188.0.0.0/8")?,
     ];
 
-    let tree_bitmap =
-        std::sync::Arc::new(MultiThreadedStore::<BeBytesAsn>::try_default()?);
-
     const MUI_DATA: [u32; 4] = [65501, 65502, 65503, 65504];
 
     let cur_ltime = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
 
+    println!("PersistOnly strategy starting...");
+    let store_config = StoreConfig {
+        persist_strategy: rotonda_store::rib::PersistStrategy::MemoryOnly,
+        persist_path: "/tmp/rotonda/".into(),
+    };
+    // let tree_bitmap = std::sync::Arc::new(
+    //     MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
+    // );
+
     let _: Vec<_> =
         vec![pfx_vec_1.clone(), pfx_vec_2.clone(), pfx_vec_3.clone()]
             .into_iter()
             .enumerate()
             .map(|(n, pfxs)| {
-                let tree_bitmap = tree_bitmap.clone();
+                let tbm = std::sync::Arc::clone(&tree_bitmap);
                 let cur_ltime = cur_ltime.clone();
 
                 std::thread::Builder::new()
@@ -451,7 +437,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
                         for (i, pfx) in pfxs.iter().enumerate() {
                             let _ = cur_ltime.fetch_add(1, Ordering::Release);
 
-                            match tree_bitmap.insert(
+                            match tbm.insert(
                                 pfx,
                                 Record::new(
                                     i as u32 + 1,
@@ -463,7 +449,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
                             ) {
                                 Ok(_) => {}
                                 Err(e) => {
-                                    println!("{}", e);
+                                    println!("Err: {}", e);
                                 }
                             };
                         }
@@ -475,7 +461,10 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
             .map(|t| t.join())
             .collect();
 
-    println!("{:#?}", tree_bitmap.prefixes_iter().collect::<Vec<_>>());
+    println!(
+        "prefixes_iter#1 :{:#?}",
+        tree_bitmap.prefixes_iter().collect::<Vec<_>>()
+    );
 
     let all_pfxs_iter = tree_bitmap.prefixes_iter().collect::<Vec<_>>();
 
@@ -626,7 +615,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
     let _: Vec<_> = wd_pfxs
         .into_iter()
         .map(|pfx: Prefix| {
-            let tree_bitmap = tree_bitmap.clone();
+            let tbm = std::sync::Arc::clone(&tree_bitmap);
             let cur_ltime = cur_ltime.clone();
 
             std::thread::Builder::new()
@@ -635,8 +624,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
                     print!("\nstart withdraw {} ---", 2);
 
                     let _ = cur_ltime.fetch_add(1, Ordering::Release);
-                    tree_bitmap
-                        .mark_mui_as_withdrawn_for_prefix(&pfx, 2, 11)
+                    tbm.mark_mui_as_withdrawn_for_prefix(&pfx, 2, 15)
                         .unwrap();
 
                     println!("--thread withdraw 2 done.");
@@ -659,6 +647,7 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
         let guard = rotonda_store::epoch::pin();
         let res = tree_bitmap.match_prefix(&pfx, &match_options, &guard);
         assert_eq!(res.prefix, Some(pfx));
+        println!("RES {:#?}", res);
         assert_eq!(
             res.prefix_meta
                 .iter()
@@ -669,8 +658,9 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
         );
     }
 
+    println!("get all prefixes");
     let match_options = MatchOptions {
-        match_type: rotonda_store::MatchType::ExactMatch,
+        match_type: rotonda_store::MatchType::EmptyMatch,
         include_withdrawn: false,
         include_less_specifics: false,
         include_more_specifics: true,
@@ -678,11 +668,53 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
         include_history: IncludeHistory::None,
     };
 
-    let pfx = Prefix::from_str("0.0.0.0/0").unwrap();
+    // let pfx0 = Prefix::from_str("128.0.0.0/2").unwrap();
+    let pfx128 = Prefix::from_str("0.0.0.0/0").unwrap();
     let guard = rotonda_store::epoch::pin();
-    let res = tree_bitmap.match_prefix(&pfx, &match_options, &guard);
+    // let res0 = tree_bitmap.match_prefix(&pfx0, &match_options, &guard);
+    let res128 = tree_bitmap.match_prefix(&pfx128, &match_options, &guard);
+
+    // println!("000 {:#?}", res0);
+    println!("128 {:#?}", res128);
+
+    assert!(tree_bitmap
+        .contains(&Prefix::from_str("185.34.14.0/24").unwrap(), None));
+
+    tree_bitmap
+        .insert(
+            &Prefix::from_str("32.0.0.0/4").unwrap(),
+            Record::new(
+                1,
+                cur_ltime.load(Ordering::Acquire),
+                RouteStatus::Active,
+                Asn::from(653400).into(),
+            ),
+            None,
+        )
+        .unwrap();
+
+    assert!(
+        tree_bitmap.contains(&Prefix::from_str("32.0.0.0/4").unwrap(), None)
+    );
 
-    println!("{:#?}", res);
+    assert_eq!(
+        tree_bitmap
+            .match_prefix(
+                &Prefix::from_str("0.0.0.0/2").unwrap(),
+                &match_options,
+                &guard
+            )
+            .more_specifics
+            .unwrap()
+            .len(),
+        1
+    );
+
+    // let guard = rotonda_store::epoch::pin();
+    // println!(
+    //     "more_specifics_iter_from {:#?}",
+    //     tree_bitmap.more_specifics_keys_from(&pfx128)
+    // );
 
     let active_len = all_pfxs_iter
         .iter()
@@ -690,8 +722,57 @@ fn test_concurrent_updates_2() -> Result<(), Box<dyn std::error::Error>> {
         .collect::<Vec<_>>()
         .len();
     assert_eq!(active_len, all_pfxs_iter.len());
-    let len_2 = res.more_specifics.unwrap().v4.len();
+    let len_2 = res128.more_specifics.unwrap().v4.len();
+    // + res128.more_specifics.unwrap().v4.len();
     assert_eq!(active_len, len_2);
 
     Ok(())
 }
+
+#[test]
+fn shit_test() -> Result<(), Box<dyn std::error::Error>> {
+    crate::common::init();
+
+    println!("PersistOnly strategy starting...");
+    let store_config = StoreConfig {
+        persist_strategy: rotonda_store::rib::PersistStrategy::MemoryOnly,
+        persist_path: "/tmp/rotonda/".into(),
+    };
+    let tree_bitmap = std::sync::Arc::new(
+        MultiThreadedStore::<NoMeta>::new_with_config(store_config)?,
+    );
+
+    let pfx1 = Prefix::from_str("185.34.0.0/16")?;
+    let pfx2 = Prefix::from_str("185.34.3.0/24")?;
+    // let search_pfx = Prefix::from_str("128.0.0.0/1")?;
+    let search_pfx = Prefix::from_str("0.0.0.0/0")?;
+
+    tree_bitmap
+        .insert(
+            &pfx1,
+            Record::new(1, 0, RouteStatus::Active, NoMeta::Empty),
+            None,
+        )
+        .unwrap();
+
+    tree_bitmap
+        .insert(
+            &pfx2,
+            Record::new(1, 0, RouteStatus::Active, NoMeta::Empty),
+            None,
+        )
+        .unwrap();
+
+    println!("-------------------");
+
+    let guard = rotonda_store::epoch::pin();
+    let mp = tree_bitmap
+        .more_specifics_iter_from(&search_pfx, None, false, &guard)
+        .collect::<Vec<_>>();
+
+    println!("more specifics : {:#?}", mp);
+
+    assert_eq!(mp.len(), 2);
+
+    Ok(())
+}

From e7b87ac463a571b5bccbfa59219e387fc9b99204 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 22 Jan 2025 15:30:11 +0100
Subject: [PATCH 051/147] add BitSpan from bit pos index method

---
 src/local_array/bit_span.rs        |  32 ++++--
 src/local_array/in_memory/query.rs | 170 +++++++++--------------------
 2 files changed, 77 insertions(+), 125 deletions(-)

diff --git a/src/local_array/bit_span.rs b/src/local_array/bit_span.rs
index 480281dd..45276c86 100644
--- a/src/local_array/bit_span.rs
+++ b/src/local_array/bit_span.rs
@@ -1,33 +1,47 @@
 #[derive(Copy, Clone, Debug)]
 pub(crate) struct BitSpan {
     pub bits: u32,
-    pub len: u8
+    pub len: u8,
 }
 
 impl BitSpan {
     pub(crate) fn new(bits: u32, len: u8) -> Self {
-        Self {
-            bits,
-            len
-        }
+        Self { bits, len }
     }
 
     // Increment the bit span by one and calculate the new length.
     #[allow(dead_code)]
     pub(crate) fn inc(&mut self) {
         self.bits += 1;
-        self.len = <u8>::max(self.len, (32 - self.bits.leading_zeros()) as u8);
+        self.len =
+            <u8>::max(self.len, (32 - self.bits.leading_zeros()) as u8);
     }
 
     #[allow(dead_code)]
     pub(crate) fn set_len_to_bits(&mut self) {
-        self.len = <u8>::max(self.len, (32 - self.bits.leading_zeros()) as u8);
+        self.len =
+            <u8>::max(self.len, (32 - self.bits.leading_zeros()) as u8);
+    }
+
+    // Deep, dark, black magic. Calculate the bit span from the index in a
+    // bitarr. This is used by iterators, so they can have one sequential i
+    // loop, that goes over all positions in a bitarr by its indexes.
+    pub fn from_bit_pos_index(mut i: u8) -> Self {
+        let bits = i as u32;
+        i += 1;
+        i |= i >> 1;
+        i |= i >> 2;
+        i |= i >> 3;
+        i = (i >> 1).count_ones() as u8;
+        Self {
+            bits: bits - ((1 << i as u32) - 1),
+            len: i,
+        }
     }
-    
 }
 
 impl std::fmt::Binary for BitSpan {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{:032b} (len {})", self.bits, self.len)
     }
-}
\ No newline at end of file
+}
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index 7808a823..74b2f77d 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -44,7 +44,6 @@ where
                         pfx.record_map
                             .get_filtered_records(
                                 options.mui,
-                                options.include_withdrawn,
                                 withdrawn_muis_bmin,
                             )
                             .into_iter()
@@ -117,7 +116,6 @@ where
                 None
             },
             more_specifics: if options.include_more_specifics {
-                println!("add more specifics");
                 Some(
                     self.more_specific_prefix_iter_from(
                         if let Some(pfx) = stored_prefix {
@@ -148,19 +146,12 @@ where
         start_node_id: StrideNodeId<AF>,
         found_pfx_vec: &mut Vec<PrefixId<AF>>,
     ) {
-        trace!(
-            "get_all_more_specifics_for_node {:?}",
-            self.retrieve_node(start_node_id)
-        );
+        trace!("{:?}", self.retrieve_node(start_node_id));
         match self.retrieve_node(start_node_id) {
             Some(SizedStrideRef::Stride3(n)) => {
                 found_pfx_vec.extend(
                     n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
                 );
-                trace!(
-                    "3 found_pfx no {:#?}",
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>()
-                );
 
                 for child_node in n.ptr_iter(start_node_id) {
                     self.get_all_more_specifics_for_node(
@@ -173,10 +164,6 @@ where
                 found_pfx_vec.extend(
                     n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
                 );
-                trace!(
-                    "4 found_pfx no {:#?}",
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>()
-                );
 
                 for child_node in n.ptr_iter(start_node_id) {
                     self.get_all_more_specifics_for_node(
@@ -189,10 +176,6 @@ where
                 found_pfx_vec.extend(
                     n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
                 );
-                trace!(
-                    "5 found_pfx no {:#?}",
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>()
-                );
 
                 for child_node in n.ptr_iter(start_node_id) {
                     self.get_all_more_specifics_for_node(
@@ -227,7 +210,6 @@ where
         for child_node in cnvec.iter() {
             self.get_all_more_specifics_for_node(*child_node, &mut msvec);
         }
-        trace!("MSVEC {:?}", msvec);
         Some(msvec)
     }
 
@@ -266,28 +248,28 @@ where
         // children of the root node. We, however, want the default prefix
         // which lives on the root node itself! We are *not* going to return
         // all of the prefixes in the tree as more-specifics.
-        // if search_pfx.get_len() == 0 {
-        // match self.load_default_route_prefix_serial() {
-        //     0 => {
-        //         return QueryResult {
-        //             prefix: None,
-        //             prefix_meta: vec![],
-        //             match_type: MatchType::EmptyMatch,
-        //             less_specifics: None,
-        //             more_specifics: None,
-        //         };
-        //     }
-
-        //     _serial => {
-        // return treequeryresult {
-        //     prefix: none,
-        //     match_type: matchtype::emptymatch,
-        //     less_specifics: none,
-        //     more_specifics: none,
-        // };
-        // }
-        // }
-        // }
+        if search_pfx.get_len() == 0 {
+            // match self.load_default_route_prefix_serial() {
+            //     0 => {
+            //         return QueryResult {
+            //             prefix: None,
+            //             prefix_meta: vec![],
+            //             match_type: MatchType::EmptyMatch,
+            //             less_specifics: None,
+            //             more_specifics: None,
+            //         };
+            //     }
+
+            //     _serial => {
+            return TreeQueryResult {
+                prefix: None,
+                match_type: MatchType::EmptyMatch,
+                less_specifics: None,
+                more_specifics: None,
+            };
+            // }
+            // }
+        }
 
         let mut stride_end = 0;
 
@@ -298,8 +280,6 @@ where
             _ => self.retrieve_node(root_node_id).unwrap(),
         };
 
-        trace!("RETRIEVED ROOT NODE {:#?}", node);
-
         let mut nibble;
         let mut nibble_len;
 
@@ -322,12 +302,11 @@ where
         };
 
         // The indexes of the more-specifics.
-        let mut more_specifics_vec: Vec<PrefixId<AF>> = vec![];
-        // let mut more_specifics_vec = if options.include_more_specifics {
-        //     Some(Vec::<PrefixId<AF>>::new())
-        // } else {
-        //     None
-        // };
+        let mut more_specifics_vec = if options.include_more_specifics {
+            Some(Vec::<PrefixId<AF>>::new())
+        } else {
+            None
+        };
 
         //---- Stride Processing --------------------------------------------
 
@@ -343,7 +322,6 @@ where
         // `post-processing` section.
 
         for stride in self.get_stride_sizes() {
-            println!("stride {}", stride);
             stride_end += stride;
 
             let last_stride = search_pfx.get_len() < stride_end;
@@ -408,7 +386,7 @@ where
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    if let Some(v) = self
+                                    more_specifics_vec = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -417,10 +395,7 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        )
-                                    {
-                                        more_specifics_vec.extend(v.iter());
-                                    }
+                                        );
                                 }
                                 break;
                             }
@@ -430,7 +405,7 @@ where
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    if let Some(v) = self
+                                    more_specifics_vec = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -439,10 +414,7 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        )
-                                    {
-                                        more_specifics_vec.extend(v);
-                                    }
+                                        );
                                 }
                                 break;
                             }
@@ -452,7 +424,7 @@ where
                         // node.
                         (None, Some(pfx_idx)) => {
                             if options.include_more_specifics {
-                                if let Some(v) = self
+                                more_specifics_vec = self
                                     .get_all_more_specifics_from_nibble(
                                         current_node,
                                         nibble,
@@ -461,10 +433,7 @@ where
                                             search_pfx.get_net(),
                                             stride_end - stride,
                                         ),
-                                    )
-                                {
-                                    more_specifics_vec.extend(v);
-                                }
+                                    );
                             }
                             match_prefix_idx = Some(pfx_idx);
                             break;
@@ -480,7 +449,7 @@ where
                                     // To make sure we don't process this
                                     // match arm more then once, we return
                                     // early here.
-                                    if let Some(v) = self
+                                    more_specifics_vec = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -489,10 +458,7 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        )
-                                    {
-                                        more_specifics_vec.extend(v);
-                                    }
+                                        );
 
                                     match_prefix_idx = None;
                                     break;
@@ -538,7 +504,7 @@ where
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    if let Some(v) = self
+                                    more_specifics_vec = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -547,10 +513,7 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        )
-                                    {
-                                        more_specifics_vec.extend(v.iter());
-                                    }
+                                        );
                                 }
                                 break;
                             }
@@ -560,7 +523,7 @@ where
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    if let Some(v) = self
+                                    more_specifics_vec = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -569,17 +532,14 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        )
-                                    {
-                                        more_specifics_vec.extend(v);
-                                    }
+                                        );
                                 }
                                 break;
                             }
                         }
                         (None, Some(pfx_idx)) => {
                             if options.include_more_specifics {
-                                if let Some(v) = self
+                                more_specifics_vec = self
                                     .get_all_more_specifics_from_nibble(
                                         current_node,
                                         nibble,
@@ -588,10 +548,7 @@ where
                                             search_pfx.get_net(),
                                             stride_end - stride,
                                         ),
-                                    )
-                                {
-                                    more_specifics_vec.extend(v);
-                                }
+                                    );
                             }
                             match_prefix_idx = Some(pfx_idx);
                             break;
@@ -602,7 +559,7 @@ where
                                     // To make sure we don't process this
                                     // match arm more then once, we return
                                     // early here.
-                                    if let Some(v) = self
+                                    more_specifics_vec = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -611,10 +568,7 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        )
-                                    {
-                                        more_specifics_vec.extend(v);
-                                    }
+                                        );
 
                                     match_prefix_idx = None;
                                     break;
@@ -629,7 +583,6 @@ where
                     }
                 }
                 SizedStrideRef::Stride5(current_node) => {
-                    println!("5 {:?}", options.match_type);
                     let search_fn = match options.match_type {
                         MatchType::ExactMatch => {
                             if options.include_less_specifics {
@@ -654,13 +607,12 @@ where
                         &mut less_specifics_vec,
                     ) {
                         (Some(n), Some(pfx_idx)) => {
-                            trace!("5 found node. found prefix");
                             match_prefix_idx = Some(pfx_idx);
                             node = self.retrieve_node(n).unwrap();
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    if let Some(v) = self
+                                    more_specifics_vec = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -669,21 +621,17 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        )
-                                    {
-                                        more_specifics_vec.extend(v);
-                                    }
+                                        );
                                 }
                                 break;
                             }
                         }
                         (Some(n), None) => {
-                            trace!("5 found a next node. No prefix here.");
                             node = self.retrieve_node(n).unwrap();
 
                             if last_stride {
                                 if options.include_more_specifics {
-                                    if let Some(v) = self
+                                    more_specifics_vec = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -692,18 +640,14 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        )
-                                    {
-                                        more_specifics_vec.extend(v);
-                                    }
+                                        );
                                 }
                                 break;
                             }
                         }
                         (None, Some(pfx_idx)) => {
-                            trace!("5 no next node. found prefix");
                             if options.include_more_specifics {
-                                if let Some(v) = self
+                                more_specifics_vec = self
                                     .get_all_more_specifics_from_nibble(
                                         current_node,
                                         nibble,
@@ -712,19 +656,15 @@ where
                                             search_pfx.get_net(),
                                             stride_end - stride,
                                         ),
-                                    )
-                                {
-                                    more_specifics_vec.extend(v);
-                                }
+                                    );
                             }
                             match_prefix_idx = Some(pfx_idx);
                             break;
                         }
                         (None, None) => {
-                            trace!("5 no next node. no prefix");
                             match options.match_type {
                                 MatchType::EmptyMatch => {
-                                    if let Some(v) = self
+                                    more_specifics_vec = self
                                         .get_all_more_specifics_from_nibble(
                                             current_node,
                                             nibble,
@@ -733,10 +673,8 @@ where
                                                 search_pfx.get_net(),
                                                 stride_end - stride,
                                             ),
-                                        )
-                                    {
-                                        more_specifics_vec.extend(v);
-                                    }
+                                        );
+
                                     match_prefix_idx = None;
                                     break;
                                 }
@@ -785,7 +723,7 @@ where
                 None
             },
             more_specifics: if options.include_more_specifics {
-                Some(more_specifics_vec)
+                more_specifics_vec
             } else {
                 None
             },

From 0c3a9439709e10b0eeb2cb8b8bf31b5c48cd5f9d Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 22 Jan 2025 15:31:01 +0100
Subject: [PATCH 052/147] add bit pos from index method

---
 src/local_array/in_memory/atomic_stride.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 70d72c8b..f66f29aa 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -518,6 +518,10 @@ where
 
     fn get_bit_pos_as_u8(nibble: u32, len: u8) -> u8;
 
+    fn bit_pos_from_index(
+        i: u8,
+    ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
+
     // Clear the bitmap to the right of the pointer and count the number of
     // ones. This number represents the index to the corresponding prefix in
     // the pfx_vec.

From 41c71569b8275da916d0fc42a3f0ddb2879eecbf Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 22 Jan 2025 15:31:19 +0100
Subject: [PATCH 053/147] add bit pos from index method

---
 src/local_array/in_memory/macros.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index a62be011..bdcdc6af 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -194,6 +194,10 @@ macro_rules! impl_primitive_atomic_stride {
                     )
                 }
 
+                fn bit_pos_from_index(i: u8) -> $pfxsize {
+                    <$pfxsize>::try_from(1).unwrap().rotate_right(1) >> i
+                }
+
                 fn get_bit_pos_as_u8(nibble: u32, len: u8) -> u8 {
                     1 << (
                             <Self as Stride>::BITS - ((1 << len) - 1) as u8

From 77d768a5852ccb421a34beee2630938d4328ecc6 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 22 Jan 2025 15:32:41 +0100
Subject: [PATCH 054/147] add include_withdrawn option to record filtering

---
 src/local_array/in_memory/query.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index 74b2f77d..195521bc 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -44,6 +44,7 @@ where
                         pfx.record_map
                             .get_filtered_records(
                                 options.mui,
+                                options.include_withdrawn,
                                 withdrawn_muis_bmin,
                             )
                             .into_iter()

From 50417b2449a27c992bbc60351e95c63bf5e14440 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 22 Jan 2025 15:36:26 +0100
Subject: [PATCH 055/147] linear iterator for NodePrefixIterator

---
 src/local_array/in_memory/node.rs | 677 ++++++++++++++++++------------
 1 file changed, 399 insertions(+), 278 deletions(-)

diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 9adb4b44..f8ae89ab 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -1,19 +1,20 @@
+use core::ascii;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
-use std::{
-    fmt::Debug,
-    marker::PhantomData,
-};
+use std::{fmt::Debug, marker::PhantomData};
 
-use log::trace;
+use log::{log_enabled, trace};
 use parking_lot_core::SpinWait;
 
 use super::super::bit_span::BitSpan;
 use super::iterators::{SizedNodeMoreSpecificIter, SizedPrefixIter};
-use super::tree::{AtomicBitmap, AtomicStride2, AtomicStride3, AtomicStride4, AtomicStride5, CasResult, Stride, Stride3, Stride4, Stride5};
+use super::tree::{
+    AtomicBitmap, AtomicStride2, AtomicStride3, AtomicStride4, AtomicStride5,
+    CasResult, Stride, Stride3, Stride4, Stride5,
+};
 
 // pub use crate::in_memory_tree::*;
-use crate::af::Zero;
 use crate::af::AddressFamily;
+use crate::af::Zero;
 use crate::local_array::types::PrefixId;
 
 //------------ TreeBitMap Node ----------------------------------------------
@@ -30,11 +31,9 @@ use crate::local_array::types::PrefixId;
 // The elision of both the collection of children nodes and the prefix nodes
 // in a treebitmap node is enabled by the storage backend for the
 // multi-threaded store, since holds its entries keyed on the [node|prefix]
-// id. (in contrast with arrays or `vec`s, that have 
-pub struct TreeBitMapNode<
-    AF,
-    S,
-> where
+// id. (in contrast with arrays or `vec`s, that have
+pub struct TreeBitMapNode<AF, S>
+where
     Self: Sized,
     S: Stride,
     AF: AddressFamily,
@@ -44,8 +43,7 @@ pub struct TreeBitMapNode<
     pub _af: PhantomData<AF>,
 }
 
-impl<AF, S> Debug
-    for TreeBitMapNode<AF, S>
+impl<AF, S> Debug for TreeBitMapNode<AF, S>
 where
     AF: AddressFamily,
     S: Stride,
@@ -58,11 +56,10 @@ where
     }
 }
 
-impl<AF, S>
-    std::fmt::Display for TreeBitMapNode<AF, S>
+impl<AF, S> std::fmt::Display for TreeBitMapNode<AF, S>
 where
     AF: AddressFamily,
-    S: Stride
+    S: Stride,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
@@ -74,52 +71,58 @@ where
     }
 }
 
-impl<AF, S>
-    TreeBitMapNode<AF, S>
+impl<AF, S> TreeBitMapNode<AF, S>
 where
     AF: AddressFamily,
-    S: Stride
+    S: Stride,
 {
     pub(crate) fn new() -> Self {
         TreeBitMapNode {
             ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
             pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
-            _af: PhantomData
+            _af: PhantomData,
         }
     }
 
     // ------- Iterators ----------------------------------------------------
 
     // Iterate over all the child node of this node
-    pub(crate) fn ptr_iter(&self, base_prefix: StrideNodeId<AF>) -> 
-        NodeChildIter<AF,S> {
-            NodeChildIter::<AF,S> {
-                base_prefix,
-                ptrbitarr: self.ptrbitarr.load(),
-                bit_span: BitSpan::new(0, 1),
-                _af: PhantomData,
-            }
+    pub(crate) fn ptr_iter(
+        &self,
+        base_prefix: StrideNodeId<AF>,
+    ) -> NodeChildIter<AF, S> {
+        NodeChildIter::<AF, S> {
+            base_prefix,
+            ptrbitarr: self.ptrbitarr.load(),
+            bit_span: BitSpan::new(0, 1),
+            _af: PhantomData,
+        }
     }
 
     // Iterate over all the prefix ids contained in this node.
     // Note that this function is *not* used by the iterator that iterates
     // over all prefixes. That one doesn't have to use the tree at all, but
     // uses the store directly.
-    pub(crate) fn pfx_iter(&self, base_prefix: StrideNodeId<AF>) -> 
-        NodePrefixIter<AF, S> {
+    pub(crate) fn pfx_iter(
+        &self,
+        base_prefix: StrideNodeId<AF>,
+    ) -> NodePrefixIter<AF, S> {
         NodePrefixIter::<AF, S> {
             pfxbitarr: self.pfxbitarr.load(),
             base_prefix,
-            bit_span: BitSpan::new(0,1 ),
+            cursor: 1,
             _af: PhantomData,
             _s: PhantomData,
         }
     }
 
     // Iterate over the more specific prefixes ids contained in this node
-    pub(crate) fn more_specific_pfx_iter(&self, base_prefix: StrideNodeId<AF>, 
-    start_bit_span: BitSpan, skip_self: bool) -> 
-        NodeMoreSpecificsPrefixIter<AF, S> {
+    pub(crate) fn more_specific_pfx_iter(
+        &self,
+        base_prefix: StrideNodeId<AF>,
+        start_bit_span: BitSpan,
+        skip_self: bool,
+    ) -> NodeMoreSpecificsPrefixIter<AF, S> {
         NodeMoreSpecificsPrefixIter::<AF, S> {
             pfxbitarr: self.pfxbitarr.load(),
             base_prefix,
@@ -132,9 +135,11 @@ where
 
     // Iterate over the nodes that contain more specifics for the requested
     // base_prefix and corresponding bit_span.
-    pub(crate) fn more_specific_ptr_iter(&self, base_prefix: StrideNodeId<AF>, 
-    start_bit_span: BitSpan) -> 
-        NodeMoreSpecificChildIter<AF, S> {
+    pub(crate) fn more_specific_ptr_iter(
+        &self,
+        base_prefix: StrideNodeId<AF>,
+        start_bit_span: BitSpan,
+    ) -> NodeMoreSpecificChildIter<AF, S> {
         NodeMoreSpecificChildIter::<AF, S> {
             ptrbitarr: self.ptrbitarr.load(),
             base_prefix,
@@ -143,10 +148,9 @@ where
         }
     }
 
-
     // ------- Search by Traversal methods -----------------------------------
 
-    // Inspects the stride (nibble, nibble_len) to see it there's already a 
+    // Inspects the stride (nibble, nibble_len) to see it there's already a
     // child node (if not at the last stride) or a prefix (if it's the last
     // stride).
     //
@@ -168,7 +172,6 @@ where
         next_stride: Option<&u8>,
         is_last_stride: bool,
     ) -> (NewNodeOrIndex<AF>, u32) {
-
         // THE CRITICAL SECTION
         //
         // UPDATING ptrbitarr & pfxbitarr
@@ -195,11 +198,11 @@ where
         // reverse is *not* true, i.e. a full nibble can also be the last
         // stride. Hence the `is_last_stride` argument
         if !is_last_stride {
-
             // We are not at the last stride
             // Check it the ptr bit is already set in this position
-            if (S::into_stride_size(ptrbitarr) & bit_pos) == 
-                <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType>::zero() {
+            if (S::into_stride_size(ptrbitarr) & bit_pos) ==
+                <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::
+                    InnerType>::zero() {
                 // Nope, set it and create a child node
 
                 match next_stride.unwrap() {
@@ -238,7 +241,8 @@ where
                 //
                 // preventing using an old ptrbitarr and overwrite bits set
                 // in the meantime elsewhere in the bitarray.
-                let mut a_ptrbitarr = self.ptrbitarr.compare_exchange(ptrbitarr,
+                let mut a_ptrbitarr = self.ptrbitarr.compare_exchange(
+                    ptrbitarr,
                     S::into_ptrbitarr_size(
                     bit_pos | S::into_stride_size(ptrbitarr),
                 ));
@@ -252,7 +256,8 @@ where
                             // Someone beat us to it, so we need to use the
                             // newer array.
                             retry_count += 1;
-                            a_ptrbitarr = self.ptrbitarr.compare_exchange(newer_array,
+                            a_ptrbitarr = self.ptrbitarr.compare_exchange(
+                                newer_array,
                                 S::into_ptrbitarr_size(
                                 bit_pos | S::into_stride_size(newer_array),
                             ));
@@ -269,7 +274,8 @@ where
             // only at the last stride do we create the bit in the prefix
             // bitmap, and only if it doesn't exist already
             if pfxbitarr & bit_pos
-                == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType as std::ops::BitAnd>::Output::zero()
+                == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::
+                    InnerType as std::ops::BitAnd>::Output::zero()
             {
 
                 // THE CRITICAL SECTION
@@ -278,7 +284,7 @@ where
                 //
                 // preventing using an old pfxbitarr and overwrite bits set
                 // in the meantime elsewhere in the bitarray.
-                let mut a_pfxbitarr = 
+                let mut a_pfxbitarr =
                 self.pfxbitarr.compare_exchange(
                     pfxbitarr, bit_pos | pfxbitarr
                 );
@@ -306,19 +312,22 @@ where
             return (NewNodeOrIndex::ExistingPrefix, retry_count);
         }
 
-        // Nodes always live at the last length of a stride (i.e. the last 
+        // Nodes always live at the last length of a stride (i.e. the last
         // nibble), so we add the stride length to the length of the
         // base_prefix (which is always the start length of the stride).
-        (NewNodeOrIndex::ExistingNode(
-            base_prefix.add_to_len(stride_len).truncate_to_len()
-        ), retry_count)
+        (
+            NewNodeOrIndex::ExistingNode(
+                base_prefix.add_to_len(stride_len).truncate_to_len(),
+            ),
+            retry_count,
+        )
     }
 
     //-------- Search nibble functions --------------------------------------
 
     // This function looks for the longest marching prefix in the provided
     // nibble, by iterating over all the bits in it and comparing that with
-    // the appropriate bytes from the requested prefix. It mutates the 
+    // the appropriate bytes from the requested prefix. It mutates the
     // `less_specifics_vec` that was passed in to hold all the prefixes found
     // along the way.
     pub(crate) fn search_stride_for_longest_match_at(
@@ -337,20 +346,25 @@ where
         trace!("start longest_match search");
         for n_l in 1..(nibble_len + 1) {
             // Move the bit in the right position.
-            nibble =
-                AddressFamily::get_nibble(search_pfx.get_net(), start_bit, n_l);
+            nibble = AddressFamily::get_nibble(
+                search_pfx.get_net(),
+                start_bit,
+                n_l,
+            );
             bit_pos = S::get_bit_pos(nibble, n_l);
 
             // Check if the prefix has been set, if so select the prefix.
             // This is not necessarily the final prefix that will be
             // returned.
 
-            // Check it there's a prefix matching in this bitmap for this 
+            // Check it there's a prefix matching in this bitmap for this
             // nibble.
             trace!("pfxbitarr {:032b}", pfxbitarr);
 
-            if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-                let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + n_l), start_bit + n_l);
+            if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize
+                as AtomicBitmap>::InnerType::zero() {
+                let f_pfx = PrefixId::new(search_pfx.get_net()
+                    .truncate_to_len(start_bit + n_l), start_bit + n_l);
                 // f_pfx.set_serial(self.get_pfx_serial(f_pfx, nibble, n_l, guard).load(Ordering::Relaxed));
 
                 // Receiving a less_specifics_vec means that the user wants
@@ -362,14 +376,17 @@ where
                     trace!("len {}", search_pfx.get_len());
                     trace!("start_bit {}", start_bit);
                     trace!("n_l {}", n_l);
-                    trace!("smaller length? {}", search_pfx.get_len() > start_bit + n_l);
+                    trace!("smaller length? {}",
+                        search_pfx.get_len() > start_bit + n_l);
                     trace!("{}", (S::into_stride_size(ptrbitarr)
                             & bit_pos)
-                            == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero());
+                            == <<S as Stride>::AtomicPfxSize
+                            as AtomicBitmap>::InnerType::zero());
                     if search_pfx.get_len() > start_bit + n_l
                         && (S::into_stride_size(ptrbitarr)
                             & bit_pos)
-                            == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+                            == <<S as Stride>::AtomicPfxSize
+                                as AtomicBitmap>::InnerType::zero()
                     {
                         ls_vec.push(f_pfx);
                     }
@@ -379,13 +396,17 @@ where
             }
         }
 
-        let base_prefix =
-            StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit);
+        let base_prefix = StrideNodeId::new_with_cleaned_id(
+            search_pfx.get_net(),
+            start_bit,
+        );
 
         // Check if this the last stride, or if they're no more children to
         // go to, if so return what we found up until now.
         if search_pfx.get_len() <= start_bit + nibble_len
-            || (S::into_stride_size(ptrbitarr) & bit_pos) == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+            || (S::into_stride_size(ptrbitarr) & bit_pos) ==
+                <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType
+                    ::zero()
         // No children or at the end, return the definitive LMP we found.
         {
             return (
@@ -419,7 +440,6 @@ where
     ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
         let pfxbitarr = self.pfxbitarr.load();
         let ptrbitarr = self.ptrbitarr.load();
-        println!("ptrbitarr {:064b}", ptrbitarr);
         // This is an exact match, so we're only considering the position of
         // the full nibble.
         let bit_pos = S::get_bit_pos(nibble, nibble_len);
@@ -432,7 +452,7 @@ where
         match search_pfx.get_len() <= start_bit + nibble_len {
             // We're at the last nibble.
             true => {
-                // Check for an actual prefix at the right position, i.e. 
+                // Check for an actual prefix at the right position, i.e.
                 // consider the complete nibble.
                 if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
                     let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + nibble_len), start_bit + nibble_len);
@@ -454,12 +474,11 @@ where
 
         (
             found_child, /* The node that has children in the next stride, if
-                            any */
-            found_pfx,   /* The exactly matching prefix, if any */
+                         any */
+            found_pfx, /* The exactly matching prefix, if any */
         )
     }
 
-
     pub(crate) fn prefix_exists_in_stride(
         &'_ self,
         search_pfx: PrefixId<AF>,
@@ -481,7 +500,7 @@ where
         match search_pfx.get_len() <= start_bit + nibble_len {
             // We're at the last nibble.
             true => {
-                // Check for an actual prefix at the right position, i.e. 
+                // Check for an actual prefix at the right position, i.e.
                 // consider the complete nibble.
                 if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
                     found_pfx = true;
@@ -502,15 +521,15 @@ where
 
         (
             found_child, /* The node that has children in the next stride, if
-                            any */
-            found_pfx,   /* The exactly matching prefix, if any */
+                         any */
+            found_pfx, /* The exactly matching prefix, if any */
         )
     }
 
     // This function looks for the exactly matching prefix in the provided
     // nibble, just like the one above, but this *does* iterate over all the
-    // bytes in the nibble to collect the less-specific prefixes of the the 
-    // search prefix. This is of course slower, so it should only be used 
+    // bytes in the nibble to collect the less-specific prefixes of the the
+    // search prefix. This is of course slower, so it should only be used
     // when the user explicitly requests less-specifics.
     pub(crate) fn search_stride_for_exact_match_with_less_specifics_at(
         &self,
@@ -520,44 +539,50 @@ where
         start_bit: u8,
         less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
     ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
-
         let pfxbitarr = self.pfxbitarr.load();
         let ptrbitarr = self.ptrbitarr.load();
         let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
         let mut found_pfx = None;
 
-        let ls_vec = less_specifics_vec
-            .as_mut()
-            .expect(concat!("You shouldn't call this function without",
-            "a `less_specifics_vec` buffer. Supply one when calling this function",
-            "or use `search_stride_for_exact_match_at`"));
+        let ls_vec = less_specifics_vec.as_mut().expect(concat!(
+            "You shouldn't call this function without",
+            "a `less_specifics_vec` buffer. Supply one when calling this
+             function",
+            "or use `search_stride_for_exact_match_at`"
+        ));
 
         for n_l in 1..(nibble_len + 1) {
             // Move the bit in the right position.
-            nibble =
-                AddressFamily::get_nibble(search_pfx.get_net(), start_bit, n_l);
+            nibble = AddressFamily::get_nibble(
+                search_pfx.get_net(),
+                start_bit,
+                n_l,
+            );
             bit_pos = S::get_bit_pos(nibble, n_l);
 
             // Check if the prefix has been set, if so select the prefix.
             // This is not necessarily the final prefix that will be
             // returned.
 
-            // Check it there's a prefix matching in this bitmap for this 
+            // Check it there's a prefix matching in this bitmap for this
             // nibble.
-            if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
+            if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as
+                AtomicBitmap>::InnerType::zero() {
                 // since we want an exact match only, we will fill the prefix
                 // field only if we're exactly at the last bit of the nibble
                 if n_l == nibble_len {
                     let f_pfx =
                         PrefixId::new(
-                            search_pfx.get_net().truncate_to_len(start_bit + n_l), start_bit + n_l);
+                            search_pfx.get_net().truncate_to_len(start_bit +
+                                n_l), start_bit + n_l);
                         found_pfx = Some(f_pfx);
                 }
 
                 // Receiving a less_specifics_vec means that the user wants to
                 // have all the last-specific prefixes returned, so add the
                 // found prefix.
-                let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + n_l), start_bit + n_l);
+                let f_pfx = PrefixId::new(search_pfx.get_net().
+                    truncate_to_len(start_bit + n_l), start_bit + n_l);
                 ls_vec.push(f_pfx);
             }
         }
@@ -569,14 +594,15 @@ where
             ls_vec.clear();
         }
 
-        // Check if this the last stride, or if they're no more children to 
+        // Check if this the last stride, or if they're no more children to
         // go to, if so return what we found up until now.
         match search_pfx.get_len() <= start_bit + nibble_len
             || (S::into_stride_size(ptrbitarr) & bit_pos)
-                == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType as std::ops::BitAnd>::Output::zero()
+                == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType
+                    as std::ops::BitAnd>::Output::zero()
         {
             // No children or at the end, return the definitive LMP we found.
-            true => { 
+            true => {
                 println!("found {:?}", found_pfx); 
                 (
                 None,      /* no more children */
@@ -586,7 +612,8 @@ where
             // There's another child, we won't return the found_pfx, since
             // we're not at the last nibble and we want an exact match only.
             false => (
-                Some(StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)),
+                Some(StrideNodeId::new_with_cleaned_id(search_pfx.get_net(),
+                    start_bit + nibble_len)),
                 None,
             ),
         }
@@ -601,8 +628,8 @@ where
         base_prefix: StrideNodeId<AF>,
     ) -> (
         Vec<StrideNodeId<AF>>, /* child nodes with more more-specifics in
-                                  this stride */
-        Vec<PrefixId<AF>>,     /* more-specific prefixes in this stride */
+                               this stride */
+        Vec<PrefixId<AF>>, /* more-specific prefixes in this stride */
     ) {
         trace!("start adding more specifics");
         let pfxbitarr = self.pfxbitarr.load();
@@ -618,7 +645,7 @@ where
         let mut found_child = None;
 
         // Is there also a child node here?
-        // Note that even without a child node, there may be more specifics 
+        // Note that even without a child node, there may be more specifics
         // further up in this pfxbitarr or children in this ptrbitarr.
         if (S::into_stride_size(ptrbitarr) & bit_pos)
             > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
@@ -637,13 +664,13 @@ where
         // and we've all strides sized 4, we would end up with a last
         // nibble_len of 1. `ms_nibble_len` will expand then from 2 up and
         // till 4.
-        // 
+        //
         // ex.:
-        // nibble: 1 , (nibble_len: 1)x
+        // nibble: 1 , (nibble_len: 1)
         // Iteration:
         // ms_nibble_len=1,n_l=0: 10, n_l=1: 11
         // ms_nibble_len=2,n_l=0: 100, n_l=1: 101, n_l=2: 110, n_l=3: 111
-        // ms_nibble_len=3,n_l=0: 1000, n_l=1: 1001, n_l=2: 1010, ..., 
+        // ms_nibble_len=3,n_l=0: 1000, n_l=1: 1001, n_l=2: 1010, ...,
         // n_l=7: 1111
 
         for ms_nibble_len in nibble_len + 1..=S::STRIDE_LEN {
@@ -672,7 +699,10 @@ where
             }
         }
 
-        trace!("found_children_with_more_specifics {:?}", found_children_with_more_specifics);
+        trace!(
+            "found_children_with_more_specifics {:?}",
+            found_children_with_more_specifics
+        );
         trace!("found_more_specifics_vec {:?}", found_more_specifics_vec);
 
         (
@@ -685,7 +715,6 @@ where
     }
 }
 
-
 // ------------ Iterator methods --------------------------------------------
 
 // ----------- NodeChildIter ------------------------------------------------
@@ -706,10 +735,10 @@ where
 //
 // Ex.:
 //
-// Stride no.          1       2       3      4       5       6       7       
-// StrideSize          5       5       4      3       3       3       3      
+// Stride no.          1       2       3      4       5       6       7
+// StrideSize          5       5       4      3       3       3       3
 // child pfxs len      /1-5   /5-10    /10-14 /15-17  /18-20  /21-23  /24-26
-// child Nodes len     /5      /10     /14    /17     /20     /23     /26 
+// child Nodes len     /5      /10     /14    /17     /20     /23     /26
 //
 // Stride no.          8      9
 // StrideSize          3      3
@@ -718,14 +747,14 @@ where
 
 #[derive(Debug, Copy, Clone)]
 pub(crate) struct NodeChildIter<AF: AddressFamily, S: Stride> {
-   base_prefix: StrideNodeId<AF>,
-   ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
-   bit_span: BitSpan, // start with 0
-   _af: PhantomData<AF>,
+    base_prefix: StrideNodeId<AF>,
+    ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
+    bit_span: BitSpan, // start with 0
+    _af: PhantomData<AF>,
 }
 
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
-    NodeChildIter<AF, S>
+impl<AF: AddressFamily, S: Stride> std::iter::Iterator
+    for NodeChildIter<AF, S>
 {
     type Item = StrideNodeId<AF>;
     fn next(&mut self) -> Option<Self::Item> {
@@ -735,7 +764,7 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
             // move the bit_span left with the amount of bits we're going to
             // loop over.
             // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
-            // becomes 0000 0000 0000 1100, then it will iterate over 
+            // becomes 0000 0000 0000 1100, then it will iterate over
             // ...1100,...1101,...1110,...1111
             let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
             if (S::into_stride_size(self.ptrbitarr) & bit_pos) >
@@ -743,8 +772,7 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
             {
                 self.bit_span.bits = cursor + 1;
                 return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
-            }    
-            
+            }
         }
         None
     }
@@ -755,8 +783,8 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
 // Create an iterator over all the child nodes that hold a more specific
 // prefixes of the specified start_bit_span. This basically the same Iterator
 // as the ChildNodeIter, except that it stops (potentially) earlier, to avoid
-// including nodes with adjacent prefixes. Starting an iterator with a 
-// `start_bit_span` of { bits: 0, len: 0 } will return all child nodes of 
+// including nodes with adjacent prefixes. Starting an iterator with a
+// `start_bit_span` of { bits: 0, len: 0 } will return all child nodes of
 // this node. In that case you could also use the `NodeChildIter` instead.
 //
 // inputs
@@ -767,13 +795,13 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
 // the tree (therefore, it's actually a Trie). Note that `base_prefix` +
 // `bit_span` define the actual starting prefix for this iterator.
 //
-// `ptrbitarr` 
-// is the bitmap that holds the slots that have child nodes. 
+// `ptrbitarr`
+// is the bitmap that holds the slots that have child nodes.
 //
-// `start_bit_span` 
-// is the bit span that is going to be used as a starting point for the 
+// `start_bit_span`
+// is the bit span that is going to be used as a starting point for the
 // iterator.
-// 
+//
 // `cursor`
 // holds the current cursor offset from the start_bit_span.bits, the sum of
 // these describe the current position in the bitmap. Used for re-entry into
@@ -798,33 +826,38 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
 // done now.
 #[derive(Debug, Copy, Clone)]
 pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily, S: Stride> {
-   base_prefix: StrideNodeId<AF>,
-   ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
-   start_bit_span: BitSpan,
-   cursor: Option<u32>,
+    base_prefix: StrideNodeId<AF>,
+    ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
+    start_bit_span: BitSpan,
+    cursor: Option<u32>,
 }
 
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
-    NodeMoreSpecificChildIter<AF, S>
+impl<AF: AddressFamily, S: Stride> std::iter::Iterator
+    for NodeMoreSpecificChildIter<AF, S>
 {
     type Item = StrideNodeId<AF>;
     fn next(&mut self) -> Option<Self::Item> {
         // Early exits
 
         // Empty bitmap
-        if self.ptrbitarr ==
-            <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType::zero() {
+        if self.ptrbitarr
+            == <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType::zero(
+            )
+        {
             trace!("empty ptrbitrarr. this iterator is done.");
             return None;
         }
 
         // Previous iteration incremented the cursor beyond the stride size.
-        if let Some(cursor) = self.cursor { 
+        if let Some(cursor) = self.cursor {
             if cursor >= (1 << (S::STRIDE_LEN - self.start_bit_span.len)) {
-                trace!("cursor.bits >= (1 << (S::STRIDE_LEN - self.
-                    start_bit_span.len))");
+                trace!(
+                    "cursor.bits >= (1 << (S::STRIDE_LEN - self.
+                    start_bit_span.len))"
+                );
                 trace!("cursor: {}", cursor);
-                trace!("start_bit_span: {} {}",
+                trace!(
+                    "start_bit_span: {} {}",
                     self.start_bit_span.bits,
                     self.start_bit_span.len
                 );
@@ -836,11 +869,14 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
         trace!("NodeMoreSpecificChildIter");
         trace!("base_prefix {}", self.base_prefix);
         trace!("stride_size {}", S::STRIDE_LEN);
-        trace!("start_bit_span bits {} len {} ",
-            self.start_bit_span.bits, self.start_bit_span.len);
+        trace!(
+            "start_bit_span bits {} len {} ",
+            self.start_bit_span.bits,
+            self.start_bit_span.len
+        );
         trace!("cursor bits {:?}", self.cursor);
         trace!("          x1  4   8  12  16  20  24  28  32");
-        trace!("ptrbitarr {:032b}x", self.ptrbitarr);
+        trace!("ptrbitarr {:032b}", self.ptrbitarr);
 
         let start = if let Some(bits) = self.cursor {
             bits
@@ -852,18 +888,16 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
         // stride (in case of a start_bit_span.bits == 0).
         let stop = <u32>::min(
             (1 << (S::STRIDE_LEN - self.start_bit_span.len)) + start,
-            (1 << S::STRIDE_LEN) - 1
-        ); 
+            (1 << S::STRIDE_LEN) - 1,
+        );
 
         trace!("start {:?} stop {}", start, stop);
         for cursor in start..=stop {
-            // move the bit_span left with the amount of bits we're going to
-            // loop over. e.g. a stride of size 4 with a nibble 0000 0000 0000
-            // 0011 becomes 0000 0000 0000 1100, then it will iterate over
+            // move the bit_span left with the amount of bits we're going to loop over.
+            // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
+            // becomes 0000 0000 0000 1100, then it will iterate over
             // ...1100,...1101,...1110,...1111
-            let bit_pos = 
-                S::get_bit_pos(
-                    cursor, S::STRIDE_LEN);
+            let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
             trace!("cmpbitarr x{:032b} {}", bit_pos, stop - start);
             if (S::into_stride_size(self.ptrbitarr) & bit_pos) >
                 <<S as Stride>::AtomicPfxSize as AtomicBitmap
@@ -873,13 +907,8 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
                 self.cursor = Some(cursor + 1);
 
                 trace!("next bit_span {} {} with cursor {:?}",
-                    self.start_bit_span.bits,
-                    self.start_bit_span.len,
-                    self.cursor
-                );
-                return Some(
-                    self.base_prefix.add_nibble(cursor, S::STRIDE_LEN)
-                );
+                    self.start_bit_span.bits, self.start_bit_span.len, self.cursor);
+                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
             }
         }
         trace!("No more nodes. End of the iterator.");
@@ -905,23 +934,23 @@ impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride5> {
     }
 }
 
-// ----------- NodePrefixIter ------------------------------------------------
+// ----------- NodePrefixIter -----------------------------------------------
 
 // Create an iterator of all prefix ids hosted by this node.
 
 // Partition for stride 3
 //
-// pfxbitarr (AF::BITS)  0 1 2  3  4  5  6   7   8   9  10  11  12  13  14 
-// bit_span (binary)     * 0 1 00 01 10 11 000 001 010 011 100 101 110 111 * *
-// bit_span (dec.)       * 0 1  0  1  2  3   0   1   2   3   4   5   6   7 * *
-// len                   0 1   2           3                               
+// pfxbitarr (AF::BITS)  0 1 2  3  4  5  6   7   8   9  10  11  12  13  14
+// bit_span (binary)     * 0 1 00 01 10 11 000 001 010 011 100 101 110 111 *
+// bit_span (dec.)       * 0 1  0  1  2  3   0   1   2   3   4   5   6   7 *
+// len                   0 1   2           3
 //
-// pfxbitarr (example)   1 0 0 0  0  0  1  1   1   0   0   0   0   0   0   0 0
-// pos (example)         0 0 0 0  0  0  0  0   1   0   0   0   0   0   0   0 0                 
+// pfxbitarr (example)   1 0 0 0  0  0  1  1   1   0   0   0   0   0   0   0
+// pos (example)         0 0 0 0  0  0  0  0   1   0   0   0   0   0   0   0
 //
 // Ex.:
-// `pos` describes the bit that is currently under consideration. 
-// 
+// `pos` describes the bit that is currently under consideration.
+//
 // `pfxbitarr` is the bitmap that contains the prefixes. Every 1 in the
 // bitmap means that the prefix is hosted by this node. Moreover, the
 // position in the bitmap describes the address part of the prefix, given
@@ -936,29 +965,79 @@ impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride5> {
 pub(crate) struct NodePrefixIter<AF: AddressFamily, S: Stride> {
     base_prefix: StrideNodeId<AF>,
     pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
-    bit_span: BitSpan, // start with 0
+    cursor: u8,
     _af: PhantomData<AF>,
     _s: PhantomData<S>,
 }
 
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator for 
-    NodePrefixIter<AF, S> {
-        type Item = PrefixId<AF>;
+impl<AF: AddressFamily, S: Stride> std::iter::Iterator
+    for NodePrefixIter<AF, S>
+{
+    type Item = PrefixId<AF>;
 
-        fn next(&mut self) -> Option<Self::Item> {
-        // iterate over all the possible values for this stride length, e.g.
-        // two bits can have 4 different values.
-        for cursor in self.bit_span.bits..(1 << S::STRIDE_LEN) {
-            
-            let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
-            if self.pfxbitarr & bit_pos >
-                <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-            {
-                self.bit_span.bits = cursor + 1;
-                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN).into());
-            }    
-            
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.pfxbitarr
+            == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
+            )
+        {
+            return None;
+        }
+        if log_enabled!(log::Level::Trace) {
+            trace!("---- NodePrefixIter -----");
+            match S::STRIDE_LEN {
+                3 => {
+                    trace!(
+                        "pfxbitarr  {:016b} ({})",
+                        self.pfxbitarr,
+                        S::STRIDE_LEN
+                    );
+                }
+                4 => {
+                    trace!(
+                        "pfxbitarr  {:032b} ({})",
+                        self.pfxbitarr,
+                        S::STRIDE_LEN
+                    );
+                }
+                5 => {
+                    trace!(
+                        "pfxbitarr  {:064b} ({})",
+                        self.pfxbitarr,
+                        S::STRIDE_LEN
+                    );
+                }
+                _ => {
+                    trace!("UNKNOWN STRIDE SIZE ENCOUNTERED.");
+                }
+            };
+        }
+
+        let cursor = self.cursor;
+
+        for i in cursor..S::BITS - 1 {
+            let bit_pos = S::bit_pos_from_index(i);
+            self.cursor += 1;
+
+            if log_enabled!(log::Level::Trace) {
+                let bs = BitSpan::from_bit_pos_index(i);
+                trace!(
+                    "{:02}: {:06b} {:064b} bit_span: {:05b} (len: {})",
+                    i,
+                    i - 1,
+                    bit_pos,
+                    bs.bits,
+                    bs.len
+                );
+            }
+
+            if self.pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize
+                as AtomicBitmap>::InnerType::zero() {
+                let bs = BitSpan::from_bit_pos_index(i);
+                return Some(self.base_prefix.add_nibble(bs.bits, bs.len).into());
+            }
         }
+
+        trace!("end of iterator NodePrefixIter for this node");
         None
     }
 }
@@ -967,20 +1046,20 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
 // are a more-specific prefix of the `base_prefix` + `start_bit_span`.
 //
 // Inputs
-// 
+//
 // `base_prefix`
 // This iterator take a `base_prefix` since the nodes themselves have no
 // knowledge of their own prefixes, those are inferred by their position in
 // the tree (therefore, it's actually a Trie). Note that `base_prefix` +
 // `bit_span` define the actual starting prefix for this iterator.
 //
-// `pfxbitarr` 
+// `pfxbitarr`
 // is the bitmap that holds the slots that have prefixes.
 //
-// `start_bit_span` 
-// is the bit span that is going to be used as a starting point for the 
+// `start_bit_span`
+// is the bit span that is going to be used as a starting point for the
 // iterator.
-// 
+//
 // `cursor`
 // holds the current cursor offset from the start_bit_span.bits, the sum of
 // these describe the current position in the bitmap. Used for re-entry into
@@ -997,15 +1076,15 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
 //
 // e.x.
 // The stride size is 5 and the starting bit span is {bits: 1, len: 3} (001)
-// This means that the stride size that we have to consider are 4 and 5. 3 
+// This means that the stride size that we have to consider are 4 and 5. 3
 // being the size of the current bit_span and 5 being the size of the total
-// stride. 
+// stride.
 // The starting point is therefore the bit_array 001. The iterator will go
 // over 001 00, 001 01, 001 10 and 001 11. The next bits to consider would be
 //  010 00 which would not fit our starting bit_span of 0010. So we have to
 // stop after 2 iterations. This means that the number of iterations is
 // determined by the difference between the number of bits in the stride size
-// (5) and the the number of bits in the start_bit_span (4). The number of 
+// (5) and the the number of bits in the start_bit_span (4). The number of
 // iterations in the above example is therefore 1 << (5 - 3) = 4.
 // Unlike the MoreSpecificPrefixIter, we will have to consider more lengths
 // than just the bit_span len. We will have to jump a few pfxbitarr bits and
@@ -1017,7 +1096,7 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator for
 pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily, S: Stride> {
     // immutables
     base_prefix: StrideNodeId<AF>,
-    pfxbitarr: <<S as super::atomic_stride::Stride>::AtomicPfxSize 
+    pfxbitarr: <<S as super::atomic_stride::Stride>::AtomicPfxSize
         as super::atomic_stride::AtomicBitmap>::InnerType,
     // we need to keep around only the `bits` part of the `bit_span`
     // technically, (it needs resetting the current state to it after each
@@ -1029,114 +1108,156 @@ pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily, S: Stride> {
     _s: PhantomData<S>,
 }
 
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator for 
-    NodeMoreSpecificsPrefixIter<AF, S> {
-        type Item = PrefixId<AF>;
+impl<AF: AddressFamily, S: Stride> std::iter::Iterator
+    for NodeMoreSpecificsPrefixIter<AF, S>
+{
+    type Item = PrefixId<AF>;
 
-        fn next(&mut self) -> Option<Self::Item> {
+    fn next(&mut self) -> Option<Self::Item> {
+        // Easy early exit conditions
 
-            // Easy early exit conditions
+        // Empty bitmap
+        if self.pfxbitarr
+            == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
+            )
+        {
+            trace!("empty pfxbitarr. This iterator is done.");
+            return None;
+        }
 
-            // Empty bitmap
-            if self.pfxbitarr == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-                trace!("empty pfxbitarr. This iterator is done.");
-                return None;
+        // No early exits, We're in business.
+        trace!("len_offset {}", ((1 << self.cursor.len) - 1));
+        trace!("start_bit {}", self.start_bit_span.bits);
+        trace!(
+            "number of check bits in len {}",
+            (1 << (self.cursor.len - self.start_bit_span.len))
+        );
+
+        trace!(
+            "next more specifics prefix iter start bits {} len {}",
+            self.start_bit_span.bits,
+            self.start_bit_span.len
+        );
+
+        let mut res = None;
+
+        // Move to the next len if we're at the first prefix-length that
+        // matches, if `skip_self` is set. Typically this flag is set for the
+        // first stride. In the consecutive strides, we don't want to skip the
+        // base prefix, since that base_prefix is a more specific prefix of
+        // the one requested in the first stride.
+        if self.skip_self
+            && (1 << (self.cursor.len - self.start_bit_span.len)) == 1
+        {
+            // self.cursor.len += (self.start_bit_span.bits & 1) as u8;
+            trace!("skipping self");
+            self.start_bit_span.bits <<= 1;
+            self.cursor.bits = self.start_bit_span.bits;
+            self.cursor.len += 1;
+            self.skip_self = false;
+            trace!(
+                "new start bits {} len {}",
+                self.start_bit_span.bits,
+                self.start_bit_span.len
+            );
+            trace!(
+                "new cursor bits {} len {}",
+                self.cursor.bits,
+                self.cursor.len
+            );
+        }
+
+        // Previous iteration or the skip_self routine may have
+        // incremented the cursor beyond the end of the stride size.
+        if self.cursor.len > S::STRIDE_LEN {
+            trace!("cursor.len > S::STRIDE_LEN. This iterator is done.");
+            return None;
+        }
+
+        loop {
+            trace!(
+                "          x1  4   8  12  16  20  24  28  32  36  40  44
+                  48  52  56  60  64"
+            );
+            trace!(
+                "cmpnibble {:064b} ({} + {}) len {} stride_size {}",
+                S::get_bit_pos(self.cursor.bits, self.cursor.len),
+                (1 << self.cursor.len) - 1,
+                self.cursor.bits,
+                self.cursor.len + self.base_prefix.get_len(),
+                S::STRIDE_LEN
+            );
+
+            trace!("pfxbitarr {:064b}", self.pfxbitarr);
+
+            if (S::get_bit_pos(self.cursor.bits, self.cursor.len)
+                | self.pfxbitarr)
+                == self.pfxbitarr
+            {
+                trace!(
+                    "found prefix with len {} at pos {} pfx len {}",
+                    self.cursor.len,
+                    self.cursor.bits,
+                    self.base_prefix.get_len() + self.cursor.len,
+                );
+                res = Some(
+                    self.base_prefix
+                        .add_nibble(self.cursor.bits, self.cursor.len)
+                        .into(),
+                );
+                trace!("found prefix {:?}", res);
             }
 
-            // No early exits, We're in business.
-            trace!("len_offset {}", ((1<< self.cursor.len) - 1));
-            trace!("start_bit {}", self.start_bit_span.bits);
-            trace!("number of check bits in len {}", 
-                (1 << (self.cursor.len - self.start_bit_span.len)));
-
-            trace!("next more specifics prefix iter start bits {} len {}",
-                self.start_bit_span.bits, self.start_bit_span.len);
-
-            let mut res = None;
-
-            // Move to the next len if we're at the first prefix-length that matches,
-            // if `skip_self` is set. Typically this flag is set for the first stride.
-            // In the consecutive strides, we don't want to skip the base prefix, since
-            // that base_prefix is a more specific prefix of the one requested in the
-            // first stride.
-            if self.skip_self && (1 << (self.cursor.len - self.start_bit_span.len)) == 1 {
-                // self.cursor.len += (self.start_bit_span.bits & 1) as u8;
-                trace!("skipping self");
+            // Determine when we're at the end of the bits applicable to
+            // this combo of this start_bit_span.
+            // bitspan offset:
+            // self.start_bit_span.bits
+            // number of matches in this length:
+            // 1 << (self.cursor.len - self.start_bit_span.len)
+            let max_pos_offset = self.start_bit_span.bits
+                + (1 << (self.cursor.len - self.start_bit_span.len))
+                - 1;
+
+            trace!(
+                "max_pos_offset {} > cursor bit_pos {}?",
+                max_pos_offset,
+                self.cursor.bits
+            );
+            trace!(
+                "number of check bits in len {}",
+                (1 << (self.cursor.len - self.start_bit_span.len))
+            );
+
+            // case 1. At the beginning or inside a prefix-length.
+            if max_pos_offset > self.cursor.bits {
+                self.cursor.bits += 1;
+            }
+            // case 2. At the end of a prefix-lengths in this stride.
+            else if self.cursor.len < S::STRIDE_LEN {
+                trace!("move len to {}", self.cursor.len + 1);
                 self.start_bit_span.bits <<= 1;
                 self.cursor.bits = self.start_bit_span.bits;
                 self.cursor.len += 1;
-                self.skip_self = false;
-                trace!("new start bits {} len {}", self.start_bit_span.bits, self.start_bit_span.len);
-                trace!("new cursor bits {} len {}", self.cursor.bits, self.cursor.len);
             }
-
-            // Previous iteration or the skip_self routine may have
-            // incremented the cursor beyond the end of the stride size.
-            if self.cursor.len > S::STRIDE_LEN {
-                trace!("cursor.len > S::STRIDE_LEN. This iterator is done.");
-                return None;
-            }
-            
-            loop {
-                trace!("          x1  4   8  12  16  20  24  28  32  36  40  44  48  52  56  60  64");
-                trace!("cmpnibble {:064b} ({} + {}) len {} stride_size {}", 
-                    S::get_bit_pos(self.cursor.bits, self.cursor.len), 
-                    (1<< self.cursor.len) - 1, 
-                    self.cursor.bits, 
-                    self.cursor.len + self.base_prefix.get_len(),
-                    S::STRIDE_LEN
+            // case 4. At the end of a prefix-length, but not at the end of the pfxbitarr.
+            else {
+                self.start_bit_span.bits <<= 1;
+                self.cursor.bits = self.start_bit_span.bits;
+                self.cursor.len += 1;
+                trace!(
+                    "end of stride, next cursor bits {} len {}",
+                    self.cursor.bits,
+                    self.cursor.len
                 );
+                return res;
+            }
 
-                trace!("pfxbitarr {:064b}", self.pfxbitarr);
-
-                if (S::get_bit_pos(self.cursor.bits, self.cursor.len) | self.pfxbitarr) == self.pfxbitarr {
-                    trace!("found prefix with len {} at pos {} pfx len {}",
-                        self.cursor.len, 
-                        self.cursor.bits,
-                        self.base_prefix.get_len() + self.cursor.len,
-                    );
-                    res = Some(self.base_prefix
-                        .add_nibble(self.cursor.bits, self.cursor.len).into());
-                    trace!("found prefix {:?}", res);
-                }
-
-                // Determine when we're at the end of the bits applicable to
-                // this combo of this start_bit_span.
-                // bitspan offset: 
-                // self.start_bit_span.bits
-                // number of matches in this length:
-                // 1 << (self.cursor.len - self.start_bit_span.len)
-                let max_pos_offset =  
-                    self.start_bit_span.bits + 
-                    (1 << (self.cursor.len - self.start_bit_span.len)) - 1;
-
-                trace!("max_pos_offset {} > cursor bit_pos {}?", max_pos_offset, self.cursor.bits);
-                trace!("number of check bits in len {}", (1 << (self.cursor.len - self.start_bit_span.len)));
-
-                // case 1. At the beginning or inside a prefix-length.
-                if max_pos_offset > self.cursor.bits {
-                    self.cursor.bits += 1;
-                } 
-                // case 2. At the end of a prefix-lengths in this stride.
-                else if self.cursor.len < S::STRIDE_LEN {
-                    trace!("move len to {}", self.cursor.len + 1);
-                    self.start_bit_span.bits <<= 1;
-                    self.cursor.bits = self.start_bit_span.bits;
-                    self.cursor.len += 1;
-                }
-                // case 4. At the end of a prefix-length, but not at the end of the pfxbitarr.
-                else {
-                    self.start_bit_span.bits <<= 1;
-                    self.cursor.bits = self.start_bit_span.bits;
-                    self.cursor.len += 1;
-                    trace!("end of stride, next cursor bits {} len {}", self.cursor.bits, self.cursor.len);
-                    return res;
-                }
-
-                trace!("some res {:?}", res);
-                if res.is_some() { return res; }
+            trace!("some res {:?}", res);
+            if res.is_some() {
+                return res;
             }
-        }    
+        }
+    }
 }
 
 impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride3> {

From 12189b7e5c55d89969d1e34fd30eba0dd2ce6afb Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 24 Jan 2025 09:52:42 +0100
Subject: [PATCH 056/147] linear iterator for NodeMoreSpecificChildIterator

---
 src/local_array/in_memory/atomic_stride.rs |  11 +-
 src/local_array/in_memory/iterators.rs     |  38 +-
 src/local_array/in_memory/macros.rs        |  25 +-
 src/local_array/in_memory/node.rs          | 547 +++++++++++++--------
 4 files changed, 398 insertions(+), 223 deletions(-)

diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index f66f29aa..7f01827b 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -6,6 +6,7 @@ use std::sync::atomic::{
 };
 
 use crate::af::Zero;
+use crate::local_array::bit_span::BitSpan;
 use crate::synth_int::AtomicU128;
 use crate::{impl_primitive_atomic_stride, AddressFamily};
 
@@ -466,7 +467,7 @@ impl From<(Result<u64, u64>, Result<u64, u64>)> for CasResult<u128> {
         }
     }
 }
-pub trait Stride:
+pub(crate) trait Stride:
     Sized
     + Debug
     + Eq
@@ -522,6 +523,14 @@ where
         i: u8,
     ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
 
+    fn ptr_bit_pos_from_index(
+        i: u8,
+    ) -> <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
+
+    fn cursor_from_bit_span(bs: BitSpan) -> u8;
+
+    fn ptr_cursor_from_bit_span(bs: BitSpan) -> u8;
+
     // Clear the bitmap to the right of the pointer and count the number of
     // ones. This number represents the index to the corresponding prefix in
     // the pfx_vec.
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index d4af5a50..7fa07785 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -357,7 +357,7 @@ impl<
                         self.parent_and_position.push(self.cur_ptr_iter);
                         let ptr_iter = next_node.more_specific_ptr_iter(
                             next_ptr,
-                            BitSpan::new(0, 0),
+                            BitSpan { bits: 0, len: 1 },
                         );
                         self.cur_ptr_iter = ptr_iter.wrap();
 
@@ -370,7 +370,7 @@ impl<
                         self.cur_pfx_iter = next_node
                             .more_specific_pfx_iter(
                                 next_ptr,
-                                BitSpan::new(0, 0),
+                                BitSpan::new(0, 1),
                                 false,
                             )
                             .wrap();
@@ -380,7 +380,7 @@ impl<
                         self.parent_and_position.push(self.cur_ptr_iter);
                         let ptr_iter = next_node.more_specific_ptr_iter(
                             next_ptr,
-                            BitSpan::new(0, 0),
+                            BitSpan { bits: 0, len: 1 },
                         );
                         self.cur_ptr_iter = ptr_iter.wrap();
 
@@ -393,7 +393,7 @@ impl<
                         self.cur_pfx_iter = next_node
                             .more_specific_pfx_iter(
                                 next_ptr,
-                                BitSpan::new(0, 0),
+                                BitSpan::new(0, 1),
                                 false,
                             )
                             .wrap();
@@ -403,7 +403,7 @@ impl<
                         self.parent_and_position.push(self.cur_ptr_iter);
                         let ptr_iter = next_node.more_specific_ptr_iter(
                             next_ptr,
-                            BitSpan::new(0, 0),
+                            BitSpan { bits: 0, len: 1 },
                         );
                         self.cur_ptr_iter = ptr_iter.wrap();
 
@@ -416,7 +416,7 @@ impl<
                         self.cur_pfx_iter = next_node
                             .more_specific_pfx_iter(
                                 next_ptr,
-                                BitSpan::new(0, 0),
+                                BitSpan::new(0, 1),
                                 false,
                             )
                             .wrap();
@@ -575,7 +575,8 @@ impl<
             // Our current prefix iterator for this node is done, look for
             // the next pfx iterator of the next child node in the current
             // ptr iterator.
-            trace!("start first ptr_iter");
+            trace!("resume ptr iterator {:?}", self.cur_ptr_iter);
+
             let mut next_ptr = self.cur_ptr_iter.next();
 
             // Our current ptr iterator is also done, maybe we have a parent
@@ -593,6 +594,7 @@ impl<
 
             if let Some(next_ptr) = next_ptr {
                 let node = if self.mui.is_none() {
+                    trace!("let's retriev node {}", next_ptr);
                     self.store.retrieve_node(next_ptr)
                 } else {
                     self.store.retrieve_node_for_mui(
@@ -609,7 +611,7 @@ impl<
                         self.parent_and_position.push(self.cur_ptr_iter);
                         let ptr_iter = next_node.more_specific_ptr_iter(
                             next_ptr,
-                            BitSpan::new(0, 0),
+                            BitSpan { bits: 0, len: 0 },
                         );
                         self.cur_ptr_iter = ptr_iter.wrap();
 
@@ -622,7 +624,7 @@ impl<
                         self.cur_pfx_iter = next_node
                             .more_specific_pfx_iter(
                                 next_ptr,
-                                BitSpan::new(0, 0),
+                                BitSpan::new(0, 1),
                                 false,
                             )
                             .wrap();
@@ -632,7 +634,7 @@ impl<
                         self.parent_and_position.push(self.cur_ptr_iter);
                         let ptr_iter = next_node.more_specific_ptr_iter(
                             next_ptr,
-                            BitSpan::new(0, 0),
+                            BitSpan { bits: 0, len: 0 },
                         );
                         self.cur_ptr_iter = ptr_iter.wrap();
 
@@ -645,7 +647,7 @@ impl<
                         self.cur_pfx_iter = next_node
                             .more_specific_pfx_iter(
                                 next_ptr,
-                                BitSpan::new(0, 0),
+                                BitSpan::new(0, 1),
                                 false,
                             )
                             .wrap();
@@ -655,7 +657,7 @@ impl<
                         self.parent_and_position.push(self.cur_ptr_iter);
                         let ptr_iter = next_node.more_specific_ptr_iter(
                             next_ptr,
-                            BitSpan::new(0, 0),
+                            BitSpan { bits: 0, len: 0 },
                         );
                         self.cur_ptr_iter = ptr_iter.wrap();
 
@@ -668,7 +670,7 @@ impl<
                         self.cur_pfx_iter = next_node
                             .more_specific_pfx_iter(
                                 next_ptr,
-                                BitSpan::new(0, 0),
+                                BitSpan::new(0, 1),
                                 false,
                             )
                             .wrap();
@@ -957,6 +959,8 @@ impl<
                                 true,
                             ),
                         );
+                        trace!("---------------------");
+                        trace!("start iterating nodes");
                         cur_ptr_iter = SizedNodeMoreSpecificIter::Stride4(
                             n.more_specific_ptr_iter(
                                 start_node_id,
@@ -1055,6 +1059,14 @@ impl<
                         );
                     }
                     SizedStrideRef::Stride4(n) => {
+                        trace!(
+                            "ALTERNATIVE {:#?}",
+                            n.add_more_specifics_at(
+                                start_bit_span.bits,
+                                start_bit_span.len,
+                                start_node_id
+                            )
+                        );
                         cur_pfx_iter = SizedPrefixIter::Stride4(
                             n.more_specific_pfx_iter(
                                 start_node_id,
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index bdcdc6af..eddfc170 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -187,7 +187,7 @@ macro_rules! impl_primitive_atomic_stride {
                 const STRIDE_LEN: u8 = $len;
 
                 fn get_bit_pos(nibble: u32, len: u8) -> $pfxsize {
-                    // trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
+                    trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
                     1 << (
                             <Self as Stride>::BITS - ((1 << len) - 1) as u8
                             - nibble as u8 - 1
@@ -198,6 +198,29 @@ macro_rules! impl_primitive_atomic_stride {
                     <$pfxsize>::try_from(1).unwrap().rotate_right(1) >> i
                 }
 
+                fn ptr_bit_pos_from_index(i: u8) -> $ptrsize {
+                    // trace!("pfx {} ptr {} strlen {}",
+                    // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
+                    <$ptrsize>::try_from(1).unwrap().rotate_right(1)
+                        >> (i + 1)
+                }
+
+                fn cursor_from_bit_span(bs: BitSpan) -> u8 {
+                    Self::get_bit_pos(bs.bits, bs.len)
+                    .leading_zeros() as u8 - 1
+                }
+
+                // Ptrbitarr searches are only done in the last half of
+                // the bitarray, in the len = S::STRIDE_LEN part. We need a
+                // complete BitSpan still to figure when to stop.
+                fn ptr_cursor_from_bit_span(bs: BitSpan) -> u8 {
+                    let p = Self::get_bit_pos(bs.bits << (4 - bs.len), 4)
+                        .leading_zeros() as u8;
+                    trace!("bs in {:?}", bs);
+                    trace!("pos {}", p);
+                    p
+                }
+
                 fn get_bit_pos_as_u8(nibble: u32, len: u8) -> u8 {
                     1 << (
                             <Self as Stride>::BITS - ((1 << len) - 1) as u8
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index f8ae89ab..3774eab4 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -1,4 +1,4 @@
-use core::ascii;
+use std::ops::Shr;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
 use std::{fmt::Debug, marker::PhantomData};
 
@@ -120,16 +120,16 @@ where
     pub(crate) fn more_specific_pfx_iter(
         &self,
         base_prefix: StrideNodeId<AF>,
-        start_bit_span: BitSpan,
-        skip_self: bool,
+        start_bs: BitSpan,
+        include_withdrawn: bool,
     ) -> NodeMoreSpecificsPrefixIter<AF, S> {
+        assert!(start_bs.len > 0);
         NodeMoreSpecificsPrefixIter::<AF, S> {
             pfxbitarr: self.pfxbitarr.load(),
             base_prefix,
-            start_bit_span,
-            cursor: start_bit_span,
-            skip_self,
-            _s: PhantomData,
+            start_bs,
+            cursor: S::cursor_from_bit_span(start_bs),
+            include_withdrawn: false,
         }
     }
 
@@ -138,13 +138,21 @@ where
     pub(crate) fn more_specific_ptr_iter(
         &self,
         base_prefix: StrideNodeId<AF>,
-        start_bit_span: BitSpan,
+        start_bs: BitSpan,
     ) -> NodeMoreSpecificChildIter<AF, S> {
+        let cursor = if start_bs.len == 0 {
+            15
+        } else {
+            S::ptr_cursor_from_bit_span(start_bs)
+        };
+        trace!("now we're really starting!");
+        trace!("start_bs {:?}", start_bs);
+        trace!("start cursor {}", cursor);
         NodeMoreSpecificChildIter::<AF, S> {
             ptrbitarr: self.ptrbitarr.load(),
             base_prefix,
-            start_bit_span,
-            cursor: None,
+            start_bs,
+            cursor,
         }
     }
 
@@ -824,12 +832,14 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
 // above example is therefore 1 << (5 - 4) = 2. Remember that a ptrbitarr
 // holds only one stride size (the largest for its stride size), so we're
 // done now.
+type PtrBitArr<S> = <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
+
 #[derive(Debug, Copy, Clone)]
 pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily, S: Stride> {
     base_prefix: StrideNodeId<AF>,
-    ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
-    start_bit_span: BitSpan,
-    cursor: Option<u32>,
+    ptrbitarr: PtrBitArr<S>,
+    start_bs: BitSpan,
+    cursor: u8,
 }
 
 impl<AF: AddressFamily, S: Stride> std::iter::Iterator
@@ -837,85 +847,152 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
 {
     type Item = StrideNodeId<AF>;
     fn next(&mut self) -> Option<Self::Item> {
-        // Early exits
-
-        // Empty bitmap
-        if self.ptrbitarr
-            == <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType::zero(
-            )
-        {
-            trace!("empty ptrbitrarr. this iterator is done.");
+        if self.ptrbitarr == PtrBitArr::<S>::zero() {
+            trace!("empty ptrbitarr. This iterator is done.");
             return None;
         }
 
-        // Previous iteration incremented the cursor beyond the stride size.
-        if let Some(cursor) = self.cursor {
-            if cursor >= (1 << (S::STRIDE_LEN - self.start_bit_span.len)) {
-                trace!(
-                    "cursor.bits >= (1 << (S::STRIDE_LEN - self.
-                    start_bit_span.len))"
-                );
-                trace!("cursor: {}", cursor);
-                trace!(
-                    "start_bit_span: {} {}",
-                    self.start_bit_span.bits,
-                    self.start_bit_span.len
-                );
-                return None;
-            }
-        }
+        let stop = <u8>::min(
+            (1 << (S::STRIDE_LEN - self.start_bs.len)) + self.cursor,
+            S::BITS - 2,
+        );
 
-        // No early exits, we're in business.
-        trace!("NodeMoreSpecificChildIter");
-        trace!("base_prefix {}", self.base_prefix);
-        trace!("stride_size {}", S::STRIDE_LEN);
         trace!(
-            "start_bit_span bits {} len {} ",
-            self.start_bit_span.bits,
-            self.start_bit_span.len
+            "base_prefix {}, start bit span {:?} start-stop cursor {}-{}",
+            self.base_prefix,
+            self.start_bs,
+            self.cursor,
+            stop
         );
-        trace!("cursor bits {:?}", self.cursor);
-        trace!("          x1  4   8  12  16  20  24  28  32");
         trace!("ptrbitarr {:032b}", self.ptrbitarr);
 
-        let start = if let Some(bits) = self.cursor {
-            bits
-        } else {
-            self.start_bit_span.bits
-        };
-        // We either stop if we have reached the maximum number of bits that
-        // we should check for this bit_span or we stop at the end of the
-        // stride (in case of a start_bit_span.bits == 0).
-        let stop = <u32>::min(
-            (1 << (S::STRIDE_LEN - self.start_bit_span.len)) + start,
-            (1 << S::STRIDE_LEN) - 1,
-        );
+        while self.cursor <= stop {
+            let bs = BitSpan::from_bit_pos_index(self.cursor);
+            // let bit_pos = S::get_bit_pos(self.cursor as u32, S::STRIDE_LEN);
+            let bit_pos = S::ptr_bit_pos_from_index(self.cursor);
 
-        trace!("start {:?} stop {}", start, stop);
-        for cursor in start..=stop {
-            // move the bit_span left with the amount of bits we're going to loop over.
-            // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
-            // becomes 0000 0000 0000 1100, then it will iterate over
-            // ...1100,...1101,...1110,...1111
-            let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
-            trace!("cmpbitarr x{:032b} {}", bit_pos, stop - start);
-            if (S::into_stride_size(self.ptrbitarr) & bit_pos) >
-                <<S as Stride>::AtomicPfxSize as AtomicBitmap
-                >::InnerType::zero()
-            {
-                trace!("bingo!");
-                self.cursor = Some(cursor + 1);
+            if log_enabled!(log::Level::Trace) {
+                trace!(
+                    "{:02}: {:05b} {:032b} bit_span: {:04b} ({:02}) (len: {})",
+                    self.cursor,
+                    self.cursor - 1,
+                    bit_pos,
+                    bs.bits,
+                    bs.bits,
+                    bs.len
+                );
+            }
 
-                trace!("next bit_span {} {} with cursor {:?}",
-                    self.start_bit_span.bits, self.start_bit_span.len, self.cursor);
-                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
+            // if bs.bits.count_ones() == bs.len as u32 {
+            //     self.cursor += (self.start_bs.bits
+            //         << (bs.len - self.start_bs.len))
+            //         as u8;
+            // }
+
+            if self.ptrbitarr & bit_pos > PtrBitArr::<S>::zero() {
+                self.cursor += 1;
+                trace!(
+                    ">> found node with more specific prefixes for
+                    base prefix {:?} bit span {:?} (cursor {})",
+                    self.base_prefix,
+                    bs,
+                    self.cursor
+                );
+                let pfx = self.base_prefix.add_nibble(bs.bits, S::STRIDE_LEN);
+                return Some(pfx);
             }
+            self.cursor += 1;
         }
-        trace!("No more nodes. End of the iterator.");
+
+        trace!("ptrbitarr iterator done.");
         None
     }
 }
 
+// impl<AF: AddressFamily, S: Stride> std::iter::Iterator
+//     for NodeMoreSpecificChildIter<AF, S>
+// {
+//     type Item = StrideNodeId<AF>;
+//     fn next(&mut self) -> Option<Self::Item> {
+//         // Early exits
+
+//         // Empty bitmap
+//         if self.ptrbitarr
+//             == <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType::zero(
+//             )
+//         {
+//             trace!("empty ptrbitrarr. this iterator is done.");
+//             return None;
+//         }
+
+//         // Previous iteration incremented the cursor beyond the stride size.
+//         if let Some(cursor) = self.cursor {
+//             if cursor >= (1 << (S::STRIDE_LEN - self.start_bit_span.len)) {
+//                 trace!(
+//                     "cursor.bits >= (1 << (S::STRIDE_LEN - self.
+//                     start_bit_span.len))"
+//                 );
+//                 trace!("cursor: {}", cursor);
+//                 trace!(
+//                     "start_bit_span: {} {}",
+//                     self.start_bit_span.bits,
+//                     self.start_bit_span.len
+//                 );
+//                 return None;
+//             }
+//         }
+
+//         // No early exits, we're in business.
+//         trace!("NodeMoreSpecificChildIter");
+//         trace!("base_prefix {}", self.base_prefix);
+//         trace!("stride_size {}", S::STRIDE_LEN);
+//         trace!(
+//             "start_bit_span bits {} len {} ",
+//             self.start_bit_span.bits,
+//             self.start_bit_span.len
+//         );
+//         trace!("cursor bits {:?}", self.cursor);
+//         trace!("          x1  4   8  12  16  20  24  28  32");
+//         trace!("ptrbitarr {:032b}", self.ptrbitarr);
+
+//         let start = if let Some(bits) = self.cursor {
+//             bits
+//         } else {
+//             self.start_bit_span.bits
+//         };
+//         // We either stop if we have reached the maximum number of bits that
+//         // we should check for this bit_span or we stop at the end of the
+//         // stride (in case of a start_bit_span.bits == 0).
+//         let stop = <u32>::min(
+//             (1 << (S::STRIDE_LEN - self.start_bit_span.len)) + start,
+//             (1 << S::STRIDE_LEN) - 1,
+//         );
+
+//         trace!("start {:?} stop {}", start, stop);
+//         for cursor in start..=stop {
+//             // move the bit_span left with the amount of bits we're going to loop over.
+//             // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
+//             // becomes 0000 0000 0000 1100, then it will iterate over
+//             // ...1100,...1101,...1110,...1111
+//             let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
+//             trace!("cmpbitarr x{:032b} {}", bit_pos, stop - start);
+//             if (S::into_stride_size(self.ptrbitarr) & bit_pos) >
+//                 <<S as Stride>::AtomicPfxSize as AtomicBitmap
+//                 >::InnerType::zero()
+//             {
+//                 trace!("bingo!");
+//                 self.cursor = Some(cursor + 1);
+
+//                 trace!("next bit_span {} {} with cursor {:?}",
+//                     self.start_bit_span.bits, self.start_bit_span.len, self.cursor);
+//                 return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
+//             }
+//         }
+//         trace!("No more nodes. End of the iterator.");
+//         None
+//     }
+// }
+
 impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride3> {
     pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
         SizedNodeMoreSpecificIter::<AF>::Stride3(self)
@@ -1092,20 +1169,19 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
 // the bit_span { bits: 2, len: 3 }, a.k.a. 0010 << 1. But now we will have
 // to go over a different amount of 1 << (5 - 4) = 2 iterations to reap the
 // next bit_spans of 0010 0 and 0010 1.
+type PfxBitArr<S> = <<S as super::atomic_stride::Stride>::AtomicPfxSize as super::atomic_stride::AtomicBitmap>::InnerType;
 
 pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily, S: Stride> {
     // immutables
     base_prefix: StrideNodeId<AF>,
-    pfxbitarr: <<S as super::atomic_stride::Stride>::AtomicPfxSize
-        as super::atomic_stride::AtomicBitmap>::InnerType,
+    pfxbitarr: PfxBitArr<S>,
     // we need to keep around only the `bits` part of the `bit_span`
     // technically, (it needs resetting the current state to it after each
     // prefix-length), but we'll keep the start-length as well for clarity
     // and increment it on a different field ('cur_len').
-    start_bit_span: BitSpan,
-    cursor: BitSpan,
-    skip_self: bool,
-    _s: PhantomData<S>,
+    start_bs: BitSpan,
+    cursor: u8,
+    include_withdrawn: bool,
 }
 
 impl<AF: AddressFamily, S: Stride> std::iter::Iterator
@@ -1114,152 +1190,207 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
     type Item = PrefixId<AF>;
 
     fn next(&mut self) -> Option<Self::Item> {
-        // Easy early exit conditions
-
         // Empty bitmap
-        if self.pfxbitarr
-            == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
-            )
-        {
+        if self.pfxbitarr == PfxBitArr::<S>::zero() {
             trace!("empty pfxbitarr. This iterator is done.");
             return None;
         }
 
-        // No early exits, We're in business.
-        trace!("len_offset {}", ((1 << self.cursor.len) - 1));
-        trace!("start_bit {}", self.start_bit_span.bits);
-        trace!(
-            "number of check bits in len {}",
-            (1 << (self.cursor.len - self.start_bit_span.len))
-        );
-
         trace!(
-            "next more specifics prefix iter start bits {} len {}",
-            self.start_bit_span.bits,
-            self.start_bit_span.len
+            "ms prefix iterator start_bs {:?} start cursor {}",
+            self.start_bs,
+            self.cursor
         );
+        trace!("pfxbitarr  {:064b}", self.pfxbitarr);
 
-        let mut res = None;
-
-        // Move to the next len if we're at the first prefix-length that
-        // matches, if `skip_self` is set. Typically this flag is set for the
-        // first stride. In the consecutive strides, we don't want to skip the
-        // base prefix, since that base_prefix is a more specific prefix of
-        // the one requested in the first stride.
-        if self.skip_self
-            && (1 << (self.cursor.len - self.start_bit_span.len)) == 1
-        {
-            // self.cursor.len += (self.start_bit_span.bits & 1) as u8;
-            trace!("skipping self");
-            self.start_bit_span.bits <<= 1;
-            self.cursor.bits = self.start_bit_span.bits;
-            self.cursor.len += 1;
-            self.skip_self = false;
-            trace!(
-                "new start bits {} len {}",
-                self.start_bit_span.bits,
-                self.start_bit_span.len
-            );
-            trace!(
-                "new cursor bits {} len {}",
-                self.cursor.bits,
-                self.cursor.len
-            );
-        }
-
-        // Previous iteration or the skip_self routine may have
-        // incremented the cursor beyond the end of the stride size.
-        if self.cursor.len > S::STRIDE_LEN {
-            trace!("cursor.len > S::STRIDE_LEN. This iterator is done.");
-            return None;
-        }
+        while self.cursor < S::BITS - 1 {
+            let res = BitSpan::from_bit_pos_index(self.cursor);
+            let bit_pos = S::bit_pos_from_index(self.cursor);
 
-        loop {
-            trace!(
-                "          x1  4   8  12  16  20  24  28  32  36  40  44
-                  48  52  56  60  64"
-            );
-            trace!(
-                "cmpnibble {:064b} ({} + {}) len {} stride_size {}",
-                S::get_bit_pos(self.cursor.bits, self.cursor.len),
-                (1 << self.cursor.len) - 1,
-                self.cursor.bits,
-                self.cursor.len + self.base_prefix.get_len(),
-                S::STRIDE_LEN
-            );
-
-            trace!("pfxbitarr {:064b}", self.pfxbitarr);
-
-            if (S::get_bit_pos(self.cursor.bits, self.cursor.len)
-                | self.pfxbitarr)
-                == self.pfxbitarr
-            {
+            if log_enabled!(log::Level::Trace) {
                 trace!(
-                    "found prefix with len {} at pos {} pfx len {}",
-                    self.cursor.len,
-                    self.cursor.bits,
-                    self.base_prefix.get_len() + self.cursor.len,
-                );
-                res = Some(
-                    self.base_prefix
-                        .add_nibble(self.cursor.bits, self.cursor.len)
-                        .into(),
+                    "{:02}: {:06b} {:064b} bit_span: {:05b} (len: {})",
+                    self.cursor,
+                    self.cursor - 1,
+                    S::bit_pos_from_index(self.cursor),
+                    res.bits,
+                    res.len
                 );
-                trace!("found prefix {:?}", res);
             }
 
-            // Determine when we're at the end of the bits applicable to
-            // this combo of this start_bit_span.
-            // bitspan offset:
-            // self.start_bit_span.bits
-            // number of matches in this length:
-            // 1 << (self.cursor.len - self.start_bit_span.len)
-            let max_pos_offset = self.start_bit_span.bits
-                + (1 << (self.cursor.len - self.start_bit_span.len))
-                - 1;
-
-            trace!(
-                "max_pos_offset {} > cursor bit_pos {}?",
-                max_pos_offset,
-                self.cursor.bits
-            );
-            trace!(
-                "number of check bits in len {}",
-                (1 << (self.cursor.len - self.start_bit_span.len))
-            );
-
-            // case 1. At the beginning or inside a prefix-length.
-            if max_pos_offset > self.cursor.bits {
-                self.cursor.bits += 1;
-            }
-            // case 2. At the end of a prefix-lengths in this stride.
-            else if self.cursor.len < S::STRIDE_LEN {
-                trace!("move len to {}", self.cursor.len + 1);
-                self.start_bit_span.bits <<= 1;
-                self.cursor.bits = self.start_bit_span.bits;
-                self.cursor.len += 1;
-            }
-            // case 4. At the end of a prefix-length, but not at the end of the pfxbitarr.
-            else {
-                self.start_bit_span.bits <<= 1;
-                self.cursor.bits = self.start_bit_span.bits;
-                self.cursor.len += 1;
-                trace!(
-                    "end of stride, next cursor bits {} len {}",
-                    self.cursor.bits,
-                    self.cursor.len
-                );
-                return res;
+            if res.bits.count_ones() == res.len as u32 {
+                self.cursor += (self.start_bs.bits
+                    << (res.len - self.start_bs.len))
+                    as u8;
             }
 
-            trace!("some res {:?}", res);
-            if res.is_some() {
-                return res;
+            if self.pfxbitarr & bit_pos > PfxBitArr::<S>::zero() {
+                self.cursor += 1;
+                trace!("found more specific prefix for bit span {:?}", res);
+                return Some(
+                    self.base_prefix.add_nibble(res.bits, res.len).into(),
+                );
             }
+            self.cursor += 1;
         }
+
+        trace!("pfxbitarr iterator done.");
+        None
     }
 }
 
+// impl<AF: AddressFamily, S: Stride> std::iter::Iterator
+//     for NodeMoreSpecificsPrefixIter<AF, S>
+// {
+//     type Item = PrefixId<AF>;
+
+//     fn next(&mut self) -> Option<Self::Item> {
+//         // Easy early exit conditions
+
+//         // Empty bitmap
+//         if self.pfxbitarr
+//             == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
+//             )
+//         {
+//             trace!("empty pfxbitarr. This iterator is done.");
+//             return None;
+//         }
+
+//         // No early exits, We're in business.
+//         trace!("len_offset {}", ((1 << self.cursor.len) - 1));
+//         trace!("start_bit {}", self.start_bit_span.bits);
+//         trace!(
+//             "number of check bits in len {}",
+//             (1 << (self.cursor.len - self.start_bit_span.len))
+//         );
+
+//         trace!(
+//             "next more specifics prefix iter start bits {} len {}",
+//             self.start_bit_span.bits,
+//             self.start_bit_span.len
+//         );
+
+//         let mut res = None;
+
+//         // Move to the next len if we're at the first prefix-length that
+//         // matches, if `skip_self` is set. Typically this flag is set for the
+//         // first stride. In the consecutive strides, we don't want to skip the
+//         // base prefix, since that base_prefix is a more specific prefix of
+//         // the one requested in the first stride.
+//         if self.skip_self
+//             && (1 << (self.cursor.len - self.start_bit_span.len)) == 1
+//         {
+//             // self.cursor.len += (self.start_bit_span.bits & 1) as u8;
+//             trace!("skipping self");
+//             self.start_bit_span.bits <<= 1;
+//             self.cursor.bits = self.start_bit_span.bits;
+//             self.cursor.len += 1;
+//             self.skip_self = false;
+//             trace!(
+//                 "new start bits {} len {}",
+//                 self.start_bit_span.bits,
+//                 self.start_bit_span.len
+//             );
+//             trace!(
+//                 "new cursor bits {} len {}",
+//                 self.cursor.bits,
+//                 self.cursor.len
+//             );
+//         }
+
+//         // Previous iteration or the skip_self routine may have
+//         // incremented the cursor beyond the end of the stride size.
+//         if self.cursor.len > S::STRIDE_LEN {
+//             trace!("cursor.len > S::STRIDE_LEN. This iterator is done.");
+//             return None;
+//         }
+
+//         loop {
+//             trace!(
+//                 "          x1  4   8  12  16  20  24  28  32  36  40  44
+//                   48  52  56  60  64"
+//             );
+//             trace!(
+//                 "cmpnibble {:064b} ({} + {}) len {} stride_size {}",
+//                 S::get_bit_pos(self.cursor.bits, self.cursor.len),
+//                 (1 << self.cursor.len) - 1,
+//                 self.cursor.bits,
+//                 self.cursor.len + self.base_prefix.get_len(),
+//                 S::STRIDE_LEN
+//             );
+
+//             trace!("pfxbitarr {:064b}", self.pfxbitarr);
+
+//             if (S::get_bit_pos(self.cursor.bits, self.cursor.len)
+//                 | self.pfxbitarr)
+//                 == self.pfxbitarr
+//             {
+//                 trace!(
+//                     "found prefix with len {} at pos {} pfx len {}",
+//                     self.cursor.len,
+//                     self.cursor.bits,
+//                     self.base_prefix.get_len() + self.cursor.len,
+//                 );
+//                 res = Some(
+//                     self.base_prefix
+//                         .add_nibble(self.cursor.bits, self.cursor.len)
+//                         .into(),
+//                 );
+//                 trace!("found prefix {:?}", res);
+//             }
+
+//             // Determine when we're at the end of the bits applicable to
+//             // this combo of this start_bit_span.
+//             // bitspan offset:
+//             // self.start_bit_span.bits
+//             // number of matches in this length:
+//             // 1 << (self.cursor.len - self.start_bit_span.len)
+//             let max_pos_offset = self.start_bit_span.bits
+//                 + (1 << (self.cursor.len - self.start_bit_span.len))
+//                 - 1;
+
+//             trace!(
+//                 "max_pos_offset {} > cursor bit_pos {}?",
+//                 max_pos_offset,
+//                 self.cursor.bits
+//             );
+//             trace!(
+//                 "number of check bits in len {}",
+//                 (1 << (self.cursor.len - self.start_bit_span.len))
+//             );
+
+//             // case 1. At the beginning or inside a prefix-length.
+//             if max_pos_offset > self.cursor.bits {
+//                 self.cursor.bits += 1;
+//             }
+//             // case 2. At the end of a prefix-lengths in this stride.
+//             else if self.cursor.len < S::STRIDE_LEN {
+//                 trace!("move len to {}", self.cursor.len + 1);
+//                 self.start_bit_span.bits <<= 1;
+//                 self.cursor.bits = self.start_bit_span.bits;
+//                 self.cursor.len += 1;
+//             }
+//             // case 4. At the end of a prefix-length, but not at the end of the pfxbitarr.
+//             else {
+//                 self.start_bit_span.bits <<= 1;
+//                 self.cursor.bits = self.start_bit_span.bits;
+//                 self.cursor.len += 1;
+//                 trace!(
+//                     "end of stride, next cursor bits {} len {}",
+//                     self.cursor.bits,
+//                     self.cursor.len
+//                 );
+//                 return res;
+//             }
+
+//             trace!("some res {:?}", res);
+//             if res.is_some() {
+//                 return res;
+//             }
+//         }
+//     }
+// }
+
 impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride3> {
     pub fn wrap(self) -> SizedPrefixIter<AF> {
         SizedPrefixIter::<AF>::Stride3(self)

From 2d6806546cccfc74144efc626a4802be6cb1421d Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Mon, 27 Jan 2025 16:41:41 +0100
Subject: [PATCH 057/147] non-looping ChildNodeIterator

---
 Cargo.toml                                 |   1 +
 src/local_array/in_memory/atomic_stride.rs |  13 +-
 src/local_array/in_memory/macros.rs        |  22 ++
 src/local_array/in_memory/node.rs          | 132 +++++---
 tests/concurrency.rs                       | 121 ++++---
 tests/more-specifics.rs                    | 358 +++++++++------------
 6 files changed, 351 insertions(+), 296 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index dad9394b..a5aaac3d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -45,6 +45,7 @@ uuid = { version = "1.11.0", features = ["v4", "fast-rng"] }
 serde = "1.0.216"
 serde_derive = "1.0.216"
 serde_json = "1.0.133"
+num-traits = "0.2.19"
 
 [dev-dependencies]
 csv         = { version = "1" }
diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 7f01827b..47548091 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -40,7 +40,9 @@ where
         + Zero
         + PartialOrd
         + std::ops::BitAnd<Output = Self::InnerType>
-        + std::ops::BitOr<Output = Self::InnerType>;
+        + std::ops::BitOr<Output = Self::InnerType>
+        + std::ops::BitXor<Output = Self::InnerType>
+        + num_traits::PrimInt;
 
     fn new() -> Self;
     fn inner(self) -> Self::InnerType;
@@ -531,6 +533,15 @@ where
 
     fn ptr_cursor_from_bit_span(bs: BitSpan) -> u8;
 
+    fn ptr_range(
+        ptrbitarr: <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::
+            InnerType,
+        range: BitSpan,
+    ) -> (
+        <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
+        u8,
+    );
+
     // Clear the bitmap to the right of the pointer and count the number of
     // ones. This number represents the index to the corresponding prefix in
     // the pfx_vec.
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index eddfc170..ae302761 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -210,6 +210,28 @@ macro_rules! impl_primitive_atomic_stride {
                     .leading_zeros() as u8 - 1
                 }
 
+                fn ptr_range(
+                    ptrbitarr: $ptrsize,
+                    bs: BitSpan
+                ) -> ($ptrsize, u8) {
+                    let start: u8 = (bs.bits << (4 - bs.len)) as u8;
+                    let stop: u8 = start + (1 << (4 - bs.len));
+                    let mask: $ptrsize = (
+                        (((1_u32 << (stop as u32 - start as u32)) - 1)
+                             as u32
+                    )
+                        .rotate_right(stop as u32) >> 16)
+                        .try_into()
+                        .unwrap();
+                    trace!("- mask      {:032b}", mask);
+                    trace!("- ptrbitarr {:032b}", ptrbitarr);
+                    trace!("- shl bitar {:032b}", ptrbitarr & mask);
+
+                    // if ptrbitarr & mask == <$ptrsize>::zero() { panic!("stop"); }
+
+                    (ptrbitarr & mask, start as u8)
+                }
+
                 // Ptrbitarr searches are only done in the last half of
                 // the bitarray, in the len = S::STRIDE_LEN part. We need a
                 // complete BitSpan still to figure when to stop.
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 3774eab4..2b365092 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -1,4 +1,4 @@
-use std::ops::Shr;
+use num_traits::PrimInt;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
 use std::{fmt::Debug, marker::PhantomData};
 
@@ -140,19 +140,18 @@ where
         base_prefix: StrideNodeId<AF>,
         start_bs: BitSpan,
     ) -> NodeMoreSpecificChildIter<AF, S> {
-        let cursor = if start_bs.len == 0 {
-            15
-        } else {
-            S::ptr_cursor_from_bit_span(start_bs)
-        };
+        let ptrbitarr = self.ptrbitarr.load();
+        let (bitrange, start_cursor) = S::ptr_range(ptrbitarr, start_bs);
+
         trace!("now we're really starting!");
         trace!("start_bs {:?}", start_bs);
-        trace!("start cursor {}", cursor);
+        trace!("ptrbitarr {:032b}", ptrbitarr);
+        // trace!("start cursor {}", start_cursor);
         NodeMoreSpecificChildIter::<AF, S> {
-            ptrbitarr: self.ptrbitarr.load(),
+            bitrange,
             base_prefix,
             start_bs,
-            cursor,
+            start_cursor,
         }
     }
 
@@ -837,9 +836,9 @@ type PtrBitArr<S> = <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
 #[derive(Debug, Copy, Clone)]
 pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily, S: Stride> {
     base_prefix: StrideNodeId<AF>,
-    ptrbitarr: PtrBitArr<S>,
+    bitrange: PtrBitArr<S>,
     start_bs: BitSpan,
-    cursor: u8,
+    start_cursor: u8,
 }
 
 impl<AF: AddressFamily, S: Stride> std::iter::Iterator
@@ -847,65 +846,92 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
 {
     type Item = StrideNodeId<AF>;
     fn next(&mut self) -> Option<Self::Item> {
-        if self.ptrbitarr == PtrBitArr::<S>::zero() {
+        if self.bitrange == PtrBitArr::<S>::zero() {
             trace!("empty ptrbitarr. This iterator is done.");
             return None;
         }
 
-        let stop = <u8>::min(
-            (1 << (S::STRIDE_LEN - self.start_bs.len)) + self.cursor,
-            S::BITS - 2,
-        );
+        let cursor = self.bitrange.leading_zeros() as u8 + 15;
+        // let cursor = <u8>::max(
+        //     self.start_cursor,
+        //     self.ptrbitarr.leading_zeros() as u8 + 15,
+        // );
+
+        // let stop = <u8>::min(
+        //     (1 << (S::STRIDE_LEN - self.start_bs.len)) + cursor,
+        //     S::BITS - 2,
+        // );
+        trace!("LZCNT {}", self.bitrange.leading_zeros());
+
+        // if self.bitrange.leading_zeros() == 0 {
+        //     trace!("bitrange   {:032b}", self.bitrange);
+        //     panic!("empty bitrange. This iterator is done.");
+        //     return None;
+        // }
 
         trace!(
             "base_prefix {}, start bit span {:?} start-stop cursor {}-{}",
             self.base_prefix,
             self.start_bs,
-            self.cursor,
-            stop
+            self.start_cursor,
+            <u8>::min(
+                (1 << (S::STRIDE_LEN - self.start_bs.len))
+                    + self.start_cursor,
+                S::BITS - 2
+            )
         );
-        trace!("ptrbitarr {:032b}", self.ptrbitarr);
 
-        while self.cursor <= stop {
-            let bs = BitSpan::from_bit_pos_index(self.cursor);
-            // let bit_pos = S::get_bit_pos(self.cursor as u32, S::STRIDE_LEN);
-            let bit_pos = S::ptr_bit_pos_from_index(self.cursor);
+        trace!("bitrange  {:032b}", self.bitrange);
 
-            if log_enabled!(log::Level::Trace) {
-                trace!(
-                    "{:02}: {:05b} {:032b} bit_span: {:04b} ({:02}) (len: {})",
-                    self.cursor,
-                    self.cursor - 1,
-                    bit_pos,
-                    bs.bits,
-                    bs.bits,
-                    bs.len
-                );
-            }
+        // if cursor > stop {
+        //     trace!("ptrbitarr iterator done.");
+        //     return None;
+        // }
 
-            // if bs.bits.count_ones() == bs.len as u32 {
-            //     self.cursor += (self.start_bs.bits
-            //         << (bs.len - self.start_bs.len))
-            //         as u8;
-            // }
+        self.bitrange = self.bitrange ^ S::ptr_bit_pos_from_index(cursor);
 
-            if self.ptrbitarr & bit_pos > PtrBitArr::<S>::zero() {
-                self.cursor += 1;
-                trace!(
-                    ">> found node with more specific prefixes for
+        trace!("mask      {:032b}", S::ptr_bit_pos_from_index(cursor));
+        trace!("next br   {:032b}", self.bitrange);
+
+        // while self.cursor <= stop {
+        // let bit_pos = S::get_bit_pos(self.cursor as u32, S::STRIDE_LEN);
+
+        let bs = BitSpan::from_bit_pos_index(cursor);
+        if log_enabled!(log::Level::Trace) {
+            let bit_pos = S::ptr_bit_pos_from_index(cursor);
+            trace!(
+                "{:02}: {:05b} {:032b} bit_span: {:04b} ({:02}) (len: {})",
+                cursor,
+                cursor - 1,
+                bit_pos,
+                bs.bits,
+                bs.bits,
+                bs.len
+            );
+            trace!(
+                ">> found node with more specific prefixes for
                     base prefix {:?} bit span {:?} (cursor {})",
-                    self.base_prefix,
-                    bs,
-                    self.cursor
-                );
-                let pfx = self.base_prefix.add_nibble(bs.bits, S::STRIDE_LEN);
-                return Some(pfx);
-            }
-            self.cursor += 1;
+                self.base_prefix,
+                bs,
+                cursor
+            );
         }
 
-        trace!("ptrbitarr iterator done.");
-        None
+        // if bs.bits.count_ones() == bs.len as u32 {
+        //     self.cursor += (self.start_bs.bits
+        //         << (bs.len - self.start_bs.len))
+        //         as u8;
+        // }
+
+        // if self.ptrbitarr & bit_pos > PtrBitArr::<S>::zero() {
+        // self.cursor += 1;
+        let pfx = self.base_prefix.add_nibble(bs.bits, S::STRIDE_LEN);
+        Some(pfx)
+        // }
+        // self.cursor += 1;
+        // }
+
+        // None
     }
 }
 
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index 6d2eef4f..ec0f5c56 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -376,15 +376,14 @@ fn test_concurrent_updates_1(
     Ok(())
 }
 
-rotonda_store::all_strategies_arced![
-    test_cc_updates_2;
-    test_concurrent_updates_2;
-    BeBytesAsn
-];
+// rotonda_store::all_strategies_arced![
+//     test_cc_updates_2;
+//     test_concurrent_updates_2;
+//     BeBytesAsn
+// ];
 
-// #[test]
-fn test_concurrent_updates_2(
-    tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>,
+#[test]
+fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>,
 ) -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
@@ -413,14 +412,13 @@ fn test_concurrent_updates_2(
 
     let cur_ltime = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
 
-    println!("PersistOnly strategy starting...");
     let store_config = StoreConfig {
         persist_strategy: rotonda_store::rib::PersistStrategy::MemoryOnly,
         persist_path: "/tmp/rotonda/".into(),
     };
-    // let tree_bitmap = std::sync::Arc::new(
-    //     MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
-    // );
+    let tree_bitmap = std::sync::Arc::new(
+        MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
+    );
 
     let _: Vec<_> =
         vec![pfx_vec_1.clone(), pfx_vec_2.clone(), pfx_vec_3.clone()]
@@ -668,14 +666,13 @@ fn test_concurrent_updates_2(
         include_history: IncludeHistory::None,
     };
 
-    // let pfx0 = Prefix::from_str("128.0.0.0/2").unwrap();
-    let pfx128 = Prefix::from_str("0.0.0.0/0").unwrap();
+    // should cover all the prefixes
+    // let pfx0 = Prefix::from_str("184.0.0.0/6").unwrap();
+    let pfx128 = Prefix::from_str("128.0.0.0/1").unwrap();
     let guard = rotonda_store::epoch::pin();
     // let res0 = tree_bitmap.match_prefix(&pfx0, &match_options, &guard);
-    let res128 = tree_bitmap.match_prefix(&pfx128, &match_options, &guard);
 
     // println!("000 {:#?}", res0);
-    println!("128 {:#?}", res128);
 
     assert!(tree_bitmap
         .contains(&Prefix::from_str("185.34.14.0/24").unwrap(), None));
@@ -697,19 +694,19 @@ fn test_concurrent_updates_2(
         tree_bitmap.contains(&Prefix::from_str("32.0.0.0/4").unwrap(), None)
     );
 
-    assert_eq!(
-        tree_bitmap
-            .match_prefix(
-                &Prefix::from_str("0.0.0.0/2").unwrap(),
-                &match_options,
-                &guard
-            )
-            .more_specifics
-            .unwrap()
-            .len(),
-        1
-    );
+    let mp02 = tree_bitmap
+        .match_prefix(
+            &Prefix::from_str("0.0.0.0/2").unwrap(),
+            &match_options,
+            &guard,
+        )
+        .more_specifics
+        .unwrap();
+    println!("0/2 {}", mp02);
+    assert_eq!(mp02.len(), 1);
 
+    let res128 = tree_bitmap.match_prefix(&pfx128, &match_options, &guard);
+    println!("128 {:#?}", res128);
     // let guard = rotonda_store::epoch::pin();
     // println!(
     //     "more_specifics_iter_from {:#?}",
@@ -722,30 +719,36 @@ fn test_concurrent_updates_2(
         .collect::<Vec<_>>()
         .len();
     assert_eq!(active_len, all_pfxs_iter.len());
-    let len_2 = res128.more_specifics.unwrap().v4.len();
-    // + res128.more_specifics.unwrap().v4.len();
-    assert_eq!(active_len, len_2);
+    // let len_2 = res0.more_specifics.unwrap().v4.len()
+    assert_eq!(active_len, res128.more_specifics.unwrap().v4.len());
 
     Ok(())
 }
 
 #[test]
-fn shit_test() -> Result<(), Box<dyn std::error::Error>> {
+fn more_specifics_short_lengths() -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
     println!("PersistOnly strategy starting...");
     let store_config = StoreConfig {
-        persist_strategy: rotonda_store::rib::PersistStrategy::MemoryOnly,
+        persist_strategy: rotonda_store::rib::PersistStrategy::PersistOnly,
         persist_path: "/tmp/rotonda/".into(),
     };
     let tree_bitmap = std::sync::Arc::new(
         MultiThreadedStore::<NoMeta>::new_with_config(store_config)?,
     );
+    let match_options = MatchOptions {
+        match_type: rotonda_store::MatchType::EmptyMatch,
+        include_withdrawn: false,
+        include_less_specifics: false,
+        include_more_specifics: true,
+        mui: None,
+        include_history: IncludeHistory::None,
+    };
 
     let pfx1 = Prefix::from_str("185.34.0.0/16")?;
     let pfx2 = Prefix::from_str("185.34.3.0/24")?;
-    // let search_pfx = Prefix::from_str("128.0.0.0/1")?;
-    let search_pfx = Prefix::from_str("0.0.0.0/0")?;
+    let pfx3 = Prefix::from_str("185.34.4.0/24")?;
 
     tree_bitmap
         .insert(
@@ -763,16 +766,52 @@ fn shit_test() -> Result<(), Box<dyn std::error::Error>> {
         )
         .unwrap();
 
-    println!("-------------------");
+    tree_bitmap
+        .insert(
+            &pfx3,
+            Record::new(1, 0, RouteStatus::Active, NoMeta::Empty),
+            None,
+        )
+        .unwrap();
 
     let guard = rotonda_store::epoch::pin();
-    let mp = tree_bitmap
-        .more_specifics_iter_from(&search_pfx, None, false, &guard)
-        .collect::<Vec<_>>();
 
-    println!("more specifics : {:#?}", mp);
+    assert!(tree_bitmap.contains(&pfx1, None));
+    assert!(tree_bitmap.contains(&pfx2, None));
+    assert!(tree_bitmap.contains(&pfx3, None));
+
+    println!("-------------------");
+    // let search_pfx = Prefix::from_str("0.0.0.0/0")?;
+    // let mp = tree_bitmap
+    //     .more_specifics_iter_from(&search_pfx, None, false, &guard)
+    //     .collect::<Vec<_>>();
+
+    // println!("more specifics : {:#?}", mp);
+
+    // assert_eq!(mp.len(), 2);
+
+    let search_pfx = Prefix::from_str("128.0.0.0/1")?;
+
+    let m = tree_bitmap.match_prefix(&search_pfx, &match_options, &guard);
+
+    // let mp = tree_bitmap
+    //     .more_specifics_iter_from(&search_pfx, None, false, &guard)
+    //     .collect::<Vec<_>>();
+
+    println!(
+        "more specifics#0: {}",
+        m.more_specifics.as_ref().unwrap()[0]
+    );
+    println!(
+        "more specifics#1: {}",
+        m.more_specifics.as_ref().unwrap()[1]
+    );
+    println!(
+        "more specifics#2: {}",
+        m.more_specifics.as_ref().unwrap()[2]
+    );
 
-    assert_eq!(mp.len(), 2);
+    assert_eq!(m.more_specifics.map(|mp| mp.len()), Some(3));
 
     Ok(())
 }
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 2b677ab3..234b3b89 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -1,216 +1,172 @@
 // type Prefix4<'a> = Prefix<u32, PrefixAs>;
-mod tests {
-    use inetnum::addr::Prefix;
-    use rotonda_store::meta_examples::PrefixAs;
-    use rotonda_store::{prelude::multi::*, prelude::*};
+use inetnum::addr::Prefix;
+use rotonda_store::meta_examples::PrefixAs;
+use rotonda_store::{prelude::multi::*, prelude::*};
 
-    use std::error::Error;
+use std::error::Error;
 
-    #[test]
-    fn test_more_specifics() -> Result<(), Box<dyn Error>> {
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
-        let pfxs = vec![
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 24), // 0
-            //
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 25), // 1
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 128).into(),
+mod common {
+    use std::io::Write;
+
+    pub fn init() {
+        let _ = env_logger::builder()
+            .format(|buf, record| writeln!(buf, "{}", record.args()))
+            .is_test(true)
+            .try_init();
+    }
+}
+
+#[test]
+fn test_more_specifics() -> Result<(), Box<dyn Error>> {
+    crate::common::init();
+
+    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    let pfxs = vec![
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 24), // 0
+        //
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 25), // 1
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 128).into(), 25), // 2
+        //
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 26), // 3
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 64).into(), 26), // 4
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 128).into(), 26), // 5
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 192).into(), 26), // 6
+        //
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 27), // 7
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 32).into(), 27), // 8
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 64).into(), 27), // 9
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 128).into(), 27), // 10
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 160).into(), 27), // 11
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 192).into(), 27), // 12
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 224).into(), 27), // 13
+        //
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 32), // 14
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 31).into(), 32), // 15
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 32).into(), 32), // 16
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 63).into(), 32), // 17
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 64).into(), 32), // 18
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 127).into(), 32), // 19
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 128).into(), 32), // 20
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 159).into(), 32), // 21
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 160).into(), 32), // 22
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 191).into(), 32), // 23
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 192).into(), 32), // 24
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 223).into(), 32), // 25
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 224).into(), 32), // 26
+        Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 255).into(), 32), // 27
+    ];
+    for pfx in pfxs.iter().flatten() {
+        tree_bitmap.insert(
+            pfx,
+            Record::new(
+                0,
+                0,
+                RouteStatus::Active,
+                PrefixAs::new_from_u32(666),
+            ),
+            None,
+        )?;
+    }
+    println!("------ end of inserts\n");
+
+    // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
+    let guard = &epoch::pin();
+    for spfx in &[
+        (
+            &Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 23),
+            None,
+            // These are the indexes to pfxs.2 vec.
+            // These are all supposed to show up in the result.
+            vec![
+                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
+                18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
+            ],
+        ),
+        (
+            &Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 24),
+            Some(Prefix::new(
+                std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
+                24,
+            )?),
+            // These are the indexes to pfxs.2 vec.
+            // These are all supposed to show up in the result.
+            vec![
+                1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
+                18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
+            ],
+        ),
+        (
+            &Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 25),
+            Some(Prefix::new(
+                std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
                 25,
-            ), // 2
-            //
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 26), // 3
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 64).into(), 26), // 4
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 128).into(),
+            )?),
+            vec![3, 4, 7, 8, 9, 14, 15, 16, 17, 18, 19],
+        ),
+        (
+            &Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 26),
+            Some(Prefix::new(
+                std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
                 26,
-            ), // 5
-            Prefix::new(
+            )?),
+            vec![7, 8, 14, 15, 16, 17],
+        ),
+        (
+            &Prefix::new(
                 std::net::Ipv4Addr::new(130, 55, 240, 192).into(),
                 26,
-            ), // 6
-            //
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 27), // 7
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 32).into(), 27), // 8
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 64).into(), 27), // 9
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 128).into(),
-                27,
-            ), // 10
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 160).into(),
-                27,
-            ), // 11
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 192).into(),
-                27,
-            ), // 12
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 224).into(),
-                27,
-            ), // 13
-            //
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 32), // 14
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 31).into(), 32), // 15
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 32).into(), 32), // 16
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 63).into(), 32), // 17
-            Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 64).into(), 32), // 18
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 127).into(),
-                32,
-            ), // 19
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 128).into(),
-                32,
-            ), // 20
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 159).into(),
-                32,
-            ), // 21
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 160).into(),
-                32,
-            ), // 22
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 191).into(),
-                32,
-            ), // 23
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 192).into(),
-                32,
-            ), // 24
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 223).into(),
-                32,
-            ), // 25
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 224).into(),
-                32,
-            ), // 26
-            Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 255).into(),
-                32,
-            ), // 27
-        ];
-        for pfx in pfxs.iter().flatten() {
-            tree_bitmap.insert(
-                pfx,
-                Record::new(
-                    0,
-                    0,
-                    RouteStatus::Active,
-                    PrefixAs::new_from_u32(666),
-                ),
-                None,
-            )?;
-        }
-        println!("------ end of inserts\n");
-
-        // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
-        let guard = &epoch::pin();
-        for spfx in &[
-            (
-                &Prefix::new(
-                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
-                    23,
-                ),
-                None,
-                // These are the indexes to pfxs.2 vec.
-                // These are all supposed to show up in the result.
-                vec![
-                    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
-                    17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
-                ],
-            ),
-            (
-                &Prefix::new(
-                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
-                    24,
-                ),
-                Some(Prefix::new(
-                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
-                    24,
-                )?),
-                // These are the indexes to pfxs.2 vec.
-                // These are all supposed to show up in the result.
-                vec![
-                    1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
-                    17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
-                ],
-            ),
-            (
-                &Prefix::new(
-                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
-                    25,
-                ),
-                Some(Prefix::new(
-                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
-                    25,
-                )?),
-                vec![3, 4, 7, 8, 9, 14, 15, 16, 17, 18, 19],
-            ),
-            (
-                &Prefix::new(
-                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
-                    26,
-                ),
-                Some(Prefix::new(
-                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
-                    26,
-                )?),
-                vec![7, 8, 14, 15, 16, 17],
             ),
-            (
-                &Prefix::new(
-                    std::net::Ipv4Addr::new(130, 55, 240, 192).into(),
-                    26,
-                ),
-                Some(Prefix::new(
-                    std::net::Ipv4Addr::new(130, 55, 240, 192).into(),
-                    26,
-                )?),
-                vec![12, 13, 24, 25, 26, 27],
-            ),
-            (
-                &Prefix::new(std::net::Ipv4Addr::new(0, 0, 0, 0).into(), 0),
-                None,
-                // These are the indexes to pfxs.2 vec.
-                // These are all supposed to show up in the result.
-                vec![
-                    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
-                    17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
-                ],
-            ),
-        ] {
-            println!("search for: {}", (*spfx.0)?);
-            let found_result = tree_bitmap.match_prefix(
-                &spfx.0.unwrap(),
-                &MatchOptions {
-                    match_type: MatchType::ExactMatch,
-                    include_withdrawn: false,
-                    include_less_specifics: false,
-                    include_more_specifics: true,
-                    mui: None,
-                    include_history: IncludeHistory::None,
-                },
-                guard,
-            );
-            println!("em/m-s: {:#?}", found_result);
+            Some(Prefix::new(
+                std::net::Ipv4Addr::new(130, 55, 240, 192).into(),
+                26,
+            )?),
+            vec![12, 13, 24, 25, 26, 27],
+        ),
+        (
+            &Prefix::new(std::net::Ipv4Addr::new(0, 0, 0, 0).into(), 0),
+            None,
+            // These are the indexes to pfxs.2 vec.
+            // These are all supposed to show up in the result.
+            vec![
+                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
+                18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
+            ],
+        ),
+    ] {
+        println!("search for: {}", (*spfx.0)?);
+        let found_result = tree_bitmap.match_prefix(
+            &spfx.0.unwrap(),
+            &MatchOptions {
+                match_type: MatchType::ExactMatch,
+                include_withdrawn: false,
+                include_less_specifics: false,
+                include_more_specifics: true,
+                mui: None,
+                include_history: IncludeHistory::None,
+            },
+            guard,
+        );
+        println!("em/m-s: {:#?}", found_result);
 
-            let more_specifics = found_result.more_specifics.unwrap();
-            assert_eq!(found_result.prefix, spfx.1);
+        assert!(tree_bitmap.contains(&pfxs[25].unwrap(), None));
+        assert!(tree_bitmap.contains(&pfxs[26].unwrap(), None));
+        assert!(tree_bitmap.contains(&pfxs[27].unwrap(), None));
+        let ms2 = tree_bitmap.more_specifics_keys_from(&spfx.0.unwrap());
+        println!("ms2 {:#?}", ms2);
+        let more_specifics = found_result.more_specifics.unwrap();
+        assert_eq!(found_result.prefix, spfx.1);
 
-            assert_eq!(&more_specifics.len(), &spfx.2.len());
+        assert_eq!(&more_specifics.len(), &spfx.2.len());
 
-            for i in spfx.2.iter() {
-                print!("{} ", i);
+        for i in spfx.2.iter() {
+            print!("{} ", i);
 
-                let result_pfx = more_specifics
-                    .iter()
-                    .find(|pfx| pfx.prefix == pfxs[*i].unwrap());
-                assert!(result_pfx.is_some());
-            }
-            println!("-----------");
+            let result_pfx = more_specifics
+                .iter()
+                .find(|pfx| pfx.prefix == pfxs[*i].unwrap());
+            assert!(result_pfx.is_some());
         }
-        Ok(())
+        println!("-----------");
     }
+    Ok(())
 }

From 16847bcd348c664579f486de4c736ddda6f04033 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Mon, 3 Feb 2025 12:34:46 +0100
Subject: [PATCH 058/147] fix NodeMoreSpecificsPrefixIter

---
 src/af.rs                              |  17 +-
 src/local_array/in_memory/iterators.rs |  85 +++++----
 src/local_array/in_memory/macros.rs    |   4 +-
 src/local_array/in_memory/node.rs      | 249 ++++++++++++++++---------
 src/local_array/in_memory/query.rs     |   1 +
 src/local_array/in_memory/tree.rs      |  88 +++++++--
 src/local_array/query.rs               |  14 --
 src/local_array/tests.rs               |  24 ++-
 tests/more-more-specifics.rs           |  48 ++++-
 tests/more-specifics.rs                | 181 ++++++++++++------
 10 files changed, 484 insertions(+), 227 deletions(-)

diff --git a/src/af.rs b/src/af.rs
index 5995e2fb..f2e6e494 100644
--- a/src/af.rs
+++ b/src/af.rs
@@ -1,3 +1,5 @@
+use crate::local_array::bit_span::BitSpan;
+
 //------------ AddressFamily (trait) ----------------------------------------
 /// The address family of an IP address as a Trait.
 ///
@@ -34,7 +36,7 @@ pub trait AddressFamily:
     fn get_nibble(net: Self, start_bit: u8, len: u8) -> u32;
 
     /// Treat self as a prefix and append the given nibble to it.
-    fn add_nibble(self, len: u8, nibble: u32, nibble_len: u8) -> (Self, u8);
+    fn add_bit_span(self, len: u8, bs: BitSpan) -> (Self, u8);
 
     fn truncate_to_len(self, len: u8) -> Self;
 
@@ -118,9 +120,9 @@ impl AddressFamily for IPv4 {
     /// let nibble = 0b1100110_u32;                             // 7-bit nibble
     /// let (new_prefix, new_len) = prefix.add_nibble(30, nibble, 7);
     /// ```
-    fn add_nibble(self, len: u8, nibble: u32, nibble_len: u8) -> (u32, u8) {
-        let res = self | (nibble << (32 - len - nibble_len) as usize);
-        (res, len + nibble_len)
+    fn add_bit_span(self, len: u8, bs: BitSpan) -> (u32, u8) {
+        let res = self | (bs.bits << (32 - len - bs.len) as usize);
+        (res, len + bs.len)
     }
 
     fn from_ipaddr(addr: std::net::IpAddr) -> u32 {
@@ -210,10 +212,9 @@ impl AddressFamily for IPv6 {
     /// let nibble = 0xF00FF00F_u32;                            // 32-bit nibble
     /// let (new_prefix, new_len) = prefix.add_nibble(112, nibble, 32);
     /// ```
-    fn add_nibble(self, len: u8, nibble: u32, nibble_len: u8) -> (Self, u8) {
-        let res =
-            self | ((nibble as u128) << (128 - len - nibble_len) as usize);
-        (res, len + nibble_len)
+    fn add_bit_span(self, len: u8, bs: BitSpan) -> (Self, u8) {
+        let res = self | ((bs.bits as u128) << (128 - len - bs.len) as usize);
+        (res, len + bs.len)
     }
 
     fn truncate_to_len(self, len: u8) -> Self {
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 7fa07785..4541c245 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -87,7 +87,7 @@ impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
                 // END OF THE LENGTH
 
                 // This length is done too, go to the next length
-                trace!("next length {}", self.cur_len + 1);
+                // trace!("next length {}", self.cur_len + 1);
                 self.cur_len += 1;
 
                 // a new length, a new life reset the level depth and cursor,
@@ -117,7 +117,7 @@ impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
                     // END OF THE LENGTH
 
                     // This length is done too, go to the next length
-                    trace!("next length {}", self.cur_len);
+                    // trace!("next length {}", self.cur_len);
                     self.cur_len += 1;
 
                     // a new length, a new life reset the level depth and
@@ -459,7 +459,7 @@ pub(crate) struct MoreSpecificPrefixIter<
     store: &'a TreeBitMap<AF, M, NB, PB>,
     cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
     cur_pfx_iter: SizedPrefixIter<AF>,
-    start_bit_span: BitSpan,
+    // start_bit_span: BitSpan,
     // skip_self: bool,
     parent_and_position: Vec<SizedNodeMoreSpecificIter<AF>>,
     // If specified, we're only iterating over records for this mui.
@@ -615,16 +615,16 @@ impl<
                         );
                         self.cur_ptr_iter = ptr_iter.wrap();
 
-                        trace!(
-                            "next stride new iterator stride 3 {:?} start \
-                        bit_span {:?}",
-                            self.cur_ptr_iter,
-                            self.start_bit_span
-                        );
+                        // trace!(
+                        //     "next stride new iterator stride 3 {:?} start \
+                        // bit_span {:?}",
+                        //     self.cur_ptr_iter,
+                        //     self.start_bit_span
+                        // );
                         self.cur_pfx_iter = next_node
                             .more_specific_pfx_iter(
                                 next_ptr,
-                                BitSpan::new(0, 1),
+                                BitSpan::new(0, 0),
                                 false,
                             )
                             .wrap();
@@ -640,14 +640,13 @@ impl<
 
                         trace!(
                             "next stride new iterator stride 4 {:?} start \
-                        bit_span {:?}",
+                        bit_span 0 0",
                             self.cur_ptr_iter,
-                            self.start_bit_span
                         );
                         self.cur_pfx_iter = next_node
                             .more_specific_pfx_iter(
                                 next_ptr,
-                                BitSpan::new(0, 1),
+                                BitSpan::new(0, 0),
                                 false,
                             )
                             .wrap();
@@ -661,16 +660,16 @@ impl<
                         );
                         self.cur_ptr_iter = ptr_iter.wrap();
 
-                        trace!(
-                            "next stride new iterator stride 5 {:?} start \
-                        bit_span {:?}",
-                            self.cur_ptr_iter,
-                            self.start_bit_span
-                        );
+                        // trace!(
+                        //     "next stride new iterator stride 5 {:?} start \
+                        // bit_span {:?}",
+                        //     self.cur_ptr_iter,
+                        //     self.start_bit_span
+                        // );
                         self.cur_pfx_iter = next_node
                             .more_specific_pfx_iter(
                                 next_ptr,
-                                BitSpan::new(0, 1),
+                                BitSpan::new(0, 0),
                                 false,
                             )
                             .wrap();
@@ -758,7 +757,7 @@ impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
             if this_level == 0 {
                 // END OF THE LENGTH
                 // This length is done too, go to the next length
-                trace!("next length {}", self.cur_len + 1);
+                // trace!("next length {}", self.cur_len + 1);
                 self.cur_len -= 1;
 
                 // a new length, a new life
@@ -897,10 +896,9 @@ impl<
             None
         } else {
             // calculate the node start_prefix_id lives in.
-            let (start_node_id, start_bit_span) =
-                self.get_node_id_for_prefix(&start_prefix_id);
+            let (start_node_id, start_ptr_span, start_pfx_span) =
+                self.get_node_and_span_for_ms_prefix(&start_prefix_id);
             trace!("start node {}", start_node_id);
-
             trace!(
                 "start prefix id {:032b} (len {})",
                 start_prefix_id.get_net(),
@@ -913,10 +911,18 @@ impl<
                 start_node_id.get_len()
             );
             trace!(
-                "start bit span  {:032b} {}",
-                start_bit_span,
-                start_bit_span.bits
+                "start pfx bit span  {:08b} {} len {}",
+                start_pfx_span.bits,
+                start_pfx_span.bits,
+                start_pfx_span.len
+            );
+            trace!(
+                "start ptr bit span  {:08b} {} len {}",
+                start_ptr_span.bits,
+                start_ptr_span.bits,
+                start_ptr_span.len
             );
+
             let cur_pfx_iter: SizedPrefixIter<AF>;
             let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
 
@@ -932,30 +938,22 @@ impl<
                         cur_pfx_iter = SizedPrefixIter::Stride3(
                             n.more_specific_pfx_iter(
                                 start_node_id,
-                                start_bit_span,
+                                start_pfx_span,
                                 true,
                             ),
                         );
                         cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
                             n.more_specific_ptr_iter(
                                 start_node_id,
-                                start_bit_span,
+                                start_ptr_span,
                             ),
                         );
                     }
                     SizedStrideRef::Stride4(n) => {
-                        trace!(
-                            "ALTERNATIVE {:#?}",
-                            n.add_more_specifics_at(
-                                start_bit_span.bits,
-                                start_bit_span.len,
-                                start_node_id
-                            )
-                        );
                         cur_pfx_iter = SizedPrefixIter::Stride4(
                             n.more_specific_pfx_iter(
                                 start_node_id,
-                                start_bit_span,
+                                start_pfx_span,
                                 true,
                             ),
                         );
@@ -964,7 +962,7 @@ impl<
                         cur_ptr_iter = SizedNodeMoreSpecificIter::Stride4(
                             n.more_specific_ptr_iter(
                                 start_node_id,
-                                start_bit_span,
+                                start_ptr_span,
                             ),
                         );
                     }
@@ -972,14 +970,14 @@ impl<
                         cur_pfx_iter = SizedPrefixIter::Stride5(
                             n.more_specific_pfx_iter(
                                 start_node_id,
-                                start_bit_span,
+                                start_pfx_span,
                                 true,
                             ),
                         );
                         cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
                             n.more_specific_ptr_iter(
                                 start_node_id,
-                                start_bit_span,
+                                start_ptr_span,
                             ),
                         );
                     }
@@ -991,7 +989,7 @@ impl<
                     store: self,
                     cur_pfx_iter,
                     cur_ptr_iter,
-                    start_bit_span,
+                    // start_bit_span,
                     parent_and_position: vec![],
                     global_withdrawn_bmin,
                     include_withdrawn,
@@ -1032,10 +1030,11 @@ impl<
                 start_node_id.get_len()
             );
             trace!(
-                "start bit span  {:032b} {}",
+                "start bit span  {:08b} {}",
                 start_bit_span,
                 start_bit_span.bits
             );
+
             let cur_pfx_iter: SizedPrefixIter<AF>;
             let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
 
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index ae302761..a33736ee 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -187,7 +187,7 @@ macro_rules! impl_primitive_atomic_stride {
                 const STRIDE_LEN: u8 = $len;
 
                 fn get_bit_pos(nibble: u32, len: u8) -> $pfxsize {
-                    trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
+                    // trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
                     1 << (
                             <Self as Stride>::BITS - ((1 << len) - 1) as u8
                             - nibble as u8 - 1
@@ -207,7 +207,7 @@ macro_rules! impl_primitive_atomic_stride {
 
                 fn cursor_from_bit_span(bs: BitSpan) -> u8 {
                     Self::get_bit_pos(bs.bits, bs.len)
-                    .leading_zeros() as u8 - 1
+                    .leading_zeros() as u8
                 }
 
                 fn ptr_range(
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 2b365092..a31f1971 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -1,3 +1,4 @@
+use inetnum::addr::Prefix;
 use num_traits::PrimInt;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
 use std::{fmt::Debug, marker::PhantomData};
@@ -123,12 +124,15 @@ where
         start_bs: BitSpan,
         include_withdrawn: bool,
     ) -> NodeMoreSpecificsPrefixIter<AF, S> {
-        assert!(start_bs.len > 0);
+        // assert!(start_bs.len > 0);
+        let pfxbitarr = self.pfxbitarr.load();
+        trace!("pfxbitarr {:032b}", pfxbitarr);
         NodeMoreSpecificsPrefixIter::<AF, S> {
-            pfxbitarr: self.pfxbitarr.load(),
+            pfxbitarr,
             base_prefix,
+            bs: start_bs,
             start_bs,
-            cursor: S::cursor_from_bit_span(start_bs),
+            // cursor: S::cursor_from_bit_span(start_bs),
             include_withdrawn: false,
         }
     }
@@ -427,7 +431,10 @@ where
         (
             // The identifier of the node that has children of the next
             // stride.
-            Some(base_prefix.add_nibble(nibble, nibble_len)),
+            Some(base_prefix.add_bit_span(BitSpan {
+                bits: nibble,
+                len: nibble_len,
+            })),
             found_pfx,
         )
     }
@@ -658,7 +665,10 @@ where
             > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
             )
         {
-            found_child = Some(base_prefix.add_nibble(nibble, nibble_len));
+            found_child = Some(base_prefix.add_bit_span(BitSpan {
+                bits: nibble,
+                len: nibble_len,
+            }));
         }
 
         if let Some(child) = found_child {
@@ -695,13 +705,17 @@ where
                 if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
                 {
                     found_children_with_more_specifics.push(
-                    base_prefix.add_nibble(ms_nibble, ms_nibble_len)
+                    base_prefix.add_bit_span(BitSpan {
+                        bits: ms_nibble, len: ms_nibble_len})
                     );
                 }
 
                 if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
                     found_more_specifics_vec.push(
-                        base_prefix.add_nibble(ms_nibble, ms_nibble_len).into()                    )
+                        base_prefix.add_bit_span(BitSpan {
+                            bits: ms_nibble,
+                            len: ms_nibble_len
+                        }).into())
                 }
             }
         }
@@ -752,10 +766,13 @@ where
 // child pfxs len      /27-29 /30-32
 // child Nodes len     /29    /32
 
+type PtrBitArr<S> = <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
+type PfxBitArr<S> = <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
+
 #[derive(Debug, Copy, Clone)]
 pub(crate) struct NodeChildIter<AF: AddressFamily, S: Stride> {
     base_prefix: StrideNodeId<AF>,
-    ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
+    ptrbitarr: PtrBitArr<S>,
     bit_span: BitSpan, // start with 0
     _af: PhantomData<AF>,
 }
@@ -774,11 +791,14 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
             // becomes 0000 0000 0000 1100, then it will iterate over
             // ...1100,...1101,...1110,...1111
             let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
-            if (S::into_stride_size(self.ptrbitarr) & bit_pos) >
-                <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+            if (S::into_stride_size(self.ptrbitarr) & bit_pos)
+                > PfxBitArr::<S>::zero()
             {
                 self.bit_span.bits = cursor + 1;
-                return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
+                return Some(self.base_prefix.add_bit_span(BitSpan {
+                    bits: cursor,
+                    len: S::STRIDE_LEN,
+                }));
             }
         }
         None
@@ -831,7 +851,6 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
 // above example is therefore 1 << (5 - 4) = 2. Remember that a ptrbitarr
 // holds only one stride size (the largest for its stride size), so we're
 // done now.
-type PtrBitArr<S> = <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
 
 #[derive(Debug, Copy, Clone)]
 pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily, S: Stride> {
@@ -925,7 +944,10 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
 
         // if self.ptrbitarr & bit_pos > PtrBitArr::<S>::zero() {
         // self.cursor += 1;
-        let pfx = self.base_prefix.add_nibble(bs.bits, S::STRIDE_LEN);
+        let pfx = self.base_prefix.add_bit_span(BitSpan {
+            bits: bs.bits,
+            len: S::STRIDE_LEN,
+        });
         Some(pfx)
         // }
         // self.cursor += 1;
@@ -1136,7 +1158,7 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
             if self.pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize
                 as AtomicBitmap>::InnerType::zero() {
                 let bs = BitSpan::from_bit_pos_index(i);
-                return Some(self.base_prefix.add_nibble(bs.bits, bs.len).into());
+                return Some(self.base_prefix.add_bit_span(bs).into());
             }
         }
 
@@ -1195,8 +1217,6 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
 // the bit_span { bits: 2, len: 3 }, a.k.a. 0010 << 1. But now we will have
 // to go over a different amount of 1 << (5 - 4) = 2 iterations to reap the
 // next bit_spans of 0010 0 and 0010 1.
-type PfxBitArr<S> = <<S as super::atomic_stride::Stride>::AtomicPfxSize as super::atomic_stride::AtomicBitmap>::InnerType;
-
 pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily, S: Stride> {
     // immutables
     base_prefix: StrideNodeId<AF>,
@@ -1205,8 +1225,9 @@ pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily, S: Stride> {
     // technically, (it needs resetting the current state to it after each
     // prefix-length), but we'll keep the start-length as well for clarity
     // and increment it on a different field ('cur_len').
+    bs: BitSpan,
     start_bs: BitSpan,
-    cursor: u8,
+    // cursor: u8,
     include_withdrawn: bool,
 }
 
@@ -1224,42 +1245,106 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
 
         trace!(
             "ms prefix iterator start_bs {:?} start cursor {}",
-            self.start_bs,
-            self.cursor
+            self.bs,
+            S::cursor_from_bit_span(self.bs)
         );
-        trace!("pfxbitarr  {:064b}", self.pfxbitarr);
+        trace!("pfx {:032b}", self.pfxbitarr);
 
-        while self.cursor < S::BITS - 1 {
-            let res = BitSpan::from_bit_pos_index(self.cursor);
-            let bit_pos = S::bit_pos_from_index(self.cursor);
+        while S::cursor_from_bit_span(self.bs) < 31 {
+            if self.bs.len > 4 {
+                trace!(
+                    "all bits have been scanned. This is iterator is done."
+                );
+                return None;
+            }
 
             if log_enabled!(log::Level::Trace) {
+                let cursor = S::cursor_from_bit_span(self.bs);
                 trace!(
-                    "{:02}: {:06b} {:064b} bit_span: {:05b} (len: {})",
-                    self.cursor,
-                    self.cursor - 1,
-                    S::bit_pos_from_index(self.cursor),
-                    res.bits,
-                    res.len
+                    "{:02}: {:032b} bit_span: {:05b} (len: {})",
+                    cursor,
+                    S::get_bit_pos(self.bs.bits, self.bs.len),
+                    self.bs.bits,
+                    self.bs.len
                 );
             }
 
-            if res.bits.count_ones() == res.len as u32 {
-                self.cursor += (self.start_bs.bits
-                    << (res.len - self.start_bs.len))
-                    as u8;
+            let bit_pos = S::get_bit_pos(self.bs.bits, self.bs.len);
+            if self.pfxbitarr & bit_pos > PfxBitArr::<S>::zero() {
+                let prefix_id: PrefixId<AF> = self
+                    .base_prefix
+                    .add_bit_span(BitSpan::from_bit_pos_index(
+                        bit_pos.leading_zeros() as u8,
+                    ))
+                    .into();
+                if log_enabled!(log::Level::Trace) {
+                    trace!(
+                        "found more specific prefix {}",
+                        Prefix::from(prefix_id)
+                    );
+                }
+                if self.bs.bits + 1
+                    < self.start_bs.bits
+                        + (1 << (self.bs.len - self.start_bs.len))
+                {
+                    self.bs = BitSpan {
+                        bits: self.bs.bits + 1,
+                        len: self.bs.len,
+                    };
+                } else {
+                    self.start_bs.bits <<= 1;
+                    self.bs = BitSpan {
+                        bits: self.start_bs.bits,
+                        len: self.bs.len + 1,
+                    };
+                }
+                return Some(prefix_id);
             }
 
-            if self.pfxbitarr & bit_pos > PfxBitArr::<S>::zero() {
-                self.cursor += 1;
-                trace!("found more specific prefix for bit span {:?}", res);
-                return Some(
-                    self.base_prefix.add_nibble(res.bits, res.len).into(),
-                );
+            if self.bs.bits + 1
+                < self.start_bs.bits
+                    + (1 << (self.bs.len - self.start_bs.len))
+            {
+                self.bs = BitSpan {
+                    bits: self.bs.bits + 1,
+                    len: self.bs.len,
+                };
+            } else {
+                self.start_bs.bits <<= 1;
+                self.bs = BitSpan {
+                    bits: self.start_bs.bits,
+                    len: self.bs.len + 1,
+                };
             }
-            self.cursor += 1;
         }
-
+        // while self.cursor < S::BITS - 1 {
+        //     let res = BitSpan::from_bit_pos_index(self.cursor);
+        //     let bit_pos = S::bit_pos_from_index(self.cursor);
+
+        //     if log_enabled!(log::Level::Trace) {
+        //         trace!(
+        //             "{:02}: {:06b} {:064b} bit_span: {:05b} (len: {})",
+        //             self.cursor,
+        //             self.cursor - 1,
+        //             S::bit_pos_from_index(self.cursor),
+        //             res.bits,
+        //             res.len
+        //         );
+        //     }
+
+        //     if res.bits.count_ones() == res.len as u32 {
+        //         self.cursor += (self.start_bs.bits
+        //             << (res.len - self.start_bs.len))
+        //             as u8;
+        //     }
+
+        //     if self.pfxbitarr & bit_pos > PfxBitArr::<S>::zero() {
+        //         self.cursor += 1;
+        //         trace!("found more specific prefix for bit span {:?}", res);
+        //         return Some(self.base_prefix.add_bit_span(res).into());
+        //     }
+        //     self.cursor += 1;
+        // }
         trace!("pfxbitarr iterator done.");
         None
     }
@@ -1493,76 +1578,66 @@ pub(crate) enum NewNodeOrIndex<AF: AddressFamily> {
 //--------------------- Per-Stride-Node-Id Type -----------------------------
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub struct StrideNodeId<AF: AddressFamily>(Option<(AF, u8)>);
+pub struct StrideNodeId<AF: AddressFamily> {
+    addr_bits: AF,
+    len: u8,
+}
 
 impl<AF: AddressFamily> StrideNodeId<AF> {
-    pub fn empty() -> Self {
-        Self(None)
-    }
-
-    pub fn dangerously_new_with_id_as_is(addr_bits: AF, len: u8) -> Self {
-        Self(Some((addr_bits, len)))
+    pub(crate) fn dangerously_new_with_id_as_is(
+        addr_bits: AF,
+        len: u8,
+    ) -> Self {
+        Self { addr_bits, len }
     }
 
     #[inline]
-    pub fn new_with_cleaned_id(addr_bits: AF, len: u8) -> Self {
-        Self(Some((addr_bits.truncate_to_len(len), len)))
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.0.is_none()
+    pub(crate) fn new_with_cleaned_id(addr_bits: AF, len: u8) -> Self {
+        Self {
+            addr_bits: addr_bits.truncate_to_len(len),
+            len,
+        }
     }
 
     pub fn get_id(&self) -> (AF, u8) {
-        self.0.unwrap()
+        (self.addr_bits, self.len)
     }
-    pub fn get_len(&self) -> u8 {
-        self.0.unwrap().1
+
+    pub(crate) fn get_len(&self) -> u8 {
+        self.len
     }
     pub fn set_len(mut self, len: u8) -> Self {
-        self.0.as_mut().unwrap().1 = len;
+        self.len = len;
         self
     }
 
-    pub fn add_to_len(mut self, len: u8) -> Self {
-        self.0.as_mut().unwrap().1 += len;
+    pub(crate) fn add_to_len(mut self, len: u8) -> Self {
+        self.len += len;
         self
     }
 
     #[inline]
-    pub fn truncate_to_len(self) -> Self {
-        let (addr_bits, len) = self.0.unwrap();
-        StrideNodeId::new_with_cleaned_id(addr_bits, len)
+    pub(crate) fn truncate_to_len(self) -> Self {
+        StrideNodeId::new_with_cleaned_id(self.addr_bits, self.len)
     }
 
     // clean out all bits that are set beyond the len. This function should
     // be used before doing any ORing to add a nibble.
     #[inline]
-    pub fn unwrap_with_cleaned_id(&self) -> (AF, u8) {
-        let (addr_bits, len) = self.0.unwrap();
-        (addr_bits.truncate_to_len(len), len)
-    }
-
-    pub fn add_nibble(&self, nibble: u32, nibble_len: u8) -> Self {
-        let (addr_bits, len) = self.unwrap_with_cleaned_id();
-        let res = addr_bits.add_nibble(len, nibble, nibble_len);
-        Self(Some(res))
+    pub(crate) fn with_cleaned_id(&self) -> (AF, u8) {
+        (self.addr_bits.truncate_to_len(self.len), self.len)
     }
 
-    pub fn into_inner(self) -> Option<(AF, u8)> {
-        self.0
+    pub(crate) fn add_bit_span(&self, bs: BitSpan) -> Self {
+        let (addr_bits, len) = self.with_cleaned_id();
+        let res = addr_bits.add_bit_span(len, bs);
+        res.into()
     }
 }
 
 impl<AF: AddressFamily> std::fmt::Display for StrideNodeId<AF> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{}",
-            self.0
-                .map(|x| format!("{}-{}", x.0, x.1))
-                .unwrap_or_else(|| "-".to_string())
-        )
+        write!(f, "{}-{}", self.addr_bits, self.len)
     }
 }
 
@@ -1570,7 +1645,15 @@ impl<AF: AddressFamily> std::convert::From<StrideNodeId<AF>>
     for PrefixId<AF>
 {
     fn from(id: StrideNodeId<AF>) -> Self {
-        let (addr_bits, len) = id.0.unwrap();
-        PrefixId::new(addr_bits, len)
+        PrefixId::new(id.addr_bits, id.len)
+    }
+}
+
+impl<AF: AddressFamily> From<(AF, u8)> for StrideNodeId<AF> {
+    fn from(value: (AF, u8)) -> Self {
+        StrideNodeId {
+            addr_bits: value.0,
+            len: value.1,
+        }
     }
 }
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index 195521bc..ef0798d1 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -117,6 +117,7 @@ where
                 None
             },
             more_specifics: if options.include_more_specifics {
+                trace!("more specifics requested. acquiring...");
                 Some(
                     self.more_specific_prefix_iter_from(
                         if let Some(pfx) = stored_prefix {
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 6401dd58..ecb33b7a 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -189,6 +189,7 @@ use crate::prefix_record::Meta;
 use crate::prelude::multi::{NodeSet, PrefixId};
 use crossbeam_epoch::{Atomic, Guard};
 use crossbeam_utils::Backoff;
+use inetnum::addr::Prefix;
 use log::{debug, error, log_enabled, trace};
 use roaring::RoaringBitmap;
 
@@ -983,7 +984,13 @@ impl<
                 // GOTCHA!
                 // Our search-prefix is stored here, so we're returning
                 // it, so its PrefixRecord can be updated by the caller.
-                trace!("found requested prefix {:?}", search_prefix_id);
+                if log_enabled!(log::Level::Trace) {
+                    trace!(
+                        "found requested prefix {} ({:?})",
+                        Prefix::from(search_prefix_id),
+                        search_prefix_id
+                    );
+                }
                 return stored_prefix;
             } else {
                 // A Collision. Follow the chain.
@@ -1027,7 +1034,13 @@ impl<
 
             if let Some(stored_prefix) = prefix_set.0.get(index) {
                 if id == stored_prefix.get_prefix_id() {
-                    trace!("found requested prefix {:?}", id);
+                    if log_enabled!(log::Level::Trace) {
+                        trace!(
+                            "found requested prefix {} ({:?})",
+                            Prefix::from(id),
+                            id
+                        );
+                    }
                     parents[level as usize] = Some((prefix_set, index));
                     return (
                         Some(stored_prefix),
@@ -1072,6 +1085,11 @@ impl<
         &self,
         prefix: &PrefixId<AF>,
     ) -> (StrideNodeId<AF>, BitSpan) {
+        trace!(
+            "prefix id bits: {:032b} len: {}",
+            prefix.get_net(),
+            prefix.get_len()
+        );
         let mut acc = 0;
         for i in self.get_stride_sizes() {
             acc += *i;
@@ -1099,6 +1117,50 @@ impl<
         }
         panic!("prefix length for {:?} is too long", prefix);
     }
+
+    pub(crate) fn get_node_and_span_for_ms_prefix(
+        &self,
+        prefix: &PrefixId<AF>,
+    ) -> (StrideNodeId<AF>, BitSpan, BitSpan) {
+        trace!(
+            "prefix id bits: {:032b} len: {}",
+            prefix.get_net(),
+            prefix.get_len()
+        );
+        let (cur_node, cur_bs) = self.get_node_id_for_prefix(prefix);
+        // let ms_bits = cur_bs.bits << 1;
+        // let ms_len = cur_bs.len + 1;
+
+        // if ms_len > 4 {
+        //     panic!("ms_len > 4 {}", ms_len);
+        // }
+        // if ms_len > 4 {
+        //     return (
+        //         cur_node.add_bit_span(cur_bs),
+        //         BitSpan { bits: 0, len: 0 },
+        //         BitSpan { bits: 0, len: 0 },
+        //     );
+        // };
+
+        (
+            cur_node, cur_bs,
+            cur_bs, // BitSpan {
+                   //     bits: ms_bits,
+                   //     len: ms_len,
+                   // },
+        )
+        // (
+        //     cur_node,
+        //     BitSpan {
+        //         bits: cur_bs.bits,
+        //         len: cur_bs.len,
+        //     },
+        //     BitSpan {
+        //         bits: cur_bs.bits,
+        //         len: cur_bs.len,
+        //     },
+        // )
+    }
     // ------- THE HASHING FUNCTION -----------------------------------------
 
     // Ok, so hashing is really hard, but we're keeping it simple, and
@@ -1178,17 +1240,17 @@ impl<
             0
         };
         let this_level = <PB>::get_bits_for_len(id.get_len(), level);
-        trace!(
-            "bits division {}; no of bits {}",
-            this_level,
-            this_level - last_level
-        );
-        trace!(
-            "calculated index ({} << {}) >> {}",
-            id.get_net(),
-            last_level,
-            ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-        );
+        // trace!(
+        //     "bits division {}; no of bits {}",
+        //     this_level,
+        //     this_level - last_level
+        // );
+        // trace!(
+        //     "calculated index ({} << {}) >> {}",
+        //     id.get_net(),
+        //     last_level,
+        //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
+        // );
         // HASHING FUNCTION
         ((id.get_net() << last_level)
             >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 6005aaf8..5afa3d14 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -82,21 +82,7 @@ where
             );
 
         QueryResult {
-            // prefix: if let Some(pfx) = prefix_id {
-            //     Prefix::new(
-            //         pfx.prefix.get_net().into_ipaddr(),
-            //         pfx.prefix.get_len(),
-            //     )
-            //     .ok()
-            // } else {
-            //     None
-            // },
             prefix,
-            // prefix_meta: prefix_id.record_map.get_filtered_records(
-            //     mui,
-            //     include_withdrawn,
-            //     self.in_memory_tree.withdrawn_muis_bmin(guard),
-            // ),
             prefix_meta: prefix
                 .map(|_pfx| {
                     self.get_value(prefix_id, mui, include_withdrawn, guard)
diff --git a/src/local_array/tests.rs b/src/local_array/tests.rs
index 06d5c441..e930430e 100644
--- a/src/local_array/tests.rs
+++ b/src/local_array/tests.rs
@@ -1,6 +1,8 @@
 #[cfg(test)]
 use std::error::Error;
 
+use crate::local_array::bit_span::BitSpan;
+
 //------------ AddressFamily bit flippers -----------------------------------
 #[test]
 fn test_af_1() -> Result<(), Box<dyn Error>> {
@@ -22,7 +24,10 @@ fn test_af_1() -> Result<(), Box<dyn Error>> {
             base_prefix.get_id().0.truncate_to_len(28),
             28
         )
-        .add_nibble(0b0101, 4)
+        .add_bit_span(BitSpan {
+            bits: 0b0101,
+            len: 4
+        })
         .get_id()
         .0,
         0b1111_1111_1111_1111_1111_1111_1111_0101
@@ -46,11 +51,24 @@ fn test_af_2() -> Result<(), Box<dyn Error>> {
     );
 
     assert_eq!(
-        nu_prefix.add_nibble(0b1010, 4).get_id().0,
+        nu_prefix
+            .add_bit_span(BitSpan {
+                bits: 0b1010,
+                len: 4
+            })
+            .get_id()
+            .0,
         0b1111_1111_1010_0000_0000_0000_0000_0000
     );
     assert_eq!(
-        nu_prefix.truncate_to_len().add_nibble(0b1010, 4).get_id().0,
+        nu_prefix
+            .truncate_to_len()
+            .add_bit_span(BitSpan {
+                bits: 0b1010,
+                len: 4
+            })
+            .get_id()
+            .0,
         0b1111_1111_1010_0000_0000_0000_0000_0000
     );
 
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index 406da8ee..d11c1fc7 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -44,18 +44,37 @@ mod tests {
 
         // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
         let guard = &epoch::pin();
-        for spfx in &[
+        for (r, spfx) in &[
             (
-                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9),
-                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9), // 0
-                vec![0, 1, 2, 3, 4, 6, 7, 9, 10, 11, 12, 13],
+                0,
+                (
+                    &Prefix::new(
+                        std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
+                        9,
+                    ),
+                    &Prefix::new(
+                        std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
+                        9,
+                    ), // 0
+                    vec![0, 1, 2, 3, 4, 6, 7, 9, 10, 11, 12, 13],
+                ),
             ),
             (
-                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8),
-                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8), // 0
-                vec![0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13],
+                1,
+                (
+                    &Prefix::new(
+                        std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
+                        8,
+                    ),
+                    &Prefix::new(
+                        std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
+                        8,
+                    ), // 0
+                    vec![0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13],
+                ),
             ),
         ] {
+            println!("start round {}", r);
             println!("search for: {:?}", spfx.0);
             let found_result = tree_bitmap.match_prefix(
                 &spfx.0.unwrap(),
@@ -71,7 +90,12 @@ mod tests {
             );
             println!("em/m-s: {:#?}", found_result);
 
-            let more_specifics = found_result.more_specifics.unwrap();
+            let more_specifics = found_result
+                .more_specifics
+                .unwrap()
+                .iter()
+                .filter(|p| p.prefix != spfx.0.unwrap())
+                .collect::<Vec<_>>();
 
             assert_eq!(found_result.prefix.unwrap(), spfx.1.unwrap());
             assert_eq!(&more_specifics.len(), &spfx.2.len());
@@ -83,6 +107,7 @@ mod tests {
                     more_specifics.iter().find(|pfx| pfx.prefix == pfxs[*i]);
                 assert!(result_pfx.is_some());
             }
+            println!("end round {}", r);
             println!("-----------");
         }
         Ok(())
@@ -154,7 +179,12 @@ mod tests {
             );
             println!("em/m-s: {}", found_result);
 
-            let more_specifics = found_result.more_specifics.unwrap();
+            let more_specifics = found_result
+                .more_specifics
+                .unwrap()
+                .iter()
+                .filter(|p| p.prefix != spfx.0.unwrap())
+                .collect::<Vec<_>>();
 
             assert_eq!(
                 found_result.prefix.unwrap(),
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 234b3b89..0c911e9e 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -71,68 +71,99 @@ fn test_more_specifics() -> Result<(), Box<dyn Error>> {
 
     // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
     let guard = &epoch::pin();
-    for spfx in &[
+    for (i, spfx) in &[
         (
-            &Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 23),
-            None,
-            // These are the indexes to pfxs.2 vec.
-            // These are all supposed to show up in the result.
-            vec![
-                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
-                18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
-            ],
+            0,
+            (
+                &Prefix::new(
+                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
+                    23,
+                ),
+                None,
+                // These are the indexes to pfxs.2 vec.
+                // These are all supposed to show up in the result.
+                vec![
+                    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+                    17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
+                ],
+            ),
         ),
         (
-            &Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 24),
-            Some(Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
-                24,
-            )?),
-            // These are the indexes to pfxs.2 vec.
-            // These are all supposed to show up in the result.
-            vec![
-                1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
-                18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
-            ],
+            1,
+            (
+                &Prefix::new(
+                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
+                    24,
+                ),
+                Some(Prefix::new(
+                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
+                    24,
+                )?),
+                // These are the indexes to pfxs.2 vec.
+                // These are all supposed to show up in the result.
+                vec![
+                    1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+                    17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
+                ],
+            ),
         ),
         (
-            &Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 25),
-            Some(Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
-                25,
-            )?),
-            vec![3, 4, 7, 8, 9, 14, 15, 16, 17, 18, 19],
+            2,
+            (
+                &Prefix::new(
+                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
+                    25,
+                ),
+                Some(Prefix::new(
+                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
+                    25,
+                )?),
+                vec![3, 4, 7, 8, 9, 14, 15, 16, 17, 18, 19],
+            ),
         ),
         (
-            &Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 26),
-            Some(Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
-                26,
-            )?),
-            vec![7, 8, 14, 15, 16, 17],
+            3,
+            (
+                &Prefix::new(
+                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
+                    26,
+                ),
+                Some(Prefix::new(
+                    std::net::Ipv4Addr::new(130, 55, 240, 0).into(),
+                    26,
+                )?),
+                vec![7, 8, 14, 15, 16, 17],
+            ),
         ),
         (
-            &Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 192).into(),
-                26,
+            4,
+            (
+                &Prefix::new(
+                    std::net::Ipv4Addr::new(130, 55, 240, 192).into(),
+                    26,
+                ),
+                Some(Prefix::new(
+                    std::net::Ipv4Addr::new(130, 55, 240, 192).into(),
+                    26,
+                )?),
+                vec![12, 13, 24, 25, 26, 27],
             ),
-            Some(Prefix::new(
-                std::net::Ipv4Addr::new(130, 55, 240, 192).into(),
-                26,
-            )?),
-            vec![12, 13, 24, 25, 26, 27],
         ),
         (
-            &Prefix::new(std::net::Ipv4Addr::new(0, 0, 0, 0).into(), 0),
-            None,
-            // These are the indexes to pfxs.2 vec.
-            // These are all supposed to show up in the result.
-            vec![
-                0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
-                18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
-            ],
+            5,
+            (
+                &Prefix::new(std::net::Ipv4Addr::new(0, 0, 0, 0).into(), 0),
+                None,
+                // These are the indexes to pfxs.2 vec.
+                // These are all supposed to show up in the result.
+                vec![
+                    0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+                    17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
+                ],
+            ),
         ),
     ] {
+        println!("round {}", i);
         println!("search for: {}", (*spfx.0)?);
         let found_result = tree_bitmap.match_prefix(
             &spfx.0.unwrap(),
@@ -146,16 +177,62 @@ fn test_more_specifics() -> Result<(), Box<dyn Error>> {
             },
             guard,
         );
-        println!("em/m-s: {:#?}", found_result);
+        // println!("em/m-s: {:#?}", found_result);
+        //
+        println!("search prefix: {}", spfx.0.unwrap());
+        if let Some(pfx) = found_result.clone().prefix {
+            println!("found prefix: {}", pfx);
+        } else {
+            println!("no found prefix");
+        }
+
+        for (i, p) in found_result
+            .clone()
+            .more_specifics
+            .unwrap()
+            .v4
+            .iter()
+            .enumerate()
+            .map(|(i, p)| (i, p.prefix))
+        {
+            println!("ms {}: {}", i, p);
+        }
+
+        println!("--");
+        println!("all prefixes");
+
+        for (i, p) in tree_bitmap
+            .prefixes_iter_v4()
+            .enumerate()
+            .map(|(i, p)| (i, p.prefix))
+        {
+            println!("ms {}: {}", i, p);
+        }
 
         assert!(tree_bitmap.contains(&pfxs[25].unwrap(), None));
         assert!(tree_bitmap.contains(&pfxs[26].unwrap(), None));
         assert!(tree_bitmap.contains(&pfxs[27].unwrap(), None));
-        let ms2 = tree_bitmap.more_specifics_keys_from(&spfx.0.unwrap());
-        println!("ms2 {:#?}", ms2);
-        let more_specifics = found_result.more_specifics.unwrap();
+        // let mut ms2 = tree_bitmap.more_specifics_keys_from(&spfx.0.unwrap());
+        // println!("ms2 {:#?}", ms2);
+        // println!("ms2 len {}", ms2.len());
+        // ms2.dedup();
+        // println!("ms2 deduped {}", ms2.len());
+        let more_specifics = found_result
+            .more_specifics
+            .unwrap()
+            .iter()
+            .filter(|p| p.prefix != spfx.0.unwrap())
+            .collect::<Vec<_>>();
+
+        println!(
+            "{:?}",
+            more_specifics
+                .iter()
+                .find(|ms| ms.prefix == spfx.0.unwrap())
+        );
         assert_eq!(found_result.prefix, spfx.1);
 
+        println!("round {}", i);
         assert_eq!(&more_specifics.len(), &spfx.2.len());
 
         for i in spfx.2.iter() {

From 59ab8b7508bb8ed49efdd859375969596e579a8a Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Feb 2025 09:20:23 +0100
Subject: [PATCH 059/147] static ms_prefix_map

---
 src/local_array/in_memory/atomic_stride.rs |   5 +
 src/local_array/in_memory/macros.rs        |  10 ++
 src/local_array/in_memory/node.rs          | 198 +++++++++++----------
 3 files changed, 115 insertions(+), 98 deletions(-)

diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 47548091..1f9fc0a8 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -542,6 +542,11 @@ where
         u8,
     );
 
+    fn ms_pfx_mask(
+        pfxbitarr: <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
+        range: BitSpan,
+    ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
+
     // Clear the bitmap to the right of the pointer and count the number of
     // ones. This number represents the index to the corresponding prefix in
     // the pfx_vec.
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index a33736ee..88763ef2 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -232,6 +232,16 @@ macro_rules! impl_primitive_atomic_stride {
                     (ptrbitarr & mask, start as u8)
                 }
 
+                fn ms_pfx_mask(
+                    pfxbitarr: $pfxsize,
+                    bs: BitSpan
+                ) -> $pfxsize {
+                    <$pfxsize>::try_from(
+                        $crate::local_array::in_memory::node::
+                        ms_prefix_mask_arr(bs) & pfxbitarr as u32
+                    ).unwrap()
+                }
+
                 // Ptrbitarr searches are only done in the last half of
                 // the bitarray, in the len = S::STRIDE_LEN part. We need a
                 // complete BitSpan still to figure when to stop.
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index a31f1971..9e97310f 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -1,5 +1,6 @@
 use inetnum::addr::Prefix;
 use num_traits::PrimInt;
+use std::arch::x86_64::_mm_addsub_ps;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
 use std::{fmt::Debug, marker::PhantomData};
 
@@ -125,14 +126,12 @@ where
         include_withdrawn: bool,
     ) -> NodeMoreSpecificsPrefixIter<AF, S> {
         // assert!(start_bs.len > 0);
-        let pfxbitarr = self.pfxbitarr.load();
+        let mut pfxbitarr = self.pfxbitarr.load();
         trace!("pfxbitarr {:032b}", pfxbitarr);
+        pfxbitarr = S::ms_pfx_mask(pfxbitarr, start_bs);
         NodeMoreSpecificsPrefixIter::<AF, S> {
             pfxbitarr,
             base_prefix,
-            bs: start_bs,
-            start_bs,
-            // cursor: S::cursor_from_bit_span(start_bs),
             include_withdrawn: false,
         }
     }
@@ -150,7 +149,6 @@ where
         trace!("now we're really starting!");
         trace!("start_bs {:?}", start_bs);
         trace!("ptrbitarr {:032b}", ptrbitarr);
-        // trace!("start cursor {}", start_cursor);
         NodeMoreSpecificChildIter::<AF, S> {
             bitrange,
             base_prefix,
@@ -1167,6 +1165,27 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
     }
 }
 
+pub const fn ms_prefix_mask_arr(bs: BitSpan) -> u32 {
+    [
+        0b_01111111111111111111111111111110, // bits = 0, len = 0
+        0b_01011001111000011111111000000000, // bits = 0, len = 1
+        0b_00100110000111100000000111111110, // bits = 1, len = 1
+        0b_00010001100000011110000000000000, // bits = 0, len = 2
+        0b_00001000011000000001111000000000, // bits = 1, len = 2
+        0b_00000100000110000000000111100000, // bits = 2, len = 2
+        0b_00000010000001100000000000011110, // bits = 3, len = 2
+        0b_00000001000000011000000000000000, // bits = 0, len = 3
+        0b_00000000100000000110000000000000, // bits = 1, len = 3
+        0b_00000000010000000001100000000000, // bits = 2, len = 3
+        0b_00000000001000000000011000000000, // bits = 3, len = 3
+        0b_00000000000100000000000110000000, // bits = 4, len = 3
+        0b_00000000000010000000000001100000, // bits = 5, len = 3
+        0b_00000000000001000000000000011000, // bits = 6, len = 3
+        0b_00000000000000100000000000000110, // bits = 7, len = 3
+        0b_00000000000000000000000000000000, // padding
+    ][(1 << bs.len) - 1 + bs.bits as usize]
+}
+
 // Creates an Iterator that returns all prefixes that exist in a node that
 // are a more-specific prefix of the `base_prefix` + `start_bit_span`.
 //
@@ -1225,9 +1244,7 @@ pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily, S: Stride> {
     // technically, (it needs resetting the current state to it after each
     // prefix-length), but we'll keep the start-length as well for clarity
     // and increment it on a different field ('cur_len').
-    bs: BitSpan,
-    start_bs: BitSpan,
-    // cursor: u8,
+    // bs: BitSpan,
     include_withdrawn: bool,
 }
 
@@ -1243,110 +1260,95 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
             return None;
         }
 
+        let cursor = self.pfxbitarr.leading_zeros() as u8;
+        let bs = BitSpan::from_bit_pos_index(cursor);
         trace!(
             "ms prefix iterator start_bs {:?} start cursor {}",
-            self.bs,
-            S::cursor_from_bit_span(self.bs)
+            bs,
+            S::cursor_from_bit_span(bs)
         );
         trace!("pfx {:032b}", self.pfxbitarr);
-
-        while S::cursor_from_bit_span(self.bs) < 31 {
-            if self.bs.len > 4 {
-                trace!(
-                    "all bits have been scanned. This is iterator is done."
-                );
-                return None;
-            }
-
-            if log_enabled!(log::Level::Trace) {
-                let cursor = S::cursor_from_bit_span(self.bs);
-                trace!(
-                    "{:02}: {:032b} bit_span: {:05b} (len: {})",
-                    cursor,
-                    S::get_bit_pos(self.bs.bits, self.bs.len),
-                    self.bs.bits,
-                    self.bs.len
-                );
-            }
-
-            let bit_pos = S::get_bit_pos(self.bs.bits, self.bs.len);
-            if self.pfxbitarr & bit_pos > PfxBitArr::<S>::zero() {
-                let prefix_id: PrefixId<AF> = self
-                    .base_prefix
-                    .add_bit_span(BitSpan::from_bit_pos_index(
-                        bit_pos.leading_zeros() as u8,
-                    ))
-                    .into();
-                if log_enabled!(log::Level::Trace) {
-                    trace!(
-                        "found more specific prefix {}",
-                        Prefix::from(prefix_id)
-                    );
-                }
-                if self.bs.bits + 1
-                    < self.start_bs.bits
-                        + (1 << (self.bs.len - self.start_bs.len))
-                {
-                    self.bs = BitSpan {
-                        bits: self.bs.bits + 1,
-                        len: self.bs.len,
-                    };
-                } else {
-                    self.start_bs.bits <<= 1;
-                    self.bs = BitSpan {
-                        bits: self.start_bs.bits,
-                        len: self.bs.len + 1,
-                    };
-                }
-                return Some(prefix_id);
-            }
-
-            if self.bs.bits + 1
-                < self.start_bs.bits
-                    + (1 << (self.bs.len - self.start_bs.len))
-            {
-                self.bs = BitSpan {
-                    bits: self.bs.bits + 1,
-                    len: self.bs.len,
-                };
-            } else {
-                self.start_bs.bits <<= 1;
-                self.bs = BitSpan {
-                    bits: self.start_bs.bits,
-                    len: self.bs.len + 1,
-                };
-            }
-        }
-        // while self.cursor < S::BITS - 1 {
-        //     let res = BitSpan::from_bit_pos_index(self.cursor);
-        //     let bit_pos = S::bit_pos_from_index(self.cursor);
+        let bit_pos = S::get_bit_pos(bs.bits, bs.len);
+        let prefix_id: PrefixId<AF> = self
+            .base_prefix
+            .add_bit_span(BitSpan::from_bit_pos_index(
+                bit_pos.leading_zeros() as u8,
+            ))
+            .into();
+        self.pfxbitarr = self.pfxbitarr ^ S::bit_pos_from_index(cursor);
+        Some(prefix_id)
+
+        // while S::cursor_from_bit_span(self.bs) < 31 {
+        //     if self.bs.len > 4 {
+        //         trace!(
+        //             "all bits have been scanned. This is iterator is done."
+        //         );
+        //         return None;
+        //     }
 
         //     if log_enabled!(log::Level::Trace) {
+        //         let cursor = S::cursor_from_bit_span(self.bs);
         //         trace!(
-        //             "{:02}: {:06b} {:064b} bit_span: {:05b} (len: {})",
-        //             self.cursor,
-        //             self.cursor - 1,
-        //             S::bit_pos_from_index(self.cursor),
-        //             res.bits,
-        //             res.len
+        //             "{:02}: {:032b} bit_span: {:05b} (len: {})",
+        //             cursor,
+        //             S::get_bit_pos(self.bs.bits, self.bs.len),
+        //             self.bs.bits,
+        //             self.bs.len
         //         );
         //     }
 
-        //     if res.bits.count_ones() == res.len as u32 {
-        //         self.cursor += (self.start_bs.bits
-        //             << (res.len - self.start_bs.len))
-        //             as u8;
+        //     let bit_pos = S::get_bit_pos(self.bs.bits, self.bs.len);
+        //     if self.pfxbitarr & bit_pos > PfxBitArr::<S>::zero() {
+        //         let prefix_id: PrefixId<AF> = self
+        //             .base_prefix
+        //             .add_bit_span(BitSpan::from_bit_pos_index(
+        //                 bit_pos.leading_zeros() as u8,
+        //             ))
+        //             .into();
+
+        //         if log_enabled!(log::Level::Trace) {
+        //             trace!(
+        //                 "found more specific prefix {}",
+        //                 Prefix::from(prefix_id)
+        //             );
+        //         }
+
+        //         if self.bs.bits + 1
+        //             < self.start_bs.bits
+        //                 + (1 << (self.bs.len - self.start_bs.len))
+        //         {
+        //             self.bs = BitSpan {
+        //                 bits: self.bs.bits + 1,
+        //                 len: self.bs.len,
+        //             };
+        //         } else {
+        //             self.start_bs.bits <<= 1;
+        //             self.bs = BitSpan {
+        //                 bits: self.start_bs.bits,
+        //                 len: self.bs.len + 1,
+        //             };
+        //         }
+        //         return Some(prefix_id);
         //     }
 
-        //     if self.pfxbitarr & bit_pos > PfxBitArr::<S>::zero() {
-        //         self.cursor += 1;
-        //         trace!("found more specific prefix for bit span {:?}", res);
-        //         return Some(self.base_prefix.add_bit_span(res).into());
+        //     if self.bs.bits + 1
+        //         < self.start_bs.bits
+        //             + (1 << (self.bs.len - self.start_bs.len))
+        //     {
+        //         self.bs = BitSpan {
+        //             bits: self.bs.bits + 1,
+        //             len: self.bs.len,
+        //         };
+        //     } else {
+        //         self.start_bs.bits <<= 1;
+        //         self.bs = BitSpan {
+        //             bits: self.start_bs.bits,
+        //             len: self.bs.len + 1,
+        //         };
         //     }
-        //     self.cursor += 1;
         // }
-        trace!("pfxbitarr iterator done.");
-        None
+        // trace!("pfxbitarr iterator done.");
+        // None
     }
 }
 

From cbc4e786e34e865f7698ecbdf20518ec48efa218 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Feb 2025 14:46:25 +0100
Subject: [PATCH 060/147] use default store

---
 examples/full_table_multiple_trees_json.rs | 10 +++++-----
 examples/numbers_treebitmap.rs             | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index a0c69074..2d164eb5 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -7,11 +7,11 @@ use std::error::Error;
 use std::fs::File;
 use std::process;
 
-#[create_store((
-    ([4, 4, 4, 4, 4, 4, 4, 4], 5, 17),
-    ([3, 4, 5, 4], 17, 29)
-))]
-struct MyStore;
+// #[create_store((
+//     ([4, 4, 4, 4, 4, 4, 4, 4], 5, 17),
+//     ([3, 4, 5, 4], 17, 29)
+// ))]
+// struct MyStore;
 
 fn main() -> Result<(), Box<dyn Error>> {
     const CSV_FILE_PATH: &str = "./data/uniq_pfx_asn_dfz_rnd.csv";
diff --git a/examples/numbers_treebitmap.rs b/examples/numbers_treebitmap.rs
index d2f5d99a..df712e5e 100644
--- a/examples/numbers_treebitmap.rs
+++ b/examples/numbers_treebitmap.rs
@@ -9,11 +9,11 @@ use std::fs::File;
 use std::net::{IpAddr, Ipv4Addr};
 use std::process;
 
-#[create_store((
-    ([4, 4, 4, 4, 4, 4, 4, 4], 5, 17),
-    ([3, 4, 5, 4], 16, 29)
-))]
-struct MyStore<PrefixAs>;
+// #[create_store((
+//     ([4, 4, 4, 4, 4, 4, 4, 4], 5, 17),
+//     ([3, 4, 5, 4], 16, 29)
+// ))]
+// struct MyStore<PrefixAs>;
 
 fn get_first_arg() -> Result<OsString, Box<dyn Error>> {
     match env::args_os().nth(1) {

From 532e459773f0e02761c71d4da6f900ff443de258 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Feb 2025 14:47:15 +0100
Subject: [PATCH 061/147] temporarily adjust tests to filter self prefix in ms

---
 tests/more-more-specifics.rs | 17 +++++++-
 tests/treebitmap.rs          | 80 ++++++++++++++++++++++--------------
 2 files changed, 66 insertions(+), 31 deletions(-)

diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index d11c1fc7..817bb6a2 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -1,4 +1,15 @@
 // type Prefix4<'a> = Prefix<u32, PrefixAs>;
+mod common {
+    use std::io::Write;
+
+    pub fn init() {
+        let _ = env_logger::builder()
+            .format(|buf, record| writeln!(buf, "{}", record.args()))
+            .is_test(true)
+            .try_init();
+    }
+}
+
 mod tests {
     use inetnum::addr::Prefix;
     use rotonda_store::{
@@ -10,6 +21,8 @@ mod tests {
     #[test]
     fn test_more_specifics_without_less_specifics(
     ) -> Result<(), Box<dyn Error>> {
+        crate::common::init();
+
         let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18)?, // 0
@@ -75,7 +88,7 @@ mod tests {
             ),
         ] {
             println!("start round {}", r);
-            println!("search for: {:?}", spfx.0);
+            println!("search for: {}", spfx.0.unwrap());
             let found_result = tree_bitmap.match_prefix(
                 &spfx.0.unwrap(),
                 &MatchOptions {
@@ -116,6 +129,8 @@ mod tests {
     #[test]
     fn test_more_specifics_with_less_specifics() -> Result<(), Box<dyn Error>>
     {
+        crate::common::init;
+
         let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18), // 0
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 76717968..232eccd3 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -98,18 +98,18 @@ mod tests {
         Ok(())
     }
 
-    rotonda_store::all_strategies![
-        tree_ipv4;
-        test_tree_ipv4;
-        PrefixAs
-    ];
+    // rotonda_store::all_strategies![
+    //     tree_ipv4;
+    //     test_tree_ipv4;
+    //     PrefixAs
+    // ];
 
-    fn test_tree_ipv4(
-        tree_bitmap: MultiThreadedStore<PrefixAs>,
+    #[test]
+    fn test_tree_ipv4(// tree_bitmap: MultiThreadedStore<PrefixAs>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        // let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             // Prefix::new_relaxed(0b0000_0000_0000_0000_0000_0000_0000_000 0_u32.into_ipaddr(), 0),
             Prefix::new_relaxed(
@@ -385,7 +385,7 @@ mod tests {
             guard,
         );
         println!("prefix {:?}", res.prefix);
-        println!("res: {:#?}", &res);
+        println!("res: {}", &res);
 
         assert_eq!(
             res.prefix.unwrap(),
@@ -487,19 +487,18 @@ mod tests {
         Ok(())
     }
 
-    rotonda_store::all_strategies![
-        multi_ranges;
-        test_multi_ranges_ipv4;
-        NoMeta
-    ];
+    // rotonda_store::all_strategies![
+    //     multi_ranges;
+    //     test_multi_ranges_ipv4;
+    //     NoMeta
+    // ];
 
-    // #[test]
-    fn test_multi_ranges_ipv4(
-        tree_bitmap: MultiThreadedStore<NoMeta>,
+    #[test]
+    fn test_multi_ranges_ipv4(// tree_bitmap: MultiThreadedStore<NoMeta>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        // let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
+        let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
         for mui in [1_u32, 2, 3, 4, 5] {
             println!("Multi Uniq ID {mui}");
 
@@ -706,9 +705,14 @@ mod tests {
 
         println!("more_specifics match {} w/ withdrawn", more_specifics);
         let more_specifics = more_specifics.more_specifics.unwrap();
-        assert_eq!(more_specifics.len(), 1);
-        assert_eq!(more_specifics.v4.len(), 1);
-        let more_specifics = &more_specifics.v4[0];
+        let ms_v4 = more_specifics
+            .v4
+            .iter()
+            .filter(|p| p.prefix != Prefix::from_str("1.0.0.0/16").unwrap())
+            .collect::<Vec<_>>();
+        assert_eq!(more_specifics.len(), 2);
+        assert_eq!(ms_v4.len(), 1);
+        let more_specifics = &ms_v4[0];
         assert_eq!(more_specifics.prefix, Prefix::from_str("1.0.0.0/17")?);
         assert_eq!(more_specifics.meta.len(), 5);
         assert_eq!(
@@ -745,9 +749,14 @@ mod tests {
 
         println!("more_specifics match {} w/o withdrawn", more_specifics);
         let more_specifics = more_specifics.more_specifics.unwrap();
-        assert_eq!(more_specifics.len(), 1);
-        assert_eq!(more_specifics.v4.len(), 1);
-        let more_specifics = &more_specifics.v4[0];
+        let ms_v4 = more_specifics
+            .v4
+            .iter()
+            .filter(|p| p.prefix != Prefix::from_str("1.0.0.0/16").unwrap())
+            .collect::<Vec<_>>();
+        assert_eq!(more_specifics.len(), 2);
+        assert_eq!(ms_v4.len(), 1);
+        let more_specifics = &ms_v4[0];
         assert_eq!(more_specifics.prefix, Prefix::from_str("1.0.0.0/17")?);
         assert_eq!(more_specifics.meta.len(), 4);
         assert_eq!(
@@ -791,9 +800,15 @@ mod tests {
         assert_eq!(more_specifics.prefix_meta.len(), 3);
 
         let more_specifics = more_specifics.more_specifics.unwrap();
-        assert_eq!(more_specifics.len(), 1);
-        assert_eq!(more_specifics.v4.len(), 1);
-        let more_specifics = &more_specifics.v4[0];
+
+        let ms_v4 = more_specifics
+            .v4
+            .iter()
+            .filter(|p| p.prefix != Prefix::from_str("1.0.0.0/16").unwrap())
+            .collect::<Vec<_>>();
+        assert_eq!(more_specifics.len(), 2);
+        assert_eq!(ms_v4.len(), 1);
+        let more_specifics = &ms_v4[0];
         assert_eq!(more_specifics.prefix, Prefix::from_str("1.0.0.0/17")?);
 
         // one more more_specific should have been added due to mui 1 being
@@ -844,9 +859,14 @@ mod tests {
         assert_eq!(more_specifics.match_type, MatchType::EmptyMatch);
 
         let more_specifics = more_specifics.more_specifics.unwrap();
-        assert_eq!(more_specifics.len(), 1);
-        assert_eq!(more_specifics.v4.len(), 1);
-        let more_specifics = &more_specifics.v4[0];
+        let ms_v4 = more_specifics
+            .v4
+            .iter()
+            .filter(|p| p.prefix != Prefix::from_str("1.0.0.0/16").unwrap())
+            .collect::<Vec<_>>();
+        assert_eq!(more_specifics.len(), 2);
+        assert_eq!(ms_v4.len(), 1);
+        let more_specifics = &ms_v4[0];
         assert_eq!(more_specifics.prefix, Prefix::from_str("1.0.0.0/17")?);
 
         // all muis should be visible for the more specifics

From c188c086d5361c3ccc1e513b57a21d8913d47d49 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Feb 2025 14:47:32 +0100
Subject: [PATCH 062/147] use default store

---
 examples/full_table_multiple_trees_json.rs | 3 ++-
 examples/numbers_treebitmap.rs             | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index 2d164eb5..82470e25 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -52,7 +52,8 @@ fn main() -> Result<(), Box<dyn Error>> {
         for n in 1..6 {
             let mut rec_vec: Vec<PrefixRecord<PrefixAs>> = vec![];
             let config = StoreConfig::default();
-            let tree_bitmap = MyStore::<PrefixAs>::new_with_config(config)?;
+            let tree_bitmap =
+                MultiThreadedStore::<PrefixAs>::new_with_config(config)?;
 
             if let Err(err) = load_prefixes(&mut rec_vec) {
                 println!("error running example: {}", err);
diff --git a/examples/numbers_treebitmap.rs b/examples/numbers_treebitmap.rs
index df712e5e..421b1963 100644
--- a/examples/numbers_treebitmap.rs
+++ b/examples/numbers_treebitmap.rs
@@ -63,8 +63,8 @@ fn main() -> Result<(), Box<dyn Error>> {
     for _strides in strides_vec.iter() {
         let mut pfxs: Vec<PrefixRecord<PrefixAs>> = vec![];
         let config = StoreConfig::default();
-        let tree_bitmap: MyStore<PrefixAs> =
-            MyStore::<PrefixAs>::new_with_config(config)?;
+        let tree_bitmap =
+            MultiThreadedStore::<PrefixAs>::new_with_config(config)?;
 
         if let Err(err) = load_prefixes(&mut pfxs) {
             println!("error running example: {}", err);

From 6eb9a47b43e598629f9f2a9a730f40f41ace2590 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Feb 2025 14:48:38 +0100
Subject: [PATCH 063/147] clean out

---
 proc_macros/src/lib.rs                     |  36 +-
 src/local_array/bit_span.rs                |  21 +-
 src/local_array/in_memory/atomic_stride.rs |   2 +-
 src/local_array/in_memory/atomic_types.rs  |  23 +-
 src/local_array/in_memory/iterators.rs     | 366 ++++++---------------
 src/local_array/in_memory/node.rs          |   2 -
 6 files changed, 146 insertions(+), 304 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 38a57851..db4d0859 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -888,24 +888,24 @@ pub fn create_store(
                 }
             }
 
-            pub fn more_specifics_keys_from(&'a self,
-                search_pfx: &Prefix,
-            ) -> Vec<Prefix> {
-
-                match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => self
-                        .v4
-                        .more_specifics_keys_from(PrefixId::from(*search_pfx)
-                        ).map(|p| Prefix::from(p)).collect(),
-                    std::net::IpAddr::V6(addr) => self
-                        .v6
-                        .more_specifics_keys_from(
-                            PrefixId::<IPv6>::from(
-                                *search_pfx
-                            ),
-                        ).map(|p| Prefix::from(p)).collect()
-                }
-            }
+            // pub fn more_specifics_keys_from(&'a self,
+            //     search_pfx: &Prefix,
+            // ) -> Vec<Prefix> {
+
+            //     match search_pfx.addr() {
+            //         std::net::IpAddr::V4(addr) => self
+            //             .v4
+            //             .more_specifics_keys_from(PrefixId::from(*search_pfx)
+            //             ).map(|p| Prefix::from(p)).collect(),
+            //         std::net::IpAddr::V6(addr) => self
+            //             .v6
+            //             .more_specifics_keys_from(
+            //                 PrefixId::<IPv6>::from(
+            //                     *search_pfx
+            //                 ),
+            //             ).map(|p| Prefix::from(p)).collect()
+            //     }
+            // }
 
             /// Return a `QuerySet` that contains all the less-specific
             /// prefixes of the `search_pfx` in the store, including the
diff --git a/src/local_array/bit_span.rs b/src/local_array/bit_span.rs
index 45276c86..fae353a8 100644
--- a/src/local_array/bit_span.rs
+++ b/src/local_array/bit_span.rs
@@ -9,20 +9,6 @@ impl BitSpan {
         Self { bits, len }
     }
 
-    // Increment the bit span by one and calculate the new length.
-    #[allow(dead_code)]
-    pub(crate) fn inc(&mut self) {
-        self.bits += 1;
-        self.len =
-            <u8>::max(self.len, (32 - self.bits.leading_zeros()) as u8);
-    }
-
-    #[allow(dead_code)]
-    pub(crate) fn set_len_to_bits(&mut self) {
-        self.len =
-            <u8>::max(self.len, (32 - self.bits.leading_zeros()) as u8);
-    }
-
     // Deep, dark, black magic. Calculate the bit span from the index in a
     // bitarr. This is used by iterators, so they can have one sequential i
     // loop, that goes over all positions in a bitarr by its indexes.
@@ -38,6 +24,13 @@ impl BitSpan {
             len: i,
         }
     }
+
+    pub(crate) fn check(&self) -> bool {
+        println!("check bit span: {:?}", self);
+        self.len < 5
+            && self.bits < 16
+            && (self.bits << (32 - self.len)) >> (32 - self.len) == self.bits
+    }
 }
 
 impl std::fmt::Binary for BitSpan {
diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 1f9fc0a8..52eeaa66 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -30,7 +30,7 @@ impl<InnerType> CasResult<InnerType> {
     }
 }
 
-pub trait AtomicBitmap
+pub(crate) trait AtomicBitmap
 where
     Self: From<Self::InnerType>,
 {
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 98a34546..188d2151 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -28,7 +28,7 @@ use super::tree::{Stride, Stride3, Stride4, Stride5};
 // ----------- Node related structs -----------------------------------------
 
 #[derive(Debug)]
-pub struct StoredNode<AF, S>
+pub(crate) struct StoredNode<AF, S>
 where
     Self: Sized,
     S: Stride,
@@ -41,17 +41,16 @@ where
     pub(crate) node_set: NodeSet<AF, S>,
 }
 
-#[allow(clippy::type_complexity)]
 #[derive(Debug)]
-pub struct NodeSet<AF: AddressFamily, S: Stride>(
-    pub OnceBoxSlice<StoredNode<AF, S>>,
+pub(crate) struct NodeSet<AF: AddressFamily, S: Stride>(
+    OnceBoxSlice<StoredNode<AF, S>>,
     // A Bitmap index that keeps track of the `multi_uniq_id`s (mui) that are
     // present in value collections in the meta-data tree in the child nodes
-    pub RwLock<RoaringBitmap>,
+    RwLock<RoaringBitmap>,
 );
 
 impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
-    pub fn init(p2_size: u8) -> Self {
+    pub(crate) fn init(p2_size: u8) -> Self {
         if log_enabled!(log::Level::Debug) {
             debug!(
                 "{} store: creating space for {} nodes",
@@ -63,7 +62,11 @@ impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
         NodeSet(OnceBoxSlice::new(p2_size), RoaringBitmap::new().into())
     }
 
-    pub fn update_rbm_index(
+    pub(crate) fn rbm(&self) -> &RwLock<RoaringBitmap> {
+        &self.1
+    }
+
+    pub(crate) fn update_rbm_index(
         &self,
         multi_uniq_id: u32,
     ) -> Result<(u32, bool), crate::prelude::multi::PrefixStoreError>
@@ -78,7 +81,7 @@ impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
         Ok((try_count, !absent))
     }
 
-    pub fn remove_from_rbm_index(
+    pub(crate) fn remove_from_rbm_index(
         &self,
         multi_uniq_id: u32,
         _guard: &crate::epoch::Guard,
@@ -94,6 +97,10 @@ impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
 
         Ok(try_count)
     }
+
+    pub(crate) fn read(&self) -> &OnceBoxSlice<StoredNode<AF, S>> {
+        &self.0
+    }
 }
 
 // ----------- Prefix related structs ---------------------------------------
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 4541c245..658c4ddc 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -278,159 +278,6 @@ impl<AF: AddressFamily> SizedPrefixIter<AF> {
     }
 }
 
-pub(crate) struct MoreSpecificPrefixOnlyIter<
-    'a,
-    AF: AddressFamily,
-    M: Meta,
-    NB: NodeBuckets<AF>,
-    PB: PrefixBuckets<AF, M>,
-    // const PREFIX_SIZE: usize,
-    // const KEY_SIZE: usize,
-> {
-    // rib: &'a Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
-    store: &'a TreeBitMap<AF, M, NB, PB>,
-    cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
-    cur_pfx_iter: SizedPrefixIter<AF>,
-    start_bit_span: BitSpan,
-    // skip_self: bool,
-    parent_and_position: Vec<SizedNodeMoreSpecificIter<AF>>,
-    // If specified, we're only iterating over records for this mui.
-    // mui: Option<u32>,
-    // This is the tree-wide index of withdrawn muis, used to rewrite the
-    // statuses of these records, or filter them out.
-    // global_withdrawn_bmin: &'a RoaringBitmap,
-    // Whether we should filter out the withdrawn records in the search result
-    // include_withdrawn: bool,
-}
-
-impl<
-        AF: AddressFamily,
-        M: Meta,
-        NB: NodeBuckets<AF>,
-        PB: PrefixBuckets<AF, M>,
-        // const PREFIX_SIZE: usize,
-        // const KEY_SIZE: usize,
-    > Iterator for MoreSpecificPrefixOnlyIter<'_, AF, M, NB, PB>
-//, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
-{
-    type Item = PrefixId<AF>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        trace!("MoreSpecificsPrefixOnlyIter");
-
-        loop {
-            // first drain the current prefix iterator until empty.
-            let next_pfx = self.cur_pfx_iter.next();
-
-            if next_pfx.is_some() {
-                return next_pfx;
-            }
-
-            // Our current prefix iterator for this node is done, look for
-            // the next pfx iterator of the next child node in the current
-            // ptr iterator.
-            trace!("cur ptrbitarr {:?}", self.cur_ptr_iter);
-            trace!("start first ptr_iter");
-            let mut next_ptr = self.cur_ptr_iter.next();
-            trace!("new ptrbitarr {:?}", next_ptr);
-
-            // Our current ptr iterator is also done, maybe we have a parent
-            if next_ptr.is_none() {
-                trace!("try for parent");
-                if let Some(cur_ptr_iter) = self.parent_and_position.pop() {
-                    trace!("continue with parent");
-                    self.cur_ptr_iter = cur_ptr_iter;
-                    next_ptr = self.cur_ptr_iter.next();
-                } else {
-                    trace!("no more parents");
-                    return None;
-                }
-            }
-
-            if let Some(next_ptr) = next_ptr {
-                let node = self.store.retrieve_node(next_ptr);
-
-                match node {
-                    Some(SizedStrideRef::Stride3(next_node)) => {
-                        // copy the current iterator into the parent vec and create
-                        // a new ptr iterator for this node
-                        self.parent_and_position.push(self.cur_ptr_iter);
-                        let ptr_iter = next_node.more_specific_ptr_iter(
-                            next_ptr,
-                            BitSpan { bits: 0, len: 1 },
-                        );
-                        self.cur_ptr_iter = ptr_iter.wrap();
-
-                        trace!(
-                            "next stride new iterator stride 3 {:?} start \
-                        bit_span {:?}",
-                            self.cur_ptr_iter,
-                            self.start_bit_span
-                        );
-                        self.cur_pfx_iter = next_node
-                            .more_specific_pfx_iter(
-                                next_ptr,
-                                BitSpan::new(0, 1),
-                                false,
-                            )
-                            .wrap();
-                    }
-                    Some(SizedStrideRef::Stride4(next_node)) => {
-                        // create new ptr iterator for this node.
-                        self.parent_and_position.push(self.cur_ptr_iter);
-                        let ptr_iter = next_node.more_specific_ptr_iter(
-                            next_ptr,
-                            BitSpan { bits: 0, len: 1 },
-                        );
-                        self.cur_ptr_iter = ptr_iter.wrap();
-
-                        trace!(
-                            "next stride new iterator stride 4 {:?} start \
-                        bit_span {:?}",
-                            self.cur_ptr_iter,
-                            self.start_bit_span
-                        );
-                        self.cur_pfx_iter = next_node
-                            .more_specific_pfx_iter(
-                                next_ptr,
-                                BitSpan::new(0, 1),
-                                false,
-                            )
-                            .wrap();
-                    }
-                    Some(SizedStrideRef::Stride5(next_node)) => {
-                        // create new ptr iterator for this node.
-                        self.parent_and_position.push(self.cur_ptr_iter);
-                        let ptr_iter = next_node.more_specific_ptr_iter(
-                            next_ptr,
-                            BitSpan { bits: 0, len: 1 },
-                        );
-                        self.cur_ptr_iter = ptr_iter.wrap();
-
-                        trace!(
-                            "next stride new iterator stride 5 {:?} start \
-                        bit_span {:?}",
-                            self.cur_ptr_iter,
-                            self.start_bit_span
-                        );
-                        self.cur_pfx_iter = next_node
-                            .more_specific_pfx_iter(
-                                next_ptr,
-                                BitSpan::new(0, 1),
-                                false,
-                            )
-                            .wrap();
-                    }
-                    None => {
-                        println!("no node here.");
-                        return None;
-                    }
-                };
-            }
-        }
-    }
-}
-
 // ----------- MoreSpecificPrefixIter ------------------------------------
 
 // A iterator over all the more-specifics for a given prefix.
@@ -1003,114 +850,111 @@ impl<
         .flatten()
     }
 
-    pub fn more_specific_prefix_only_iter_from(
-        &'a self,
-        start_prefix_id: PrefixId<AF>,
-    ) -> impl Iterator<Item = PrefixId<AF>> + 'a {
-        trace!("more specifics (prefix only) for {:?}", start_prefix_id);
-
-        // A v4 /32 or a v6 /128 doesn't have more specific prefixes 🤓.
-        if start_prefix_id.get_len() >= AF::BITS {
-            None
-        } else {
-            // calculate the node start_prefix_id lives in.
-            let (start_node_id, start_bit_span) =
-                self.get_node_id_for_prefix(&start_prefix_id);
-            trace!("start node {}", start_node_id);
-
-            trace!(
-                "start prefix id {:032b} (len {})",
-                start_prefix_id.get_net(),
-                start_prefix_id.get_len()
-            );
-            trace!(
-                "start node id   {:032b} (bits {} len {})",
-                start_node_id.get_id().0,
-                start_node_id.get_id().0,
-                start_node_id.get_len()
-            );
-            trace!(
-                "start bit span  {:08b} {}",
-                start_bit_span,
-                start_bit_span.bits
-            );
-
-            let cur_pfx_iter: SizedPrefixIter<AF>;
-            let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
-
-            let node = self.retrieve_node(start_node_id);
-
-            if let Some(node) = node {
-                match node {
-                    SizedStrideRef::Stride3(n) => {
-                        cur_pfx_iter = SizedPrefixIter::Stride3(
-                            n.more_specific_pfx_iter(
-                                start_node_id,
-                                start_bit_span,
-                                true,
-                            ),
-                        );
-                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
-                            n.more_specific_ptr_iter(
-                                start_node_id,
-                                start_bit_span,
-                            ),
-                        );
-                    }
-                    SizedStrideRef::Stride4(n) => {
-                        trace!(
-                            "ALTERNATIVE {:#?}",
-                            n.add_more_specifics_at(
-                                start_bit_span.bits,
-                                start_bit_span.len,
-                                start_node_id
-                            )
-                        );
-                        cur_pfx_iter = SizedPrefixIter::Stride4(
-                            n.more_specific_pfx_iter(
-                                start_node_id,
-                                start_bit_span,
-                                true,
-                            ),
-                        );
-                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride4(
-                            n.more_specific_ptr_iter(
-                                start_node_id,
-                                start_bit_span,
-                            ),
-                        );
-                    }
-                    SizedStrideRef::Stride5(n) => {
-                        cur_pfx_iter = SizedPrefixIter::Stride5(
-                            n.more_specific_pfx_iter(
-                                start_node_id,
-                                start_bit_span,
-                                true,
-                            ),
-                        );
-                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
-                            n.more_specific_ptr_iter(
-                                start_node_id,
-                                start_bit_span,
-                            ),
-                        );
-                    }
-                };
-
-                Some(MoreSpecificPrefixOnlyIter {
-                    store: self,
-                    cur_pfx_iter,
-                    cur_ptr_iter,
-                    start_bit_span,
-                    parent_and_position: vec![],
-                })
-            } else {
-                None
-            }
-        }
-        .into_iter()
-        .flatten()
-    }
+    // pub fn more_specific_prefix_only_iter_from(
+    //     &'a self,
+    //     start_prefix_id: PrefixId<AF>,
+    // ) -> impl Iterator<Item = PrefixId<AF>> + 'a {
+    //     trace!("more specifics (prefix only) for {:?}", start_prefix_id);
+
+    //     // A v4 /32 or a v6 /128 doesn't have more specific prefixes 🤓.
+    //     if start_prefix_id.get_len() >= AF::BITS {
+    //         None
+    //     } else {
+    //         // calculate the node start_prefix_id lives in.
+    //         let (start_node_id, start_bit_span) =
+    //             self.get_node_id_for_prefix(&start_prefix_id);
+    //         trace!("start node {}", start_node_id);
+
+    //         trace!(
+    //             "start prefix id {:032b} (len {})",
+    //             start_prefix_id.get_net(),
+    //             start_prefix_id.get_len()
+    //         );
+    //         trace!(
+    //             "start node id   {:032b} (bits {} len {})",
+    //             start_node_id.get_id().0,
+    //             start_node_id.get_id().0,
+    //             start_node_id.get_len()
+    //         );
+    //         trace!(
+    //             "start bit span  {:08b} {}",
+    //             start_bit_span,
+    //             start_bit_span.bits
+    //         );
+
+    //         let cur_pfx_iter: SizedPrefixIter<AF>;
+    //         let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
+
+    //         let node = self.retrieve_node(start_node_id);
+
+    //         if let Some(node) = node {
+    //             match node {
+    //                 SizedStrideRef::Stride3(n) => {
+    //                     cur_pfx_iter = SizedPrefixIter::Stride3(
+    //                         n.more_specific_pfx_iter(
+    //                             start_node_id,
+    //                             start_bit_span,
+    //                         ),
+    //                     );
+    //                     cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
+    //                         n.more_specific_ptr_iter(
+    //                             start_node_id,
+    //                             start_bit_span,
+    //                         ),
+    //                     );
+    //                 }
+    //                 SizedStrideRef::Stride4(n) => {
+    //                     trace!(
+    //                         "ALTERNATIVE {:#?}",
+    //                         n.add_more_specifics_at(
+    //                             start_bit_span.bits,
+    //                             start_bit_span.len,
+    //                             start_node_id
+    //                         )
+    //                     );
+    //                     cur_pfx_iter = SizedPrefixIter::Stride4(
+    //                         n.more_specific_pfx_iter(
+    //                             start_node_id,
+    //                             start_bit_span,
+    //                         ),
+    //                     );
+    //                     cur_ptr_iter = SizedNodeMoreSpecificIter::Stride4(
+    //                         n.more_specific_ptr_iter(
+    //                             start_node_id,
+    //                             start_bit_span,
+    //                         ),
+    //                     );
+    //                 }
+    //                 SizedStrideRef::Stride5(n) => {
+    //                     cur_pfx_iter = SizedPrefixIter::Stride5(
+    //                         n.more_specific_pfx_iter(
+    //                             start_node_id,
+    //                             start_bit_span,
+    //                         ),
+    //                     );
+    //                     cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
+    //                         n.more_specific_ptr_iter(
+    //                             start_node_id,
+    //                             start_bit_span,
+    //                         ),
+    //                     );
+    //                 }
+    //             };
+
+    //             Some(MoreSpecificPrefixOnlyIter {
+    //                 store: self,
+    //                 cur_pfx_iter,
+    //                 cur_ptr_iter,
+    //                 start_bit_span,
+    //                 parent_and_position: vec![],
+    //             })
+    //         } else {
+    //             None
+    //         }
+    //     }
+    //     .into_iter()
+    //     .flatten()
+    // }
 
     // Iterator over all less-specific prefixes, starting from the given
     // prefix at the given level and cursor.
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 9e97310f..a2e60fe6 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -1,6 +1,4 @@
-use inetnum::addr::Prefix;
 use num_traits::PrimInt;
-use std::arch::x86_64::_mm_addsub_ps;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
 use std::{fmt::Debug, marker::PhantomData};
 

From 4ca7ee302a56cbd43e28a6538d2028bd0b893409 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Feb 2025 14:49:23 +0100
Subject: [PATCH 064/147] fix ms_prefix bitmap & other small things

---
 src/local_array/in_memory/iterators.rs |  13 +-
 src/local_array/in_memory/node.rs      | 370 ++-----------------------
 src/local_array/in_memory/tree.rs      |   4 +-
 src/local_array/query.rs               |  15 +-
 src/local_array/rib/default_store.rs   |   1 +
 src/local_array/rib/macros.rs          |  14 +-
 src/prelude/mod.rs                     |   2 +-
 7 files changed, 38 insertions(+), 381 deletions(-)

diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 658c4ddc..db32d2cb 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -444,11 +444,8 @@ impl<
                     trace!("let's retriev node {}", next_ptr);
                     self.store.retrieve_node(next_ptr)
                 } else {
-                    self.store.retrieve_node_for_mui(
-                        next_ptr,
-                        self.mui.unwrap(),
-                        // self.guard,
-                    )
+                    self.store
+                        .retrieve_node_for_mui(next_ptr, self.mui.unwrap())
                 };
 
                 match node {
@@ -472,7 +469,6 @@ impl<
                             .more_specific_pfx_iter(
                                 next_ptr,
                                 BitSpan::new(0, 0),
-                                false,
                             )
                             .wrap();
                     }
@@ -494,7 +490,6 @@ impl<
                             .more_specific_pfx_iter(
                                 next_ptr,
                                 BitSpan::new(0, 0),
-                                false,
                             )
                             .wrap();
                     }
@@ -517,7 +512,6 @@ impl<
                             .more_specific_pfx_iter(
                                 next_ptr,
                                 BitSpan::new(0, 0),
-                                false,
                             )
                             .wrap();
                     }
@@ -786,7 +780,6 @@ impl<
                             n.more_specific_pfx_iter(
                                 start_node_id,
                                 start_pfx_span,
-                                true,
                             ),
                         );
                         cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
@@ -801,7 +794,6 @@ impl<
                             n.more_specific_pfx_iter(
                                 start_node_id,
                                 start_pfx_span,
-                                true,
                             ),
                         );
                         trace!("---------------------");
@@ -818,7 +810,6 @@ impl<
                             n.more_specific_pfx_iter(
                                 start_node_id,
                                 start_pfx_span,
-                                true,
                             ),
                         );
                         cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index a2e60fe6..784e2282 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -121,16 +121,14 @@ where
         &self,
         base_prefix: StrideNodeId<AF>,
         start_bs: BitSpan,
-        include_withdrawn: bool,
     ) -> NodeMoreSpecificsPrefixIter<AF, S> {
-        // assert!(start_bs.len > 0);
+        assert!(start_bs.check());
         let mut pfxbitarr = self.pfxbitarr.load();
         trace!("pfxbitarr {:032b}", pfxbitarr);
         pfxbitarr = S::ms_pfx_mask(pfxbitarr, start_bs);
         NodeMoreSpecificsPrefixIter::<AF, S> {
             pfxbitarr,
             base_prefix,
-            include_withdrawn: false,
         }
     }
 
@@ -867,15 +865,6 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
         }
 
         let cursor = self.bitrange.leading_zeros() as u8 + 15;
-        // let cursor = <u8>::max(
-        //     self.start_cursor,
-        //     self.ptrbitarr.leading_zeros() as u8 + 15,
-        // );
-
-        // let stop = <u8>::min(
-        //     (1 << (S::STRIDE_LEN - self.start_bs.len)) + cursor,
-        //     S::BITS - 2,
-        // );
         trace!("LZCNT {}", self.bitrange.leading_zeros());
 
         // if self.bitrange.leading_zeros() == 0 {
@@ -898,19 +887,11 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
 
         trace!("bitrange  {:032b}", self.bitrange);
 
-        // if cursor > stop {
-        //     trace!("ptrbitarr iterator done.");
-        //     return None;
-        // }
-
         self.bitrange = self.bitrange ^ S::ptr_bit_pos_from_index(cursor);
 
         trace!("mask      {:032b}", S::ptr_bit_pos_from_index(cursor));
         trace!("next br   {:032b}", self.bitrange);
 
-        // while self.cursor <= stop {
-        // let bit_pos = S::get_bit_pos(self.cursor as u32, S::STRIDE_LEN);
-
         let bs = BitSpan::from_bit_pos_index(cursor);
         if log_enabled!(log::Level::Trace) {
             let bit_pos = S::ptr_bit_pos_from_index(cursor);
@@ -932,111 +913,14 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
             );
         }
 
-        // if bs.bits.count_ones() == bs.len as u32 {
-        //     self.cursor += (self.start_bs.bits
-        //         << (bs.len - self.start_bs.len))
-        //         as u8;
-        // }
-
-        // if self.ptrbitarr & bit_pos > PtrBitArr::<S>::zero() {
-        // self.cursor += 1;
         let pfx = self.base_prefix.add_bit_span(BitSpan {
             bits: bs.bits,
             len: S::STRIDE_LEN,
         });
         Some(pfx)
-        // }
-        // self.cursor += 1;
-        // }
-
-        // None
     }
 }
 
-// impl<AF: AddressFamily, S: Stride> std::iter::Iterator
-//     for NodeMoreSpecificChildIter<AF, S>
-// {
-//     type Item = StrideNodeId<AF>;
-//     fn next(&mut self) -> Option<Self::Item> {
-//         // Early exits
-
-//         // Empty bitmap
-//         if self.ptrbitarr
-//             == <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType::zero(
-//             )
-//         {
-//             trace!("empty ptrbitrarr. this iterator is done.");
-//             return None;
-//         }
-
-//         // Previous iteration incremented the cursor beyond the stride size.
-//         if let Some(cursor) = self.cursor {
-//             if cursor >= (1 << (S::STRIDE_LEN - self.start_bit_span.len)) {
-//                 trace!(
-//                     "cursor.bits >= (1 << (S::STRIDE_LEN - self.
-//                     start_bit_span.len))"
-//                 );
-//                 trace!("cursor: {}", cursor);
-//                 trace!(
-//                     "start_bit_span: {} {}",
-//                     self.start_bit_span.bits,
-//                     self.start_bit_span.len
-//                 );
-//                 return None;
-//             }
-//         }
-
-//         // No early exits, we're in business.
-//         trace!("NodeMoreSpecificChildIter");
-//         trace!("base_prefix {}", self.base_prefix);
-//         trace!("stride_size {}", S::STRIDE_LEN);
-//         trace!(
-//             "start_bit_span bits {} len {} ",
-//             self.start_bit_span.bits,
-//             self.start_bit_span.len
-//         );
-//         trace!("cursor bits {:?}", self.cursor);
-//         trace!("          x1  4   8  12  16  20  24  28  32");
-//         trace!("ptrbitarr {:032b}", self.ptrbitarr);
-
-//         let start = if let Some(bits) = self.cursor {
-//             bits
-//         } else {
-//             self.start_bit_span.bits
-//         };
-//         // We either stop if we have reached the maximum number of bits that
-//         // we should check for this bit_span or we stop at the end of the
-//         // stride (in case of a start_bit_span.bits == 0).
-//         let stop = <u32>::min(
-//             (1 << (S::STRIDE_LEN - self.start_bit_span.len)) + start,
-//             (1 << S::STRIDE_LEN) - 1,
-//         );
-
-//         trace!("start {:?} stop {}", start, stop);
-//         for cursor in start..=stop {
-//             // move the bit_span left with the amount of bits we're going to loop over.
-//             // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
-//             // becomes 0000 0000 0000 1100, then it will iterate over
-//             // ...1100,...1101,...1110,...1111
-//             let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
-//             trace!("cmpbitarr x{:032b} {}", bit_pos, stop - start);
-//             if (S::into_stride_size(self.ptrbitarr) & bit_pos) >
-//                 <<S as Stride>::AtomicPfxSize as AtomicBitmap
-//                 >::InnerType::zero()
-//             {
-//                 trace!("bingo!");
-//                 self.cursor = Some(cursor + 1);
-
-//                 trace!("next bit_span {} {} with cursor {:?}",
-//                     self.start_bit_span.bits, self.start_bit_span.len, self.cursor);
-//                 return Some(self.base_prefix.add_nibble(cursor, S::STRIDE_LEN));
-//             }
-//         }
-//         trace!("No more nodes. End of the iterator.");
-//         None
-//     }
-// }
-
 impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride3> {
     pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
         SizedNodeMoreSpecificIter::<AF>::Stride3(self)
@@ -1097,10 +981,7 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
     type Item = PrefixId<AF>;
 
     fn next(&mut self) -> Option<Self::Item> {
-        if self.pfxbitarr
-            == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
-            )
-        {
+        if self.pfxbitarr == PfxBitArr::<S>::zero() {
             return None;
         }
         if log_enabled!(log::Level::Trace) {
@@ -1180,6 +1061,22 @@ pub const fn ms_prefix_mask_arr(bs: BitSpan) -> u32 {
         0b_00000000000010000000000001100000, // bits = 5, len = 3
         0b_00000000000001000000000000011000, // bits = 6, len = 3
         0b_00000000000000100000000000000110, // bits = 7, len = 3
+        0b_00000000000000010000000000000000, // bits = 0, len = 4
+        0b_00000000000000001000000000000000, // bits = 1, len = 4
+        0b_00000000000000000100000000000000, // bits = 2, len = 4
+        0b_00000000000000000010000000000000, // bits = 3, len = 4
+        0b_00000000000000000001000000000000, // bits = 4, len = 4
+        0b_00000000000000000000100000000000, // bits = 5, len = 4
+        0b_00000000000000000000010000000000, // bits = 6, len = 4
+        0b_00000000000000000000001000000000, // bits = 7, len = 4
+        0b_00000000000000000000000100000000, // bits = 8, len = 4
+        0b_00000000000000000000000010000000, // bits = 9, len = 4
+        0b_00000000000000000000000001000000, // bits =10, len = 4
+        0b_00000000000000000000000000100000, // bits =11, len = 4
+        0b_00000000000000000000000000010000, // bits =12, len = 4
+        0b_00000000000000000000000000001000, // bits =13, len = 4
+        0b_00000000000000000000000000000100, // bits =14, len = 4
+        0b_00000000000000000000000000000010, // bits =15, len = 4
         0b_00000000000000000000000000000000, // padding
     ][(1 << bs.len) - 1 + bs.bits as usize]
 }
@@ -1235,15 +1132,8 @@ pub const fn ms_prefix_mask_arr(bs: BitSpan) -> u32 {
 // to go over a different amount of 1 << (5 - 4) = 2 iterations to reap the
 // next bit_spans of 0010 0 and 0010 1.
 pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily, S: Stride> {
-    // immutables
     base_prefix: StrideNodeId<AF>,
     pfxbitarr: PfxBitArr<S>,
-    // we need to keep around only the `bits` part of the `bit_span`
-    // technically, (it needs resetting the current state to it after each
-    // prefix-length), but we'll keep the start-length as well for clarity
-    // and increment it on a different field ('cur_len').
-    // bs: BitSpan,
-    include_withdrawn: bool,
 }
 
 impl<AF: AddressFamily, S: Stride> std::iter::Iterator
@@ -1275,233 +1165,9 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
             .into();
         self.pfxbitarr = self.pfxbitarr ^ S::bit_pos_from_index(cursor);
         Some(prefix_id)
-
-        // while S::cursor_from_bit_span(self.bs) < 31 {
-        //     if self.bs.len > 4 {
-        //         trace!(
-        //             "all bits have been scanned. This is iterator is done."
-        //         );
-        //         return None;
-        //     }
-
-        //     if log_enabled!(log::Level::Trace) {
-        //         let cursor = S::cursor_from_bit_span(self.bs);
-        //         trace!(
-        //             "{:02}: {:032b} bit_span: {:05b} (len: {})",
-        //             cursor,
-        //             S::get_bit_pos(self.bs.bits, self.bs.len),
-        //             self.bs.bits,
-        //             self.bs.len
-        //         );
-        //     }
-
-        //     let bit_pos = S::get_bit_pos(self.bs.bits, self.bs.len);
-        //     if self.pfxbitarr & bit_pos > PfxBitArr::<S>::zero() {
-        //         let prefix_id: PrefixId<AF> = self
-        //             .base_prefix
-        //             .add_bit_span(BitSpan::from_bit_pos_index(
-        //                 bit_pos.leading_zeros() as u8,
-        //             ))
-        //             .into();
-
-        //         if log_enabled!(log::Level::Trace) {
-        //             trace!(
-        //                 "found more specific prefix {}",
-        //                 Prefix::from(prefix_id)
-        //             );
-        //         }
-
-        //         if self.bs.bits + 1
-        //             < self.start_bs.bits
-        //                 + (1 << (self.bs.len - self.start_bs.len))
-        //         {
-        //             self.bs = BitSpan {
-        //                 bits: self.bs.bits + 1,
-        //                 len: self.bs.len,
-        //             };
-        //         } else {
-        //             self.start_bs.bits <<= 1;
-        //             self.bs = BitSpan {
-        //                 bits: self.start_bs.bits,
-        //                 len: self.bs.len + 1,
-        //             };
-        //         }
-        //         return Some(prefix_id);
-        //     }
-
-        //     if self.bs.bits + 1
-        //         < self.start_bs.bits
-        //             + (1 << (self.bs.len - self.start_bs.len))
-        //     {
-        //         self.bs = BitSpan {
-        //             bits: self.bs.bits + 1,
-        //             len: self.bs.len,
-        //         };
-        //     } else {
-        //         self.start_bs.bits <<= 1;
-        //         self.bs = BitSpan {
-        //             bits: self.start_bs.bits,
-        //             len: self.bs.len + 1,
-        //         };
-        //     }
-        // }
-        // trace!("pfxbitarr iterator done.");
-        // None
     }
 }
 
-// impl<AF: AddressFamily, S: Stride> std::iter::Iterator
-//     for NodeMoreSpecificsPrefixIter<AF, S>
-// {
-//     type Item = PrefixId<AF>;
-
-//     fn next(&mut self) -> Option<Self::Item> {
-//         // Easy early exit conditions
-
-//         // Empty bitmap
-//         if self.pfxbitarr
-//             == <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
-//             )
-//         {
-//             trace!("empty pfxbitarr. This iterator is done.");
-//             return None;
-//         }
-
-//         // No early exits, We're in business.
-//         trace!("len_offset {}", ((1 << self.cursor.len) - 1));
-//         trace!("start_bit {}", self.start_bit_span.bits);
-//         trace!(
-//             "number of check bits in len {}",
-//             (1 << (self.cursor.len - self.start_bit_span.len))
-//         );
-
-//         trace!(
-//             "next more specifics prefix iter start bits {} len {}",
-//             self.start_bit_span.bits,
-//             self.start_bit_span.len
-//         );
-
-//         let mut res = None;
-
-//         // Move to the next len if we're at the first prefix-length that
-//         // matches, if `skip_self` is set. Typically this flag is set for the
-//         // first stride. In the consecutive strides, we don't want to skip the
-//         // base prefix, since that base_prefix is a more specific prefix of
-//         // the one requested in the first stride.
-//         if self.skip_self
-//             && (1 << (self.cursor.len - self.start_bit_span.len)) == 1
-//         {
-//             // self.cursor.len += (self.start_bit_span.bits & 1) as u8;
-//             trace!("skipping self");
-//             self.start_bit_span.bits <<= 1;
-//             self.cursor.bits = self.start_bit_span.bits;
-//             self.cursor.len += 1;
-//             self.skip_self = false;
-//             trace!(
-//                 "new start bits {} len {}",
-//                 self.start_bit_span.bits,
-//                 self.start_bit_span.len
-//             );
-//             trace!(
-//                 "new cursor bits {} len {}",
-//                 self.cursor.bits,
-//                 self.cursor.len
-//             );
-//         }
-
-//         // Previous iteration or the skip_self routine may have
-//         // incremented the cursor beyond the end of the stride size.
-//         if self.cursor.len > S::STRIDE_LEN {
-//             trace!("cursor.len > S::STRIDE_LEN. This iterator is done.");
-//             return None;
-//         }
-
-//         loop {
-//             trace!(
-//                 "          x1  4   8  12  16  20  24  28  32  36  40  44
-//                   48  52  56  60  64"
-//             );
-//             trace!(
-//                 "cmpnibble {:064b} ({} + {}) len {} stride_size {}",
-//                 S::get_bit_pos(self.cursor.bits, self.cursor.len),
-//                 (1 << self.cursor.len) - 1,
-//                 self.cursor.bits,
-//                 self.cursor.len + self.base_prefix.get_len(),
-//                 S::STRIDE_LEN
-//             );
-
-//             trace!("pfxbitarr {:064b}", self.pfxbitarr);
-
-//             if (S::get_bit_pos(self.cursor.bits, self.cursor.len)
-//                 | self.pfxbitarr)
-//                 == self.pfxbitarr
-//             {
-//                 trace!(
-//                     "found prefix with len {} at pos {} pfx len {}",
-//                     self.cursor.len,
-//                     self.cursor.bits,
-//                     self.base_prefix.get_len() + self.cursor.len,
-//                 );
-//                 res = Some(
-//                     self.base_prefix
-//                         .add_nibble(self.cursor.bits, self.cursor.len)
-//                         .into(),
-//                 );
-//                 trace!("found prefix {:?}", res);
-//             }
-
-//             // Determine when we're at the end of the bits applicable to
-//             // this combo of this start_bit_span.
-//             // bitspan offset:
-//             // self.start_bit_span.bits
-//             // number of matches in this length:
-//             // 1 << (self.cursor.len - self.start_bit_span.len)
-//             let max_pos_offset = self.start_bit_span.bits
-//                 + (1 << (self.cursor.len - self.start_bit_span.len))
-//                 - 1;
-
-//             trace!(
-//                 "max_pos_offset {} > cursor bit_pos {}?",
-//                 max_pos_offset,
-//                 self.cursor.bits
-//             );
-//             trace!(
-//                 "number of check bits in len {}",
-//                 (1 << (self.cursor.len - self.start_bit_span.len))
-//             );
-
-//             // case 1. At the beginning or inside a prefix-length.
-//             if max_pos_offset > self.cursor.bits {
-//                 self.cursor.bits += 1;
-//             }
-//             // case 2. At the end of a prefix-lengths in this stride.
-//             else if self.cursor.len < S::STRIDE_LEN {
-//                 trace!("move len to {}", self.cursor.len + 1);
-//                 self.start_bit_span.bits <<= 1;
-//                 self.cursor.bits = self.start_bit_span.bits;
-//                 self.cursor.len += 1;
-//             }
-//             // case 4. At the end of a prefix-length, but not at the end of the pfxbitarr.
-//             else {
-//                 self.start_bit_span.bits <<= 1;
-//                 self.cursor.bits = self.start_bit_span.bits;
-//                 self.cursor.len += 1;
-//                 trace!(
-//                     "end of stride, next cursor bits {} len {}",
-//                     self.cursor.bits,
-//                     self.cursor.len
-//                 );
-//                 return res;
-//             }
-
-//             trace!("some res {:?}", res);
-//             if res.is_some() {
-//                 return res;
-//             }
-//         }
-//     }
-// }
-
 impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride3> {
     pub fn wrap(self) -> SizedPrefixIter<AF> {
         SizedPrefixIter::<AF>::Stride3(self)
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index ecb33b7a..c42d1379 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -184,9 +184,9 @@
 // produce a fix for it).
 
 use crate::local_array::bit_span::BitSpan;
-use crate::local_array::in_memory::atomic_types::StoredNode;
+use crate::local_array::in_memory::atomic_types::{NodeSet, StoredNode};
 use crate::prefix_record::Meta;
-use crate::prelude::multi::{NodeSet, PrefixId};
+use crate::prelude::multi::PrefixId;
 use crossbeam_epoch::{Atomic, Guard};
 use crossbeam_utils::Backoff;
 use inetnum::addr::Prefix;
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 5afa3d14..42b92f17 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -72,7 +72,6 @@ where
         } else {
             None
         };
-        // let prefix = result.0;
         let more_specifics_vec =
             self.in_memory_tree.more_specific_prefix_iter_from(
                 prefix_id,
@@ -137,13 +136,13 @@ where
         }
     }
 
-    pub fn more_specifics_keys_from(
-        &self,
-        prefix_id: PrefixId<AF>,
-    ) -> impl Iterator<Item = PrefixId<AF>> + '_ {
-        self.in_memory_tree
-            .more_specific_prefix_only_iter_from(prefix_id)
-    }
+    // pub fn more_specifics_keys_from(
+    //     &self,
+    //     prefix_id: PrefixId<AF>,
+    // ) -> impl Iterator<Item = PrefixId<AF>> + '_ {
+    //     self.in_memory_tree
+    //         .more_specific_prefix_iter_from(prefix_id)
+    // }
 
     pub fn more_specifics_iter_from(
         &'a self,
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index c4d9bf46..997a3334 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -1,3 +1,4 @@
+use crate::local_array::in_memory::atomic_types::NodeSet;
 use crate::prelude::multi::*;
 use crate::prelude::*;
 
diff --git a/src/local_array/rib/macros.rs b/src/local_array/rib/macros.rs
index 2e5d8fab..52a2b54b 100644
--- a/src/local_array/rib/macros.rs
+++ b/src/local_array/rib/macros.rs
@@ -23,7 +23,7 @@ macro_rules! impl_search_level {
                     // };
                     // let this_node = stored_node.load(Ordering::Acquire, guard);
 
-                    match nodes.0.get(index) {
+                    match nodes.read().get(index) {
                         None => None,
                         Some(stored_node) => {
                             let StoredNode { node_id, node, node_set, .. } = stored_node;
@@ -80,7 +80,7 @@ macro_rules! impl_search_level_for_mui {
                     // };
                     // let this_node = stored_node.load(Ordering::Acquire, guard);
 
-                    match nodes.0.get(index) {
+                    match nodes.read().get(index) {
                         None => None,
                         Some(this_node) => {
                             let StoredNode { node_id, node, node_set, .. } = this_node;
@@ -89,7 +89,7 @@ macro_rules! impl_search_level_for_mui {
                             // stored in this node, meaning the mui does not
                             // appear anywhere in the sub-tree formed from
                             // this node.
-                            let bmin = node_set.1.read().unwrap(); // load(Ordering::Acquire, guard).deref()
+                            let bmin = node_set.rbm().read().unwrap(); // load(Ordering::Acquire, guard).deref()
                             if !bmin.contains($mui) {
                                 return None;
                             }
@@ -155,7 +155,7 @@ macro_rules! retrieve_node_mut_closure {
                 let index = Self::hash_node_id($id, level);
                 let node;
 
-                match nodes.0.get(index) {
+                match nodes.read().get(index) {
                     // This arm only ever gets called in multi-threaded code
                     // where our thread (running this code *now*), andgot ahead
                     // of another thread: After the other thread created the
@@ -172,7 +172,7 @@ macro_rules! retrieve_node_mut_closure {
                         let node_set = NodeSet::init(next_level - this_level);
 
                         // See if we can create the node
-                        (node, _) = nodes.0.get_or_init(index, || StoredNode {
+                        (node, _) = nodes.read().get_or_init(index, || StoredNode {
                             node_id: $id,
                             node: TreeBitMapNode::new(),
                             node_set
@@ -263,7 +263,7 @@ macro_rules! store_node_closure {
                 // HASHING FUNCTION
                 let index = Self::hash_node_id($id, level);
 
-                match nodes.0.get(index) {
+                match nodes.read().get(index) {
                     None => {
                         // No node exists, so we create one here.
                         let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level + 1);
@@ -287,7 +287,7 @@ macro_rules! store_node_closure {
                         let ptrbitarr = new_node.ptrbitarr.load();
                         let pfxbitarr = new_node.pfxbitarr.load();
 
-                        let (stored_node, its_us) = nodes.0.get_or_init(
+                        let (stored_node, its_us) = nodes.read().get_or_init(
                             index,
                             || StoredNode {
                                 node_id: $id,
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index 68c42f5e..f7563a8c 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -17,7 +17,7 @@ pub mod multi {
 
     pub use crate::local_array::errors::PrefixStoreError;
     pub use crate::local_array::in_memory::atomic_types::{
-        NodeBuckets, NodeSet, PrefixBuckets, PrefixSet,
+        NodeBuckets, PrefixBuckets, PrefixSet,
     };
     pub use crate::local_array::in_memory::iterators;
     pub use crate::local_array::in_memory::node::StrideNodeId;

From 25f13e2ad8a76bbc15405ca9aec0a441e3da7b3b Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Feb 2025 14:52:00 +0100
Subject: [PATCH 065/147] simplify iterator constructor

---
 src/local_array/in_memory/iterators.rs | 46 ++++++++------------------
 src/local_array/in_memory/tree.rs      | 43 ------------------------
 2 files changed, 14 insertions(+), 75 deletions(-)

diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index db32d2cb..367d163b 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -737,8 +737,8 @@ impl<
             None
         } else {
             // calculate the node start_prefix_id lives in.
-            let (start_node_id, start_ptr_span, start_pfx_span) =
-                self.get_node_and_span_for_ms_prefix(&start_prefix_id);
+            let (start_node_id, start_bs) =
+                self.get_node_id_for_prefix(&start_prefix_id);
             trace!("start node {}", start_node_id);
             trace!(
                 "start prefix id {:032b} (len {})",
@@ -753,15 +753,15 @@ impl<
             );
             trace!(
                 "start pfx bit span  {:08b} {} len {}",
-                start_pfx_span.bits,
-                start_pfx_span.bits,
-                start_pfx_span.len
+                start_bs.bits,
+                start_bs.bits,
+                start_bs.len
             );
             trace!(
                 "start ptr bit span  {:08b} {} len {}",
-                start_ptr_span.bits,
-                start_ptr_span.bits,
-                start_ptr_span.len
+                start_bs.bits,
+                start_bs.bits,
+                start_bs.len
             );
 
             let cur_pfx_iter: SizedPrefixIter<AF>;
@@ -777,46 +777,28 @@ impl<
                 match node {
                     SizedStrideRef::Stride3(n) => {
                         cur_pfx_iter = SizedPrefixIter::Stride3(
-                            n.more_specific_pfx_iter(
-                                start_node_id,
-                                start_pfx_span,
-                            ),
+                            n.more_specific_pfx_iter(start_node_id, start_bs),
                         );
                         cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
-                            n.more_specific_ptr_iter(
-                                start_node_id,
-                                start_ptr_span,
-                            ),
+                            n.more_specific_ptr_iter(start_node_id, start_bs),
                         );
                     }
                     SizedStrideRef::Stride4(n) => {
                         cur_pfx_iter = SizedPrefixIter::Stride4(
-                            n.more_specific_pfx_iter(
-                                start_node_id,
-                                start_pfx_span,
-                            ),
+                            n.more_specific_pfx_iter(start_node_id, start_bs),
                         );
                         trace!("---------------------");
                         trace!("start iterating nodes");
                         cur_ptr_iter = SizedNodeMoreSpecificIter::Stride4(
-                            n.more_specific_ptr_iter(
-                                start_node_id,
-                                start_ptr_span,
-                            ),
+                            n.more_specific_ptr_iter(start_node_id, start_bs),
                         );
                     }
                     SizedStrideRef::Stride5(n) => {
                         cur_pfx_iter = SizedPrefixIter::Stride5(
-                            n.more_specific_pfx_iter(
-                                start_node_id,
-                                start_pfx_span,
-                            ),
+                            n.more_specific_pfx_iter(start_node_id, start_bs),
                         );
                         cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
-                            n.more_specific_ptr_iter(
-                                start_node_id,
-                                start_ptr_span,
-                            ),
+                            n.more_specific_ptr_iter(start_node_id, start_bs),
                         );
                     }
                 };
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index c42d1379..4df1ae79 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -1118,49 +1118,6 @@ impl<
         panic!("prefix length for {:?} is too long", prefix);
     }
 
-    pub(crate) fn get_node_and_span_for_ms_prefix(
-        &self,
-        prefix: &PrefixId<AF>,
-    ) -> (StrideNodeId<AF>, BitSpan, BitSpan) {
-        trace!(
-            "prefix id bits: {:032b} len: {}",
-            prefix.get_net(),
-            prefix.get_len()
-        );
-        let (cur_node, cur_bs) = self.get_node_id_for_prefix(prefix);
-        // let ms_bits = cur_bs.bits << 1;
-        // let ms_len = cur_bs.len + 1;
-
-        // if ms_len > 4 {
-        //     panic!("ms_len > 4 {}", ms_len);
-        // }
-        // if ms_len > 4 {
-        //     return (
-        //         cur_node.add_bit_span(cur_bs),
-        //         BitSpan { bits: 0, len: 0 },
-        //         BitSpan { bits: 0, len: 0 },
-        //     );
-        // };
-
-        (
-            cur_node, cur_bs,
-            cur_bs, // BitSpan {
-                   //     bits: ms_bits,
-                   //     len: ms_len,
-                   // },
-        )
-        // (
-        //     cur_node,
-        //     BitSpan {
-        //         bits: cur_bs.bits,
-        //         len: cur_bs.len,
-        //     },
-        //     BitSpan {
-        //         bits: cur_bs.bits,
-        //         len: cur_bs.len,
-        //     },
-        // )
-    }
     // ------- THE HASHING FUNCTION -----------------------------------------
 
     // Ok, so hashing is really hard, but we're keeping it simple, and

From 075d64b1a22486997a18c7fa4fbea3e7f478a92a Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Feb 2025 14:53:40 +0100
Subject: [PATCH 066/147] cleanup

---
 src/local_array/in_memory/iterators.rs | 106 -------------------------
 1 file changed, 106 deletions(-)

diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 367d163b..b1013e68 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -823,112 +823,6 @@ impl<
         .flatten()
     }
 
-    // pub fn more_specific_prefix_only_iter_from(
-    //     &'a self,
-    //     start_prefix_id: PrefixId<AF>,
-    // ) -> impl Iterator<Item = PrefixId<AF>> + 'a {
-    //     trace!("more specifics (prefix only) for {:?}", start_prefix_id);
-
-    //     // A v4 /32 or a v6 /128 doesn't have more specific prefixes 🤓.
-    //     if start_prefix_id.get_len() >= AF::BITS {
-    //         None
-    //     } else {
-    //         // calculate the node start_prefix_id lives in.
-    //         let (start_node_id, start_bit_span) =
-    //             self.get_node_id_for_prefix(&start_prefix_id);
-    //         trace!("start node {}", start_node_id);
-
-    //         trace!(
-    //             "start prefix id {:032b} (len {})",
-    //             start_prefix_id.get_net(),
-    //             start_prefix_id.get_len()
-    //         );
-    //         trace!(
-    //             "start node id   {:032b} (bits {} len {})",
-    //             start_node_id.get_id().0,
-    //             start_node_id.get_id().0,
-    //             start_node_id.get_len()
-    //         );
-    //         trace!(
-    //             "start bit span  {:08b} {}",
-    //             start_bit_span,
-    //             start_bit_span.bits
-    //         );
-
-    //         let cur_pfx_iter: SizedPrefixIter<AF>;
-    //         let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
-
-    //         let node = self.retrieve_node(start_node_id);
-
-    //         if let Some(node) = node {
-    //             match node {
-    //                 SizedStrideRef::Stride3(n) => {
-    //                     cur_pfx_iter = SizedPrefixIter::Stride3(
-    //                         n.more_specific_pfx_iter(
-    //                             start_node_id,
-    //                             start_bit_span,
-    //                         ),
-    //                     );
-    //                     cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
-    //                         n.more_specific_ptr_iter(
-    //                             start_node_id,
-    //                             start_bit_span,
-    //                         ),
-    //                     );
-    //                 }
-    //                 SizedStrideRef::Stride4(n) => {
-    //                     trace!(
-    //                         "ALTERNATIVE {:#?}",
-    //                         n.add_more_specifics_at(
-    //                             start_bit_span.bits,
-    //                             start_bit_span.len,
-    //                             start_node_id
-    //                         )
-    //                     );
-    //                     cur_pfx_iter = SizedPrefixIter::Stride4(
-    //                         n.more_specific_pfx_iter(
-    //                             start_node_id,
-    //                             start_bit_span,
-    //                         ),
-    //                     );
-    //                     cur_ptr_iter = SizedNodeMoreSpecificIter::Stride4(
-    //                         n.more_specific_ptr_iter(
-    //                             start_node_id,
-    //                             start_bit_span,
-    //                         ),
-    //                     );
-    //                 }
-    //                 SizedStrideRef::Stride5(n) => {
-    //                     cur_pfx_iter = SizedPrefixIter::Stride5(
-    //                         n.more_specific_pfx_iter(
-    //                             start_node_id,
-    //                             start_bit_span,
-    //                         ),
-    //                     );
-    //                     cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
-    //                         n.more_specific_ptr_iter(
-    //                             start_node_id,
-    //                             start_bit_span,
-    //                         ),
-    //                     );
-    //                 }
-    //             };
-
-    //             Some(MoreSpecificPrefixOnlyIter {
-    //                 store: self,
-    //                 cur_pfx_iter,
-    //                 cur_ptr_iter,
-    //                 start_bit_span,
-    //                 parent_and_position: vec![],
-    //             })
-    //         } else {
-    //             None
-    //         }
-    //     }
-    //     .into_iter()
-    //     .flatten()
-    // }
-
     // Iterator over all less-specific prefixes, starting from the given
     // prefix at the given level and cursor.
     pub fn less_specific_prefix_iter(

From 289d0c4ba78497acea508479e3f394d9eff187b5 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 7 Feb 2025 14:01:20 +0100
Subject: [PATCH 067/147] split out in_mem and prefix_cht iterators

---
 src/local_array/bit_span.rs                |    2 +-
 src/local_array/in_memory/atomic_types.rs  |   65 +-
 src/local_array/in_memory/iterators.rs     | 1043 ++++++++++----------
 src/local_array/in_memory/query.rs         |  261 +++--
 src/local_array/in_memory/tree.rs          |  316 +-----
 src/local_array/mod.rs                     |    1 +
 src/local_array/prefix_cht/cht.rs          |  329 ++++++
 src/local_array/prefix_cht/iterators.rs    |  244 +++++
 src/local_array/prefix_cht/iterators_cp.rs |  879 +++++++++++++++++
 src/local_array/prefix_cht/mod.rs          |    2 +
 src/local_array/query.rs                   |  279 ++++--
 src/local_array/rib/rib.rs                 |  102 +-
 src/local_array/types.rs                   |   20 +
 13 files changed, 2536 insertions(+), 1007 deletions(-)
 create mode 100644 src/local_array/prefix_cht/cht.rs
 create mode 100644 src/local_array/prefix_cht/iterators.rs
 create mode 100644 src/local_array/prefix_cht/iterators_cp.rs
 create mode 100644 src/local_array/prefix_cht/mod.rs

diff --git a/src/local_array/bit_span.rs b/src/local_array/bit_span.rs
index fae353a8..d8b839d3 100644
--- a/src/local_array/bit_span.rs
+++ b/src/local_array/bit_span.rs
@@ -26,7 +26,7 @@ impl BitSpan {
     }
 
     pub(crate) fn check(&self) -> bool {
-        println!("check bit span: {:?}", self);
+        // println!("check bit span: {:?}", self);
         self.len < 5
             && self.bits < 16
             && (self.bits << (32 - self.len)) >> (32 - self.len) == self.bits
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 188d2151..75702e9f 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -371,15 +371,17 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         let c_map = Arc::clone(&self.0);
         let record_map = c_map.lock().unwrap();
 
-        record_map.get(&mui).and_then(|r| {
-            if include_withdrawn {
-                Some(PublicRecord::from((mui, r)))
-            } else if r.route_status() == RouteStatus::Active {
-                Some(PublicRecord::from((mui, r)))
-            } else {
-                None
-            }
-        })
+        record_map
+            .get(&mui)
+            .and_then(|r| -> Option<PublicRecord<M>> {
+                if include_withdrawn
+                    || r.route_status() == RouteStatus::Active
+                {
+                    Some(PublicRecord::from((mui, r)))
+                } else {
+                    None
+                }
+            })
     }
 
     pub fn best_backup(&self, tbi: M::TBI) -> (Option<u32>, Option<u32>) {
@@ -412,6 +414,22 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         })
     }
 
+    pub fn get_filtered_record_for_mui(
+        &self,
+        mui: u32,
+        include_withdrawn: bool,
+        bmin: &RoaringBitmap,
+    ) -> Option<PublicRecord<M>> {
+        match include_withdrawn {
+            false => self.get_record_for_mui(mui, include_withdrawn),
+            true => self.get_record_for_mui_with_rewritten_status(
+                mui,
+                bmin,
+                RouteStatus::Withdrawn,
+            ),
+        }
+    }
+
     // Helper to filter out records that are not-active (Inactive or
     // Withdrawn), or whose mui appears in the global withdrawn index.
     pub fn get_filtered_records(
@@ -419,13 +437,32 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         mui: Option<u32>,
         include_withdrawn: bool,
         bmin: &RoaringBitmap,
-    ) -> Vec<PublicRecord<M>> {
+    ) -> Option<Vec<PublicRecord<M>>> {
         if let Some(mui) = mui {
-            self.get_record_for_mui(mui, include_withdrawn)
-                .into_iter()
-                .collect()
+            self.get_filtered_record_for_mui(mui, include_withdrawn, bmin)
+                .map(|r| vec![r])
         } else {
-            self.as_active_records_not_in_bmin(bmin)
+            match include_withdrawn {
+                false => {
+                    let recs = self.as_active_records_not_in_bmin(bmin);
+                    if recs.is_empty() {
+                        None
+                    } else {
+                        Some(recs)
+                    }
+                }
+                true => {
+                    let recs = self.as_records_with_rewritten_status(
+                        bmin,
+                        RouteStatus::Withdrawn,
+                    );
+                    if recs.is_empty() {
+                        None
+                    } else {
+                        Some(recs)
+                    }
+                }
+            }
         }
     }
 
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index b1013e68..80a0e7db 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -19,11 +19,11 @@
 // contention, since every lookup has to go through the levels near the root
 // in the TreeBitMap.
 
-use super::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
-use super::node::{SizedStrideRef, StrideNodeId};
-use super::tree::{Stride3, Stride4, Stride5, TreeBitMap};
-use crate::local_array::types::RouteStatus;
-use crate::PublicRecord;
+use crate::local_array::in_memory::atomic_types::NodeBuckets;
+use crate::local_array::in_memory::node::{SizedStrideRef, StrideNodeId};
+use crate::local_array::in_memory::tree::{
+    Stride3, Stride4, Stride5, TreeBitMap,
+};
 use crate::{
     af::AddressFamily,
     local_array::{
@@ -33,211 +33,208 @@ use crate::{
         },
         types::PrefixId,
     },
-    prefix_record::Meta,
 };
 
-use crossbeam_epoch::Guard;
 use inetnum::addr::Prefix;
-use log::trace;
-use roaring::RoaringBitmap;
+use log::{log_enabled, trace};
 
 // ----------- PrefixIter ---------------------------------------------------
 
 // Iterator over all the prefixes in the storage. This Iterator does *not* use
 // the tree, it iterates over all the length arrays in the CustomAllocStorage.
 
-pub(crate) struct PrefixIter<
-    'a,
-    AF: AddressFamily + 'a,
-    M: Meta + 'a,
-    PB: PrefixBuckets<AF, M>,
-> {
-    prefixes: &'a PB,
-    cur_len: u8,
-    cur_bucket: &'a PrefixSet<AF, M>,
-    cur_level: u8,
-    // level depth of IPv4 as defined in rotonda-macros/maps.rs Option(parent,
-    // cursor position at the parent) 32 is the max number of levels in IPv6,
-    // which is the max number of of both IPv4 and IPv6.
-    parents: [Option<(&'a PrefixSet<AF, M>, usize)>; 32],
-    cursor: usize,
-}
-
-impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
-    Iterator for PrefixIter<'a, AF, M, PB>
-{
-    type Item = (inetnum::addr::Prefix, Vec<PublicRecord<M>>);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        trace!(
-            "starting next loop for level {} cursor {} (len {})",
-            self.cur_level,
-            self.cursor,
-            self.cur_len
-        );
-
-        loop {
-            if self.cur_len > AF::BITS {
-                // This is the end, my friend
-                trace!("reached max length {}, returning None", self.cur_len);
-                return None;
-            }
-
-            if PB::get_bits_for_len(self.cur_len, self.cur_level) == 0 {
-                // END OF THE LENGTH
-
-                // This length is done too, go to the next length
-                // trace!("next length {}", self.cur_len + 1);
-                self.cur_len += 1;
-
-                // a new length, a new life reset the level depth and cursor,
-                // but also empty all the parents
-                self.cur_level = 0;
-                self.cursor = 0;
-                self.parents = [None; 32];
-
-                // let's continue, get the prefixes for the next length
-                self.cur_bucket =
-                    self.prefixes.get_root_prefix_set(self.cur_len);
-                continue;
-            }
-            let bucket_size = 1_usize
-                << (if self.cur_level > 0 {
-                    PB::get_bits_for_len(self.cur_len, self.cur_level)
-                        - PB::get_bits_for_len(
-                            self.cur_len,
-                            self.cur_level - 1,
-                        )
-                } else {
-                    PB::get_bits_for_len(self.cur_len, self.cur_level)
-                });
-
-            if self.cursor >= bucket_size {
-                if self.cur_level == 0 {
-                    // END OF THE LENGTH
-
-                    // This length is done too, go to the next length
-                    // trace!("next length {}", self.cur_len);
-                    self.cur_len += 1;
-
-                    // a new length, a new life reset the level depth and
-                    // cursor, but also empty all the parents
-                    self.cur_level = 0;
-                    self.cursor = 0;
-                    self.parents = [None; 32];
-
-                    if self.cur_len > AF::BITS {
-                        // This is the end, my friend
-                        return None;
-                    }
-
-                    // let's continue, get the prefixes for the next length
-                    self.cur_bucket =
-                        self.prefixes.get_root_prefix_set(self.cur_len);
-                } else {
-                    // END OF THIS BUCKET GO BACK UP ONE LEVEL
-
-                    // The level is done, but the length isn't Go back up one
-                    // level and continue
-                    match self.parents[self.cur_level as usize] {
-                        Some(parent) => {
-                            // There is a parent, go back up. Since we're
-                            // doing depth-first we have to check if there's a
-                            // prefix directly at the parent and return that.
-                            self.cur_level -= 1;
-
-                            // move the current bucket to the parent and move
-                            // the cursor position where we left off. The next
-                            // run of the loop will read it.
-                            self.cur_bucket = parent.0;
-                            self.cursor = parent.1 + 1;
-
-                            continue;
-                        }
-                        None => {
-                            trace!(
-                                "c {} lvl {} len {}",
-                                self.cursor,
-                                self.cur_level,
-                                self.cur_len
-                            );
-                            panic!(
-                                "Where do we belong? Where do we come from?"
-                            );
-                        }
-                    };
-                }
-            };
-
-            // we're somewhere in the PrefixSet iteration, read the next
-            // StoredPrefix. We are doing depth-first iteration, so we check
-            // for a child first and descend into that if it exists.
-
-            if let Some(s_pfx) = self.cur_bucket.get_by_index(self.cursor) {
-                // DEPTH FIRST ITERATION
-                match s_pfx.get_next_bucket() {
-                    Some(bucket) => {
-                        // DESCEND ONe LEVEL There's a child here, descend into
-                        // it, but... trace!("C. got next bucket {:?}", bucket);
-
-                        // save our parent and cursor position first, and then..
-                        self.parents[(self.cur_level + 1) as usize] =
-                            Some((self.cur_bucket, self.cursor));
-
-                        // move to the next bucket,
-                        self.cur_bucket = bucket;
-
-                        // increment the level and reset the cursor.
-                        self.cur_level += 1;
-                        self.cursor = 0;
-
-                        // If there's a child here there MUST be a prefix here,
-                        // as well.
-                        // if let Some(meta) =
-                        //     s_pfx.get_stored_prefix(self.guard).map(|p| {
-                        //         if log_enabled!(log::Level::Trace) {
-                        //             // There's a prefix here, that's the next one
-                        //             trace!("D. found prefix {:?}", p.prefix);
-                        //         }
-                        //         p.record_map.as_records()
-                        //     })
-                        // {
-                        return Some((
-                            s_pfx.get_prefix_id().into(),
-                            s_pfx.record_map.as_records(),
-                        ));
-                        // } else {
-                        //     panic!(
-                        //         "No prefix here, but there's a child here?"
-                        //     );
-                        // }
-                    }
-                    None => {
-                        // No reference to another PrefixSet, all that's left, is
-                        // checking for a prefix at the current cursor position.
-                        // if let Some(meta) =
-                        //     s_pfx.get_stored_prefix(self.guard).map(|p| {
-                        //         // There's a prefix here, that's the next one
-                        //         if log_enabled!(log::Level::Debug) {
-                        //             debug!("E. found prefix {:?}", p.prefix);
-                        //         }
-                        //         p.record_map.as_records()
-                        //     })
-                        // {
-                        self.cursor += 1;
-                        return Some((
-                            s_pfx.get_prefix_id().into(),
-                            s_pfx.record_map.as_records(),
-                        ));
-                        // }
-                    }
-                };
-            } else {
-                self.cursor += 1;
-            }
-        }
-    }
-}
+// pub(crate) struct PrefixIter<
+//     'a,
+//     AF: AddressFamily + 'a,
+//     M: Meta + 'a,
+//     PB: PrefixBuckets<AF, M>,
+// > {
+//     prefixes: &'a PB,
+//     cur_len: u8,
+//     cur_bucket: &'a PrefixSet<AF, M>,
+//     cur_level: u8,
+//     // level depth of IPv4 as defined in rotonda-macros/maps.rs Option(parent,
+//     // cursor position at the parent) 32 is the max number of levels in IPv6,
+//     // which is the max number of of both IPv4 and IPv6.
+//     parents: [Option<(&'a PrefixSet<AF, M>, usize)>; 32],
+//     cursor: usize,
+// }
+
+// impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
+//     Iterator for PrefixIter<'a, AF, M, PB>
+// {
+//     type Item = (inetnum::addr::Prefix, Vec<PublicRecord<M>>);
+
+//     fn next(&mut self) -> Option<Self::Item> {
+//         trace!(
+//             "starting next loop for level {} cursor {} (len {})",
+//             self.cur_level,
+//             self.cursor,
+//             self.cur_len
+//         );
+
+//         loop {
+//             if self.cur_len > AF::BITS {
+//                 // This is the end, my friend
+//                 trace!("reached max length {}, returning None", self.cur_len);
+//                 return None;
+//             }
+
+//             if PB::get_bits_for_len(self.cur_len, self.cur_level) == 0 {
+//                 // END OF THE LENGTH
+
+//                 // This length is done too, go to the next length
+//                 // trace!("next length {}", self.cur_len + 1);
+//                 self.cur_len += 1;
+
+//                 // a new length, a new life reset the level depth and cursor,
+//                 // but also empty all the parents
+//                 self.cur_level = 0;
+//                 self.cursor = 0;
+//                 self.parents = [None; 32];
+
+//                 // let's continue, get the prefixes for the next length
+//                 self.cur_bucket =
+//                     self.prefixes.get_root_prefix_set(self.cur_len);
+//                 continue;
+//             }
+//             let bucket_size = 1_usize
+//                 << (if self.cur_level > 0 {
+//                     PB::get_bits_for_len(self.cur_len, self.cur_level)
+//                         - PB::get_bits_for_len(
+//                             self.cur_len,
+//                             self.cur_level - 1,
+//                         )
+//                 } else {
+//                     PB::get_bits_for_len(self.cur_len, self.cur_level)
+//                 });
+
+//             if self.cursor >= bucket_size {
+//                 if self.cur_level == 0 {
+//                     // END OF THE LENGTH
+
+//                     // This length is done too, go to the next length
+//                     // trace!("next length {}", self.cur_len);
+//                     self.cur_len += 1;
+
+//                     // a new length, a new life reset the level depth and
+//                     // cursor, but also empty all the parents
+//                     self.cur_level = 0;
+//                     self.cursor = 0;
+//                     self.parents = [None; 32];
+
+//                     if self.cur_len > AF::BITS {
+//                         // This is the end, my friend
+//                         return None;
+//                     }
+
+//                     // let's continue, get the prefixes for the next length
+//                     self.cur_bucket =
+//                         self.prefixes.get_root_prefix_set(self.cur_len);
+//                 } else {
+//                     // END OF THIS BUCKET GO BACK UP ONE LEVEL
+
+//                     // The level is done, but the length isn't Go back up one
+//                     // level and continue
+//                     match self.parents[self.cur_level as usize] {
+//                         Some(parent) => {
+//                             // There is a parent, go back up. Since we're
+//                             // doing depth-first we have to check if there's a
+//                             // prefix directly at the parent and return that.
+//                             self.cur_level -= 1;
+
+//                             // move the current bucket to the parent and move
+//                             // the cursor position where we left off. The next
+//                             // run of the loop will read it.
+//                             self.cur_bucket = parent.0;
+//                             self.cursor = parent.1 + 1;
+
+//                             continue;
+//                         }
+//                         None => {
+//                             trace!(
+//                                 "c {} lvl {} len {}",
+//                                 self.cursor,
+//                                 self.cur_level,
+//                                 self.cur_len
+//                             );
+//                             panic!(
+//                                 "Where do we belong? Where do we come from?"
+//                             );
+//                         }
+//                     };
+//                 }
+//             };
+
+//             // we're somewhere in the PrefixSet iteration, read the next
+//             // StoredPrefix. We are doing depth-first iteration, so we check
+//             // for a child first and descend into that if it exists.
+
+//             if let Some(s_pfx) = self.cur_bucket.get_by_index(self.cursor) {
+//                 // DEPTH FIRST ITERATION
+//                 match s_pfx.get_next_bucket() {
+//                     Some(bucket) => {
+//                         // DESCEND ONe LEVEL There's a child here, descend into
+//                         // it, but... trace!("C. got next bucket {:?}", bucket);
+
+//                         // save our parent and cursor position first, and then..
+//                         self.parents[(self.cur_level + 1) as usize] =
+//                             Some((self.cur_bucket, self.cursor));
+
+//                         // move to the next bucket,
+//                         self.cur_bucket = bucket;
+
+//                         // increment the level and reset the cursor.
+//                         self.cur_level += 1;
+//                         self.cursor = 0;
+
+//                         // If there's a child here there MUST be a prefix here,
+//                         // as well.
+//                         // if let Some(meta) =
+//                         //     s_pfx.get_stored_prefix(self.guard).map(|p| {
+//                         //         if log_enabled!(log::Level::Trace) {
+//                         //             // There's a prefix here, that's the next one
+//                         //             trace!("D. found prefix {:?}", p.prefix);
+//                         //         }
+//                         //         p.record_map.as_records()
+//                         //     })
+//                         // {
+//                         return Some((
+//                             s_pfx.get_prefix_id().into(),
+//                             s_pfx.record_map.as_records(),
+//                         ));
+//                         // } else {
+//                         //     panic!(
+//                         //         "No prefix here, but there's a child here?"
+//                         //     );
+//                         // }
+//                     }
+//                     None => {
+//                         // No reference to another PrefixSet, all that's left, is
+//                         // checking for a prefix at the current cursor position.
+//                         // if let Some(meta) =
+//                         //     s_pfx.get_stored_prefix(self.guard).map(|p| {
+//                         //         // There's a prefix here, that's the next one
+//                         //         if log_enabled!(log::Level::Debug) {
+//                         //             debug!("E. found prefix {:?}", p.prefix);
+//                         //         }
+//                         //         p.record_map.as_records()
+//                         //     })
+//                         // {
+//                         self.cursor += 1;
+//                         return Some((
+//                             s_pfx.get_prefix_id().into(),
+//                             s_pfx.record_map.as_records(),
+//                         ));
+//                         // }
+//                     }
+//                 };
+//             } else {
+//                 self.cursor += 1;
+//             }
+//         }
+//     }
+// }
 
 // ----------- Sized Wrappers -----------------------------------------------
 
@@ -253,7 +250,7 @@ pub(crate) enum SizedNodeMoreSpecificIter<AF: AddressFamily> {
 }
 
 impl<AF: AddressFamily> SizedNodeMoreSpecificIter<AF> {
-    fn next(&mut self) -> Option<StrideNodeId<AF>> {
+    pub(crate) fn next(&mut self) -> Option<StrideNodeId<AF>> {
         match self {
             SizedNodeMoreSpecificIter::Stride3(iter) => iter.next(),
             SizedNodeMoreSpecificIter::Stride4(iter) => iter.next(),
@@ -269,7 +266,7 @@ pub(crate) enum SizedPrefixIter<AF: AddressFamily> {
 }
 
 impl<AF: AddressFamily> SizedPrefixIter<AF> {
-    fn next(&mut self) -> Option<PrefixId<AF>> {
+    pub(crate) fn next(&mut self) -> Option<PrefixId<AF>> {
         match self {
             SizedPrefixIter::Stride3(iter) => iter.next(),
             SizedPrefixIter::Stride4(iter) => iter.next(),
@@ -299,34 +296,34 @@ impl<AF: AddressFamily> SizedPrefixIter<AF> {
 pub(crate) struct MoreSpecificPrefixIter<
     'a,
     AF: AddressFamily,
-    M: Meta,
+    // M: Meta,
     NB: NodeBuckets<AF>,
-    PB: PrefixBuckets<AF, M>,
+    // PB: PrefixBuckets<AF, M>,
 > {
-    store: &'a TreeBitMap<AF, M, NB, PB>,
+    tree: &'a TreeBitMap<AF, NB>,
     cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
     cur_pfx_iter: SizedPrefixIter<AF>,
     // start_bit_span: BitSpan,
     // skip_self: bool,
     parent_and_position: Vec<SizedNodeMoreSpecificIter<AF>>,
     // If specified, we're only iterating over records for this mui.
-    mui: Option<u32>,
+    // mui: Option<u32>,
     // This is the tree-wide index of withdrawn muis, used to rewrite the
     // statuses of these records, or filter them out.
-    global_withdrawn_bmin: &'a RoaringBitmap,
+    // global_withdrawn_bmin: &'a RoaringBitmap,
     // Whether we should filter out the withdrawn records in the search result
-    include_withdrawn: bool,
+    // include_withdrawn: bool,
 }
 
 impl<
         'a,
         AF: AddressFamily + 'a,
-        M: Meta,
+        // M: Meta,
         NB: NodeBuckets<AF>,
-        PB: PrefixBuckets<AF, M>,
-    > Iterator for MoreSpecificPrefixIter<'a, AF, M, NB, PB>
+        // PB: PrefixBuckets<AF, M>,
+    > Iterator for MoreSpecificPrefixIter<'a, AF, NB>
 {
-    type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
+    type Item = PrefixId<AF>;
 
     fn next(&mut self) -> Option<Self::Item> {
         trace!("MoreSpecificsPrefixIter");
@@ -336,87 +333,88 @@ impl<
             let next_pfx = self.cur_pfx_iter.next();
 
             if next_pfx.is_some() {
+                return next_pfx;
                 // If we have a mui, we have to deal slightly different with
                 // the records: There can only be one record for a (prefix,
                 // mui) combination, and the record may be filtered out by the
                 // global status of the mui, or its local status. In that case
                 // we don't return here (because that would result in a Prefix
                 // with an empty record vec).
-                if let Some(mui) = self.mui {
-                    if let Some(p) = self
-                        .store
-                        .non_recursive_retrieve_prefix(
-                            next_pfx.unwrap_or_else(|| {
-                                panic!(
-                                "BOOM! More-specific prefix {:?} disappeared \
-                                from the store",
-                                next_pfx
-                            )
-                            }),
-                            // self.guard,
-                        )
-                        .0
-                    {
-                        // We may either have to rewrite the local status with
-                        // the provided global status OR we may have to omit
-                        // all of the records with either global of local
-                        // withdrawn status.
-                        if self.include_withdrawn {
-                            if let Some(rec) = p
-                                .record_map
-                                .get_record_for_mui_with_rewritten_status(
-                                    mui,
-                                    self.global_withdrawn_bmin,
-                                    RouteStatus::Withdrawn,
-                                )
-                            {
-                                return Some((p.prefix, vec![rec]));
-                            }
-                        } else if let Some(rec) = p
-                            .record_map
-                            .get_record_for_mui(mui, self.include_withdrawn)
-                        {
-                            return Some((p.prefix, vec![rec]));
-                        }
-                    };
-                } else {
-                    return self
-                        .store
-                        .non_recursive_retrieve_prefix(
-                            next_pfx.unwrap_or_else(|| {
-                                panic!(
-                                "BOOM! More-specific prefix {:?} disappeared \
-                                from the store",
-                                next_pfx
-                            )
-                            }),
-                            // self.guard,
-                        )
-                        .0
-                        .map(|p| {
-                            // Just like the mui specific records, we may have
-                            // to either rewrite the local status (if the user
-                            // wants the withdrawn records) or omit them.
-                            if self.include_withdrawn {
-                                (
-                                    p.prefix,
-                                    p.record_map
-                                        .as_records_with_rewritten_status(
-                                            self.global_withdrawn_bmin,
-                                            RouteStatus::Withdrawn,
-                                        ),
-                                )
-                            } else {
-                                (
-                                    p.prefix,
-                                    p.record_map
-                                        .as_active_records_not_in_bmin(
-                                            self.global_withdrawn_bmin,
-                                        ),
-                                )
-                            }
-                        });
-                }
+                // next_pfx
+                // if let Some(mui) = self.mui {
+                //     if let Some(p) = self
+                //         .store
+                //         .non_recursive_retrieve_prefix(
+                //             next_pfx.unwrap_or_else(|| {
+                //                 panic!(
+                //                 "BOOM! More-specific prefix {:?} disappeared \
+                //                 from the store",
+                //                 next_pfx
+                //             )
+                //             }),
+                //         )
+                //         .0
+                //     {
+                //         // We may either have to rewrite the local status with
+                //         // the provided global status OR we may have to omit
+                //         // all of the records with either global of local
+                //         // withdrawn status.
+                //         if self.include_withdrawn {
+                //             if let Some(rec) = p
+                //                 .record_map
+                //                 .get_record_for_mui_with_rewritten_status(
+                //                     mui,
+                //                     self.global_withdrawn_bmin,
+                //                     RouteStatus::Withdrawn,
+                //                 )
+                //             {
+                //                 return Some((p.prefix, vec![rec]));
+                //             }
+                //         } else if let Some(rec) = p
+                //             .record_map
+                //             .get_record_for_mui(mui, self.include_withdrawn)
+                //         {
+                //             return Some((p.prefix, vec![rec]));
+                //         }
+                //     };
+                // } else {
+                //     return self
+                //         .store
+                //         .non_recursive_retrieve_prefix(
+                //             next_pfx.unwrap_or_else(|| {
+                //                 panic!(
+                //                 "BOOM! More-specific prefix {:?} disappeared \
+                //                 from the store",
+                //                 next_pfx
+                //             )
+                //             }),
+                //             // self.guard,
+                //         )
+                //         .0
+                //         .map(|p| {
+                //             // Just like the mui specific records, we may have
+                //             // to either rewrite the local status (if the user
+                //             // wants the withdrawn records) or omit them.
+                //             if self.include_withdrawn {
+                //                 (
+                //                     p.prefix,
+                //                     p.record_map
+                //                         .as_records_with_rewritten_status(
+                //                             self.global_withdrawn_bmin,
+                //                             RouteStatus::Withdrawn,
+                //                         ),
+                //                 )
+                //             } else {
+                //                 (
+                //                     p.prefix,
+                //                     p.record_map
+                //                         .as_active_records_not_in_bmin(
+                //                             self.global_withdrawn_bmin,
+                //                         ),
+                //                 )
+                //             }
+                //         });
+                // }
             }
 
             // Our current prefix iterator for this node is done, look for
@@ -440,13 +438,13 @@ impl<
             }
 
             if let Some(next_ptr) = next_ptr {
-                let node = if self.mui.is_none() {
-                    trace!("let's retriev node {}", next_ptr);
-                    self.store.retrieve_node(next_ptr)
-                } else {
-                    self.store
-                        .retrieve_node_for_mui(next_ptr, self.mui.unwrap())
-                };
+                // let node = if self.mui.is_none() {
+                //     trace!("let's retriev node {}", next_ptr);
+                let node = self.tree.retrieve_node(next_ptr);
+                // } else {
+                //     self.store
+                //         .retrieve_node_for_mui(next_ptr, self.mui.unwrap())
+                // };
 
                 match node {
                     Some(SizedStrideRef::Stride3(next_node)) => {
@@ -525,6 +523,33 @@ impl<
     }
 }
 
+pub(crate) struct LMPrefixIter<'a, AF: AddressFamily, NB: NodeBuckets<AF>> {
+    tree: &'a TreeBitMap<AF, NB>,
+    prefix: PrefixId<AF>,
+}
+
+impl<AF: AddressFamily, NB: NodeBuckets<AF>> Iterator
+    for LMPrefixIter<'_, AF, NB>
+{
+    type Item = PrefixId<AF>;
+    fn next(&mut self) -> Option<Self::Item> {
+        trace!("search lm prefix for {:?}", self.prefix);
+
+        loop {
+            if self.prefix.get_len() == 0 {
+                return None;
+            }
+
+            if self.tree.prefix_exists(self.prefix) {
+                return Some(self.prefix);
+            }
+
+            self.prefix =
+                self.prefix.truncate_to_len(self.prefix.get_len() - 1);
+        }
+    }
+}
+
 // ----------- LessSpecificPrefixIter ---------------------------------------
 
 // This iterator iterates over all the less-specifics for a given prefix. It
@@ -534,179 +559,186 @@ impl<
 
 pub(crate) struct LessSpecificPrefixIter<
     'a,
-    AF: AddressFamily + 'a,
-    M: Meta + 'a,
-    PB: PrefixBuckets<AF, M>,
+    AF: AddressFamily,
+    NB: NodeBuckets<AF>,
 > {
-    prefixes: &'a PB,
-    cur_len: u8,
-    cur_bucket: &'a PrefixSet<AF, M>,
+    tree: &'a TreeBitMap<AF, NB>,
+    prefix: PrefixId<AF>,
     cur_level: u8,
-    cur_prefix_id: PrefixId<AF>,
-    mui: Option<u32>,
-    // Whether to include withdrawn records, both globally and local.
-    include_withdrawn: bool,
-    // This is the tree-wide index of withdrawn muis, used to filter out the
-    // records for those.
-    global_withdrawn_bmin: &'a RoaringBitmap,
 }
 
-impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
-    Iterator for LessSpecificPrefixIter<'a, AF, M, PB>
+impl<AF: AddressFamily, NB: NodeBuckets<AF>> Iterator
+    for LessSpecificPrefixIter<'_, AF, NB>
 {
-    type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
+    type Item = PrefixId<AF>;
 
     // This iterator moves down all prefix lengths, starting with the length
     // of the (search prefix - 1), looking for shorter prefixes, where the
     // its bits are the same as the bits of the search prefix.
     fn next(&mut self) -> Option<Self::Item> {
-        trace!("search next less-specific for {:?}", self.cur_prefix_id);
+        trace!("search next less-specific for {:?}", self.prefix);
+        self.cur_level = self.cur_level.saturating_sub(1);
 
         loop {
-            if self.cur_len == 0 {
-                // This is the end, my friend
-                trace!("reached min length {}, returning None", self.cur_len);
+            if self.cur_level == 0 {
                 return None;
             }
 
-            // shave a bit of the current prefix.
-            trace!(
-                "truncate to len {} (level {})",
-                self.cur_len,
-                self.cur_level
-            );
-            self.cur_prefix_id = PrefixId::new(
-                self.cur_prefix_id.get_net().truncate_to_len(self.cur_len),
-                self.cur_len,
-            );
-
-            let last_level = if self.cur_level > 0 {
-                PB::get_bits_for_len(self.cur_len, self.cur_level - 1)
-            } else {
-                0
-            };
-
-            let this_level =
-                PB::get_bits_for_len(self.cur_len, self.cur_level);
-
-            // NOT THE HASHING FUNCTION
-            let index = ((self.cur_prefix_id.get_net() << last_level)
-                >> ((AF::BITS - (this_level - last_level)) % AF::BITS))
-                .dangerously_truncate_to_u32()
-                as usize;
-
-            if this_level == 0 {
-                // END OF THE LENGTH
-                // This length is done too, go to the next length
-                // trace!("next length {}", self.cur_len + 1);
-                self.cur_len -= 1;
-
-                // a new length, a new life
-                // reset the level depth and cursor,
-                // but also empty all the parents
-                self.cur_level = 0;
-                // self.parents = [None; 26];
-
-                // let's continue, get the prefixes for the next length
-                self.cur_bucket =
-                    self.prefixes.get_root_prefix_set(self.cur_len);
-                continue;
+            let lvl_pfx = self.prefix.truncate_to_len(self.cur_level);
+            if self.tree.prefix_exists(lvl_pfx) {
+                self.cur_level = self.cur_level.saturating_sub(1);
+                return Some(lvl_pfx);
             }
 
-            // LEVEL DEPTH ITERATION
-            if let Some(stored_prefix) = self.cur_bucket.get_by_index(index) {
-                // if let Some(stored_prefix) =
-                // s_pfx.get_stored_prefix(self.guard)
-                // {
-                trace!("get_record {:?}", stored_prefix.record_map);
-                let pfx_rec = if let Some(mui) = self.mui {
-                    // We don't have to check for the appearance of We may
-                    // either have to rewrite the local status with the
-                    // provided global status OR we may have to omit all of
-                    // the records with either global of local withdrawn
-                    // status.
-                    if self.include_withdrawn {
-                        stored_prefix
-                            .record_map
-                            .get_record_for_mui_with_rewritten_status(
-                                mui,
-                                self.global_withdrawn_bmin,
-                                RouteStatus::Withdrawn,
-                            )
-                            .into_iter()
-                            .collect()
-                    } else {
-                        stored_prefix
-                            .record_map
-                            .get_record_for_mui(mui, self.include_withdrawn)
-                            .into_iter()
-                            .collect()
-                    }
-                } else {
-                    // Just like the mui specific records, we may have
-                    // to either rewrite the local status (if the user
-                    // wants the withdrawn records) or omit them.
-                    if self.include_withdrawn {
-                        stored_prefix
-                            .record_map
-                            .as_records_with_rewritten_status(
-                                self.global_withdrawn_bmin,
-                                RouteStatus::Withdrawn,
-                            )
-                    } else {
-                        stored_prefix
-                            .record_map
-                            .as_active_records_not_in_bmin(
-                                self.global_withdrawn_bmin,
-                            )
-                    }
-                };
-                // There is a prefix here, but we need to check if it's
-                // the right one.
-                if self.cur_prefix_id == stored_prefix.prefix {
-                    trace!("found requested prefix {:?}", self.cur_prefix_id);
-                    self.cur_len -= 1;
-                    self.cur_level = 0;
-                    self.cur_bucket =
-                        self.prefixes.get_root_prefix_set(self.cur_len);
-                    return if !pfx_rec.is_empty() {
-                        Some((stored_prefix.prefix, pfx_rec))
-                    } else {
-                        None
-                    };
-                };
-                // Advance to the next level or the next len.
-                match stored_prefix
-                    .next_bucket.is_empty()
-                    // .0
-                    // .load(Ordering::SeqCst, self.guard)
-                    // .is_null()
-                {
-                    // No child here, move one length down.
-                    true => {
-                        self.cur_len -= 1;
-                        self.cur_level = 0;
-                        self.cur_bucket =
-                            self.prefixes.get_root_prefix_set(self.cur_len);
-                    }
-                    // There's a child, move a level up and set the child
-                    // as current. Length remains the same.
-                    false => {
-                        self.cur_bucket = &stored_prefix.next_bucket;
-                        self.cur_level += 1;
-                    }
-                };
-            } else {
-                trace!("no prefix at this level. Move one down.");
-                self.cur_len -= 1;
-                self.cur_level = 0;
-                self.cur_bucket =
-                    self.prefixes.get_root_prefix_set(self.cur_len);
-            }
-            // }
+            self.cur_level = self.cur_level.saturating_sub(1);
         }
     }
 }
+// loop {
+//     if self.cur_len == 0 {
+//         // This is the end, my friend
+//         trace!("reached min length {}, returning None", self.cur_len);
+//         return None;
+//     }
+
+//     // shave a bit of the current prefix.
+//     trace!(
+//         "truncate to len {} (level {})",
+//         self.cur_len,
+//         self.cur_level
+//     );
+//     self.cur_prefix_id = PrefixId::new(
+//         self.cur_prefix_id.get_net().truncate_to_len(self.cur_len),
+//         self.cur_len,
+//     );
+
+//     let last_level = if self.cur_level > 0 {
+//         PB::get_bits_for_len(self.cur_len, self.cur_level - 1)
+//     } else {
+//         0
+//     };
+
+//     let this_level =
+//         PB::get_bits_for_len(self.cur_len, self.cur_level);
+
+//     // NOT THE HASHING FUNCTION
+//     let index = ((self.cur_prefix_id.get_net() << last_level)
+//         >> ((AF::BITS - (this_level - last_level)) % AF::BITS))
+//         .dangerously_truncate_to_u32()
+//         as usize;
+
+//     if this_level == 0 {
+//         // END OF THE LENGTH
+//         // This length is done too, go to the next length
+//         // trace!("next length {}", self.cur_len + 1);
+//         self.cur_len -= 1;
+
+//         // a new length, a new life
+//         // reset the level depth and cursor,
+//         // but also empty all the parents
+//         self.cur_level = 0;
+//         // self.parents = [None; 26];
+
+//         // let's continue, get the prefixes for the next length
+//         self.cur_bucket =
+//             self.prefixes.get_root_prefix_set(self.cur_len);
+//         continue;
+//     }
+
+//     // LEVEL DEPTH ITERATION
+//     if let Some(stored_prefix) = self.cur_bucket.get_by_index(index) {
+//         // if let Some(stored_prefix) =
+//         // s_pfx.get_stored_prefix(self.guard)
+//         // {
+//         // trace!("get_record {:?}", stored_prefix.record_map);
+//         // let pfx_rec = if let Some(mui) = self.mui {
+//         // We don't have to check for the appearance of We may
+//         // either have to rewrite the local status with the
+//         // provided global status OR we may have to omit all of
+//         // the records with either global of local withdrawn
+//         // status.
+//         // if self.include_withdrawn {
+//         // let pfx_rec = stored_prefix
+//         //     .record_map
+//         //     .get_record_for_mui_with_rewritten_status(
+//         //         mui,
+//         //         self.global_withdrawn_bmin,
+//         //         RouteStatus::Withdrawn,
+//         //     )
+//         //     .into_iter()
+//         //     .collect();
+//         // } else {
+//         let pfx_rec = stored_prefix
+//             .record_map
+//             .get_record_for_mui(mui, self.include_withdrawn)
+//             .into_iter()
+//             .collect();
+//         // }
+//         // } else {
+//         // Just like the mui specific records, we may have
+//         // to either rewrite the local status (if the user
+//         // wants the withdrawn records) or omit them.
+//         //     if self.include_withdrawn {
+//         //         stored_prefix
+//         //             .record_map
+//         //             .as_records_with_rewritten_status(
+//         //                 self.global_withdrawn_bmin,
+//         //                 RouteStatus::Withdrawn,
+//         //             )
+//         //     } else {
+//         //         stored_prefix
+//         //             .record_map
+//         //             .as_active_records_not_in_bmin(
+//         //                 self.global_withdrawn_bmin,
+//         //             )
+//         //     }
+//         // };
+//         // There is a prefix here, but we need to check if it's
+//         // the right one.
+//         if self.cur_prefix_id == stored_prefix.prefix {
+//             trace!("found requested prefix {:?}", self.cur_prefix_id);
+//             self.cur_len -= 1;
+//             self.cur_level = 0;
+//             self.cur_bucket =
+//                 self.prefixes.get_root_prefix_set(self.cur_len);
+//             return if !pfx_rec.is_empty() {
+//                 Some((stored_prefix.prefix, pfx_rec))
+//             } else {
+//                 None
+//             };
+//         };
+//         // Advance to the next level or the next len.
+//         match stored_prefix
+//             .next_bucket.is_empty()
+//             // .0
+//             // .load(Ordering::SeqCst, self.guard)
+//             // .is_null()
+//         {
+//             // No child here, move one length down.
+//             true => {
+//                 self.cur_len -= 1;
+//                 self.cur_level = 0;
+//                 self.cur_bucket =
+//                     self.prefixes.get_root_prefix_set(self.cur_len);
+//             }
+//             // There's a child, move a level up and set the child
+//             // as current. Length remains the same.
+//             false => {
+//                 self.cur_bucket = &stored_prefix.next_bucket;
+//                 self.cur_level += 1;
+//             }
+//         };
+//     } else {
+//         trace!("no prefix at this level. Move one down.");
+//         self.cur_len -= 1;
+//         self.cur_level = 0;
+//         self.cur_bucket =
+//             self.prefixes.get_root_prefix_set(self.cur_len);
+//     }
+//     // }
+// }
+//     }
+// }
 
 // ----------- Iterator initialization methods for Rib -----------------------
 
@@ -716,20 +748,20 @@ impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
 impl<
         'a,
         AF: AddressFamily,
-        M: crate::prefix_record::Meta,
+        // M: crate::prefix_record::Meta,
         NB: NodeBuckets<AF>,
-        PB: PrefixBuckets<AF, M>,
-    > TreeBitMap<AF, M, NB, PB>
+        // PB: PrefixBuckets<AF, M>,
+    > TreeBitMap<AF, NB>
 {
     // Iterator over all more-specific prefixes, starting from the given
     // prefix at the given level and cursor.
     pub fn more_specific_prefix_iter_from(
         &'a self,
         start_prefix_id: PrefixId<AF>,
-        mui: Option<u32>,
-        include_withdrawn: bool,
-        guard: &'a Guard,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        // mui: Option<u32>,
+        // include_withdrawn: bool,
+        // guard: &'a Guard,
+    ) -> impl Iterator<Item = PrefixId<AF>> + 'a {
         trace!("more specifics for {:?}", start_prefix_id);
 
         // A v4 /32 or a v6 /128 doesn't have more specific prefixes 🤓.
@@ -767,11 +799,11 @@ impl<
             let cur_pfx_iter: SizedPrefixIter<AF>;
             let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
 
-            let node = if let Some(mui) = mui {
-                self.retrieve_node_for_mui(start_node_id, mui)
-            } else {
-                self.retrieve_node(start_node_id)
-            };
+            // let node = if let Some(mui) = mui {
+            // self.retrieve_node_for_mui(start_node_id, mui)
+            // } else {
+            let node = self.retrieve_node(start_node_id);
+            // };
 
             if let Some(node) = node {
                 match node {
@@ -803,17 +835,16 @@ impl<
                     }
                 };
 
-                let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
+                // let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
 
                 Some(MoreSpecificPrefixIter {
-                    store: self,
+                    tree: self,
                     cur_pfx_iter,
                     cur_ptr_iter,
-                    // start_bit_span,
                     parent_and_position: vec![],
-                    global_withdrawn_bmin,
-                    include_withdrawn,
-                    mui,
+                    // global_withdrawn_bmin,
+                    // include_withdrawn,
+                    // mui,
                 })
             } else {
                 None
@@ -830,50 +861,60 @@ impl<
         start_prefix_id: PrefixId<AF>,
         // Indicate whether we want to return only records for a specific mui,
         // None indicates returning all records for a prefix.
-        mui: Option<u32>,
-        include_withdrawn: bool,
-        guard: &'a Guard,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-        trace!("less specifics for {:?}", start_prefix_id);
-        trace!("level {}, len {}", 0, start_prefix_id.get_len());
+        // mui: Option<u32>,
+        // include_withdrawn: bool,
+        // guard: &'a Guard,
+    ) -> impl Iterator<Item = PrefixId<AF>> + 'a {
+        if log_enabled!(log::Level::Trace) {
+            trace!("less specifics for {}", Prefix::from(start_prefix_id));
+            trace!("level {}, len {}", 0, start_prefix_id.get_len());
+        }
 
         // We could just let the /0 prefix search the tree and have it return
         // an empty iterator, but to avoid having to read out the root node
         // for this prefix, we'll just return an empty iterator. None can be
         // turned into an Iterator!
-        if start_prefix_id.get_len() < 1 {
-            None
-        } else {
-            let cur_len = start_prefix_id.get_len() - 1;
-            let cur_bucket = self.prefix_buckets.get_root_prefix_set(cur_len);
-            let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
-
-            Some(LessSpecificPrefixIter {
-                prefixes: &self.prefix_buckets,
-                cur_len,
-                cur_bucket,
-                cur_level: 0,
-                cur_prefix_id: start_prefix_id,
-                mui,
-                global_withdrawn_bmin,
-                include_withdrawn,
-            })
+        // if start_prefix_id.get_len() < 1 {
+        //     None
+        // } else {
+        // let cur_len = start_prefix_id.get_len() - 1;
+        // let cur_bucket = self.prefix_store.get_root_prefix_set(cur_len);
+        // let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
+
+        // Some(
+
+        LessSpecificPrefixIter {
+            // prefixes: &self.prefix_buckets,
+            tree: self,
+            prefix: start_prefix_id,
+            // cur_len,
+            // cur_bucket,
+            cur_level: start_prefix_id.get_len(),
+            // cur_prefix_id: start_prefix_id,
+            // mui,
+            // global_withdrawn_bmin,
+            // include_withdrawn,
         }
-        .into_iter()
-        .flatten()
+        //     })
+        // }
+        // .into_iter()
+        // .flatten()
     }
 
-    // Iterator over all the prefixes in the in_memory store.
-    pub fn prefixes_iter(
+    pub fn longest_matching_prefix(
         &'a self,
-    ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
-        PrefixIter {
-            prefixes: &self.prefix_buckets,
-            cur_bucket: self.prefix_buckets.get_root_prefix_set(0),
-            cur_len: 0,
-            cur_level: 0,
-            cursor: 0,
-            parents: [None; 32],
+        prefix: PrefixId<AF>,
+    ) -> Option<PrefixId<AF>> {
+        if log_enabled!(log::Level::Trace) {
+            trace!("lmp for {}", Prefix::from(prefix));
         }
+
+        LMPrefixIter { tree: self, prefix }.next()
+    }
+
+    // Iterator over all the prefixes in the in_memory store.
+    pub fn prefixes_iter(&'a self) -> impl Iterator<Item = Prefix> + 'a {
+        self.more_specific_prefix_iter_from(PrefixId::new(AF::zero(), 0))
+            .map(Prefix::from)
     }
 }
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index ef0798d1..1a40c263 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -12,134 +12,179 @@ use crate::{MatchOptions, MatchType};
 
 use super::super::in_memory::tree::Stride;
 use super::super::types::PrefixId;
-use super::atomic_types::{NodeBuckets, PrefixBuckets};
+use super::atomic_types::NodeBuckets;
 use super::node::StrideNodeId;
 use super::tree::TreeBitMap;
 
-impl<AF, M, NB, PB> TreeBitMap<AF, M, NB, PB>
+impl<AF, NB> TreeBitMap<AF, NB>
 where
     AF: AddressFamily,
-    M: Meta,
     NB: NodeBuckets<AF>,
-    PB: PrefixBuckets<AF, M>,
 {
-    pub(crate) fn match_prefix<'a>(
-        &'a self,
+    pub(crate) fn match_prefix(
+        &self,
         search_pfx: PrefixId<AF>,
         options: &MatchOptions,
-        guard: &'a Guard,
-    ) -> FamilyQueryResult<AF, M> {
-        // `non_recursive_retrieve_prefix` returns an exact match only, so no
-        // longest matching prefix!
-        let withdrawn_muis_bmin = self.withdrawn_muis_bmin(guard);
-        let mut stored_prefix =
-            self.non_recursive_retrieve_prefix(search_pfx).0.map(|pfx| {
-                (
-                    pfx.prefix,
-                    if !options.include_withdrawn {
-                        // Filter out all the withdrawn records, both
-                        // with globally withdrawn muis, and with local
-                        // statuses
-                        // set to Withdrawn.
-                        pfx.record_map
-                            .get_filtered_records(
-                                options.mui,
-                                options.include_withdrawn,
-                                withdrawn_muis_bmin,
-                            )
-                            .into_iter()
-                            .collect()
+    ) -> TreeQueryResult<AF> {
+        let mut ls_pfxs = None;
+        let mut ms_pfxs = None;
+        let lm_pfx = self.longest_matching_prefix(search_pfx);
+
+        let (prefix, match_type) = match options.match_type {
+            MatchType::ExactMatch => {
+                if let Some(p) = lm_pfx {
+                    if p == search_pfx {
+                        (lm_pfx, MatchType::ExactMatch)
                     } else {
-                        // Do no filter out any records, but do rewrite
-                        // the local statuses of the records with muis
-                        // that appear in the specified bitmap index.
-                        pfx.record_map.as_records_with_rewritten_status(
-                            withdrawn_muis_bmin,
-                            RouteStatus::Withdrawn,
-                        )
-                    },
-                )
-            });
-
-        // Check if we have an actual exact match, if not then fetch the
-        // first lesser-specific with the greatest length, that's the Longest
-        // matching prefix, but only if the user requested a longest match or
-        // empty match.
-        let match_type = match (&options.match_type, &stored_prefix) {
-            // we found an exact match, we don't need to do anything.
-            (_, Some((_pfx, meta))) if !meta.is_empty() => {
-                MatchType::ExactMatch
-            }
-            // we didn't find an exact match, but the user requested it
-            // so we need to find the longest matching prefix.
-            (MatchType::LongestMatch | MatchType::EmptyMatch, _) => {
-                stored_prefix = self
-                    .less_specific_prefix_iter(
-                        search_pfx,
-                        options.mui,
-                        options.include_withdrawn,
-                        guard,
-                    )
-                    .max_by(|p0, p1| p0.0.get_len().cmp(&p1.0.get_len()));
-                if stored_prefix.is_some() {
-                    MatchType::LongestMatch
+                        (None, MatchType::EmptyMatch)
+                    }
                 } else {
-                    MatchType::EmptyMatch
+                    (None, MatchType::EmptyMatch)
                 }
             }
-            // We got an empty match, but the user requested an exact match,
-            // even so, we're going to look for more and/or less specifics if
-            // the user asked for it.
-            (MatchType::ExactMatch, _) => MatchType::EmptyMatch,
+            _ => (lm_pfx, MatchType::LongestMatch),
         };
 
-        FamilyQueryResult {
-            prefix: stored_prefix.as_ref().map(|sp| sp.0),
-            prefix_meta: stored_prefix
-                .as_ref()
-                .map(|pfx| pfx.1.clone())
-                .unwrap_or_default(),
-            less_specifics: if options.include_less_specifics {
-                Some(
-                    self.less_specific_prefix_iter(
-                        if let Some(ref pfx) = stored_prefix {
-                            pfx.0
-                        } else {
-                            search_pfx
-                        },
-                        options.mui,
-                        options.include_withdrawn,
-                        guard,
-                    )
-                    .collect::<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>>(),
-                )
-            } else {
-                None
-            },
-            more_specifics: if options.include_more_specifics {
-                trace!("more specifics requested. acquiring...");
-                Some(
-                    self.more_specific_prefix_iter_from(
-                        if let Some(pfx) = stored_prefix {
-                            pfx.0
-                        } else {
-                            search_pfx
-                        },
-                        options.mui,
-                        options.include_withdrawn,
-                        guard,
-                    )
+        if options.include_less_specifics {
+            ls_pfxs = Some(
+                self.less_specific_prefix_iter(search_pfx)
                     .collect::<Vec<_>>(),
-                )
-                // The user requested more specifics, but there aren't any, so
-                // we need to return an empty vec, not a None.
-            } else {
-                None
-            },
+            )
+        }
+
+        if options.include_more_specifics {
+            ms_pfxs = Some(
+                self.more_specific_prefix_iter_from(search_pfx)
+                    .filter(|p| p != &search_pfx)
+                    .collect::<Vec<_>>(),
+            );
+        }
+
+        TreeQueryResult {
             match_type,
+            prefix,
+            less_specifics: ls_pfxs,
+            more_specifics: ms_pfxs,
         }
     }
 
+    // pub(crate) fn match_prefix<'a>(
+    //     &'a self,
+    //     search_pfx: PrefixId<AF>,
+    //     options: &MatchOptions,
+    //     guard: &'a Guard,
+    // ) -> FamilyQueryResult<AF, M> {
+    //     // `non_recursive_retrieve_prefix` returns an exact match only, so no
+    //     // longest matching prefix!
+    //     let withdrawn_muis_bmin = self.withdrawn_muis_bmin(guard);
+    //     let mut stored_prefix =
+    //         self.non_recursive_retrieve_prefix(search_pfx).0.map(|pfx| {
+    //             (
+    //                 pfx.prefix,
+    //                 if !options.include_withdrawn {
+    //                     // Filter out all the withdrawn records, both
+    //                     // with globally withdrawn muis, and with local
+    //                     // statuses
+    //                     // set to Withdrawn.
+    //                     pfx.record_map
+    //                         .get_filtered_records(
+    //                             options.mui,
+    //                             options.include_withdrawn,
+    //                             withdrawn_muis_bmin,
+    //                         )
+    //                         .into_iter()
+    //                         .collect()
+    //                 } else {
+    //                     // Do no filter out any records, but do rewrite
+    //                     // the local statuses of the records with muis
+    //                     // that appear in the specified bitmap index.
+    //                     pfx.record_map.as_records_with_rewritten_status(
+    //                         withdrawn_muis_bmin,
+    //                         RouteStatus::Withdrawn,
+    //                     )
+    //                 },
+    //             )
+    //         });
+
+    //     // Check if we have an actual exact match, if not then fetch the
+    //     // first lesser-specific with the greatest length, that's the Longest
+    //     // matching prefix, but only if the user requested a longest match or
+    //     // empty match.
+    //     let match_type = match (&options.match_type, &stored_prefix) {
+    //         // we found an exact match, we don't need to do anything.
+    //         (_, Some((_pfx, meta))) if !meta.is_empty() => {
+    //             MatchType::ExactMatch
+    //         }
+    //         // we didn't find an exact match, but the user requested it
+    //         // so we need to find the longest matching prefix.
+    //         (MatchType::LongestMatch | MatchType::EmptyMatch, _) => {
+    //             stored_prefix = self
+    //                 .less_specific_prefix_iter(
+    //                     search_pfx,
+    //                     options.mui,
+    //                     options.include_withdrawn,
+    //                     guard,
+    //                 )
+    //                 .max_by(|p0, p1| p0.0.get_len().cmp(&p1.0.get_len()));
+    //             if stored_prefix.is_some() {
+    //                 MatchType::LongestMatch
+    //             } else {
+    //                 MatchType::EmptyMatch
+    //             }
+    //         }
+    //         // We got an empty match, but the user requested an exact match,
+    //         // even so, we're going to look for more and/or less specifics if
+    //         // the user asked for it.
+    //         (MatchType::ExactMatch, _) => MatchType::EmptyMatch,
+    //     };
+
+    //     FamilyQueryResult {
+    //         prefix: stored_prefix.as_ref().map(|sp| sp.0),
+    //         prefix_meta: stored_prefix
+    //             .as_ref()
+    //             .map(|pfx| pfx.1.clone())
+    //             .unwrap_or_default(),
+    //         less_specifics: if options.include_less_specifics {
+    //             Some(
+    //                 self.less_specific_prefix_iter(
+    //                     if let Some(ref pfx) = stored_prefix {
+    //                         pfx.0
+    //                     } else {
+    //                         search_pfx
+    //                     },
+    //                     options.mui,
+    //                     options.include_withdrawn,
+    //                     guard,
+    //                 )
+    //                 .collect::<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>>(),
+    //             )
+    //         } else {
+    //             None
+    //         },
+    //         more_specifics: if options.include_more_specifics {
+    //             trace!("more specifics requested. acquiring...");
+    //             Some(
+    //                 self.more_specific_prefix_iter_from(
+    //                     if let Some(pfx) = stored_prefix {
+    //                         pfx.0
+    //                     } else {
+    //                         search_pfx
+    //                     },
+    //                     options.mui,
+    //                     options.include_withdrawn,
+    //                     guard,
+    //                 )
+    //                 .collect::<Vec<_>>(),
+    //             )
+    //             // The user requested more specifics, but there aren't any, so
+    //             // we need to return an empty vec, not a None.
+    //         } else {
+    //             None
+    //         },
+    //         match_type,
+    //     }
+    // }
+
     // This function assembles all entries in the `pfx_vec` of all child nodes
     // of the `start_node` into one vec, starting from itself and then
     // recursively assembling adding all `pfx_vec`s of its children.
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 4df1ae79..4ae9ac51 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -185,10 +185,8 @@
 
 use crate::local_array::bit_span::BitSpan;
 use crate::local_array::in_memory::atomic_types::{NodeSet, StoredNode};
-use crate::prefix_record::Meta;
 use crate::prelude::multi::PrefixId;
 use crossbeam_epoch::{Atomic, Guard};
-use crossbeam_utils::Backoff;
 use inetnum::addr::Prefix;
 use log::{debug, error, log_enabled, trace};
 use roaring::RoaringBitmap;
@@ -198,14 +196,12 @@ use std::sync::atomic::{
 };
 use std::{fmt::Debug, marker::PhantomData};
 
-use super::atomic_types::{
-    MultiMapValue, NodeBuckets, PrefixBuckets, PrefixSet, StoredPrefix,
-};
+use super::atomic_types::NodeBuckets;
 use crate::af::AddressFamily;
-use crate::rib::{Counters, UpsertReport};
+use crate::rib::Counters;
 use crate::{
     impl_search_level, impl_search_level_for_mui, insert_match,
-    retrieve_node_mut_closure, store_node_closure, PublicRecord,
+    retrieve_node_mut_closure, store_node_closure,
 };
 
 use super::super::errors::PrefixStoreError;
@@ -222,33 +218,33 @@ use ansi_term::Colour;
 #[derive(Debug)]
 pub struct TreeBitMap<
     AF: AddressFamily,
-    M: Meta,
+    // M: Meta,
     NB: NodeBuckets<AF>,
-    PB: PrefixBuckets<AF, M>,
+    // PB: PrefixBuckets<AF, M>,
 > {
     pub(crate) node_buckets: NB,
-    pub(crate) prefix_buckets: PB,
+    // pub(crate) prefix_buckets: PB,
     pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     counters: Counters,
     _af: PhantomData<AF>,
-    _m: PhantomData<M>,
+    // _m: PhantomData<M>,
 }
 
 impl<
         AF: AddressFamily,
-        M: Meta,
+        // M: Meta,
         NB: NodeBuckets<AF>,
-        PB: PrefixBuckets<AF, M>,
-    > TreeBitMap<AF, M, NB, PB>
+        // PB: PrefixBuckets<AF, M>,
+    > TreeBitMap<AF, NB>
 {
     pub(crate) fn new() -> Result<Self, Box<dyn std::error::Error>> {
         let tree_bitmap = Self {
             node_buckets: NodeBuckets::init(),
-            prefix_buckets: PB::init(),
+            // prefix_buckets: PB::init(),
             withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
             _af: PhantomData,
-            _m: PhantomData,
+            // _m: PhantomData,
         };
 
         let root_node = match Self::get_first_stride_size() {
@@ -376,9 +372,32 @@ impl<
     }
 
     pub fn prefix_exists(&self, prefix_id: PrefixId<AF>) -> bool {
+        trace!("pe exists {:?}?", prefix_id);
+        let (node_id, bs) = self.get_node_id_for_prefix(&prefix_id);
+
+        match self.retrieve_node(node_id) {
+            Some(SizedStrideRef::Stride4(n)) => {
+                let pfxbitarr = n.pfxbitarr.load();
+                pfxbitarr & Stride4::get_bit_pos(bs.bits, bs.len) > 0
+            }
+            None => false,
+            Some(n) => {
+                panic!(
+                    "unsupported stride length: {:?} node_id {}",
+                    n, node_id
+                );
+            }
+        }
+    }
+
+    pub fn prefix_exists_legacy(&self, prefix_id: PrefixId<AF>) -> bool {
         // if prefix_id.get_len() == 0 {
         //     return self.update_default_route_prefix_meta(mui);
         // }
+        trace!("exists {}?", Prefix::from(prefix_id));
+        if prefix_id.get_len() == 0 {
+            return false;
+        }
 
         let mut stride_end: u8 = 0;
         let mut cur_i = self.get_root_node_id();
@@ -387,6 +406,8 @@ impl<
         loop {
             let stride = self.get_stride_sizes()[level as usize];
             stride_end += stride;
+            trace!("stride end {}", stride_end);
+            assert!(stride_end <= AF::BITS);
             let nibble_len = if prefix_id.get_len() < stride_end {
                 stride + prefix_id.get_len() - stride_end
             } else {
@@ -450,7 +471,10 @@ impl<
             if let Some(next_id) = node_result {
                 cur_i = next_id;
                 level += 1;
+                continue;
             }
+
+            return false;
         }
     }
 
@@ -537,86 +561,6 @@ impl<
         }
     }
 
-    pub(crate) fn upsert_prefix(
-        &self,
-        prefix: PrefixId<AF>,
-        record: PublicRecord<M>,
-        update_path_selections: Option<M::TBI>,
-        guard: &Guard,
-    ) -> Result<(UpsertReport, Option<MultiMapValue<M>>), PrefixStoreError>
-    {
-        let mut prefix_is_new = true;
-        let mut mui_is_new = true;
-
-        let (mui_count, cas_count) =
-            match self.non_recursive_retrieve_prefix_mut(prefix) {
-                // There's no StoredPrefix at this location yet. Create a new
-                // PrefixRecord and try to store it in the empty slot.
-                (stored_prefix, false) => {
-                    if log_enabled!(log::Level::Debug) {
-                        debug!(
-                            "{} store: Create new prefix record",
-                            std::thread::current()
-                                .name()
-                                .unwrap_or("unnamed-thread")
-                        );
-                    }
-
-                    let (mui_count, retry_count) =
-                        stored_prefix.record_map.upsert_record(record)?;
-
-                    // See if someone beat us to creating the record.
-                    if mui_count.is_some() {
-                        mui_is_new = false;
-                        prefix_is_new = false;
-                    }
-
-                    (mui_count, retry_count)
-                }
-                // There already is a StoredPrefix with a record at this
-                // location.
-                (stored_prefix, true) => {
-                    if log_enabled!(log::Level::Debug) {
-                        debug!(
-                        "{} store: Found existing prefix record for {}/{}",
-                        std::thread::current()
-                            .name()
-                            .unwrap_or("unnamed-thread"),
-                        prefix.get_net(),
-                        prefix.get_len()
-                    );
-                    }
-                    prefix_is_new = false;
-
-                    // Update the already existing record_map with our
-                    // caller's record.
-                    stored_prefix.set_ps_outdated(guard)?;
-
-                    let (mui_count, retry_count) =
-                        stored_prefix.record_map.upsert_record(record)?;
-                    mui_is_new = mui_count.is_none();
-
-                    if let Some(tbi) = update_path_selections {
-                        stored_prefix
-                            .calculate_and_store_best_backup(&tbi, guard)?;
-                    }
-
-                    (mui_count, retry_count)
-                }
-            };
-
-        let count = mui_count.as_ref().map(|m| m.1).unwrap_or(1);
-        Ok((
-            UpsertReport {
-                prefix_new: prefix_is_new,
-                cas_count,
-                mui_new: mui_is_new,
-                mui_count: count,
-            },
-            mui_count.map(|m| m.0),
-        ))
-    }
-
     // Yes, we're hating this. But, the root node has no room for a serial of
     // the prefix 0/0 (the default route), which doesn't even matter, unless,
     // UNLESS, somebody wants to store a default route. So we have to store a
@@ -912,155 +856,6 @@ impl<
         self.counters.get_nodes_count()
     }
 
-    // This function is used by the upsert_prefix function above.
-    //
-    // We're using a Chained Hash Table and this function returns one of:
-    // - a StoredPrefix that already exists for this search_prefix_id
-    // - the Last StoredPrefix in the chain.
-    // - an error, if no StoredPrefix whatsoever can be found in the store.
-    //
-    // The error condition really shouldn't happen, because that basically
-    // means the root node for that particular prefix length doesn't exist.
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn non_recursive_retrieve_prefix_mut(
-        &self,
-        search_prefix_id: PrefixId<AF>,
-    ) -> (&StoredPrefix<AF, M>, bool) {
-        trace!("non_recursive_retrieve_prefix_mut_with_guard");
-        let mut prefix_set = self
-            .prefix_buckets
-            .get_root_prefix_set(search_prefix_id.get_len());
-        let mut level: u8 = 0;
-
-        trace!("root prefix_set {:?}", prefix_set);
-        loop {
-            // HASHING FUNCTION
-            let index = TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(
-                search_prefix_id,
-                level,
-            );
-
-            // probe the slot with the index that's the result of the hashing.
-            let stored_prefix = match prefix_set.0.get(index) {
-                Some(p) => {
-                    trace!("prefix set found.");
-                    (p, true)
-                }
-                None => {
-                    // We're at the end of the chain and haven't found our
-                    // search_prefix_id anywhere. Return the end-of-the-chain
-                    // StoredPrefix, so the caller can attach a new one.
-                    trace!(
-                        "no record. returning last found record in level
-                        {}, with index {}.",
-                        level,
-                        index
-                    );
-                    let index = TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(
-                        search_prefix_id,
-                        level,
-                    );
-                    trace!("calculate next index {}", index);
-                    let var_name = (
-                        prefix_set
-                            .0
-                            .get_or_init(index, || {
-                                StoredPrefix::new::<PB>(
-                                    PrefixId::new(
-                                        search_prefix_id.get_net(),
-                                        search_prefix_id.get_len(),
-                                    ),
-                                    level,
-                                )
-                            })
-                            .0,
-                        false,
-                    );
-                    var_name
-                }
-            };
-
-            if search_prefix_id == stored_prefix.0.prefix {
-                // GOTCHA!
-                // Our search-prefix is stored here, so we're returning
-                // it, so its PrefixRecord can be updated by the caller.
-                if log_enabled!(log::Level::Trace) {
-                    trace!(
-                        "found requested prefix {} ({:?})",
-                        Prefix::from(search_prefix_id),
-                        search_prefix_id
-                    );
-                }
-                return stored_prefix;
-            } else {
-                // A Collision. Follow the chain.
-                level += 1;
-                prefix_set = &stored_prefix.0.next_bucket;
-                continue;
-            }
-        }
-    }
-
-    // This function is used by the match_prefix, and [more|less]_specifics
-    // public methods on the TreeBitMap (indirectly).
-    #[allow(clippy::type_complexity)]
-    pub fn non_recursive_retrieve_prefix(
-        &self,
-        id: PrefixId<AF>,
-    ) -> (
-        Option<&StoredPrefix<AF, M>>,
-        Option<(
-            PrefixId<AF>,
-            u8,
-            &PrefixSet<AF, M>,
-            [Option<(&PrefixSet<AF, M>, usize)>; 32],
-            usize,
-        )>,
-    ) {
-        let mut prefix_set =
-            self.prefix_buckets.get_root_prefix_set(id.get_len());
-        let mut parents = [None; 32];
-        let mut level: u8 = 0;
-        let backoff = Backoff::new();
-
-        loop {
-            // The index of the prefix in this array (at this len and
-            // level) is calculated by performing the hash function
-            // over the prefix.
-
-            // HASHING FUNCTION
-            let index =
-                TreeBitMap::<AF, M, NB, PB>::hash_prefix_id(id, level);
-
-            if let Some(stored_prefix) = prefix_set.0.get(index) {
-                if id == stored_prefix.get_prefix_id() {
-                    if log_enabled!(log::Level::Trace) {
-                        trace!(
-                            "found requested prefix {} ({:?})",
-                            Prefix::from(id),
-                            id
-                        );
-                    }
-                    parents[level as usize] = Some((prefix_set, index));
-                    return (
-                        Some(stored_prefix),
-                        Some((id, level, prefix_set, parents, index)),
-                    );
-                };
-
-                // Advance to the next level.
-                prefix_set = &stored_prefix.next_bucket;
-                level += 1;
-                backoff.spin();
-                continue;
-            }
-
-            trace!("no prefix found for {:?}", id);
-            parents[level as usize] = Some((prefix_set, index));
-            return (None, Some((id, level, prefix_set, parents, index)));
-        }
-    }
-
     pub fn get_prefixes_count(&self) -> usize {
         self.counters.get_prefixes_count().iter().sum()
     }
@@ -1188,31 +983,6 @@ impl<
             >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
             .dangerously_truncate_to_u32() as usize
     }
-
-    pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
-        // And, this is all of our hashing function.
-        let last_level = if level > 0 {
-            <PB>::get_bits_for_len(id.get_len(), level - 1)
-        } else {
-            0
-        };
-        let this_level = <PB>::get_bits_for_len(id.get_len(), level);
-        // trace!(
-        //     "bits division {}; no of bits {}",
-        //     this_level,
-        //     this_level - last_level
-        // );
-        // trace!(
-        //     "calculated index ({} << {}) >> {}",
-        //     id.get_net(),
-        //     last_level,
-        //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-        // );
-        // HASHING FUNCTION
-        ((id.get_net() << last_level)
-            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
-            .dangerously_truncate_to_u32() as usize
-    }
 }
 
 // Partition for stride 4
@@ -1250,10 +1020,10 @@ impl<
 #[cfg(feature = "cli")]
 impl<
         AF: AddressFamily,
-        M: Meta,
+        // M: Meta,
         NB: NodeBuckets<AF>,
-        PB: PrefixBuckets<AF, M>,
-    > std::fmt::Display for TreeBitMap<AF, M, NB, PB>
+        // PB: PrefixBuckets<AF, M>,
+    > std::fmt::Display for TreeBitMap<AF, NB>
 {
     fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(_f, "{} prefixes created", self.get_prefixes_count())?;
diff --git a/src/local_array/mod.rs b/src/local_array/mod.rs
index 2d511450..bc4bee30 100644
--- a/src/local_array/mod.rs
+++ b/src/local_array/mod.rs
@@ -1,5 +1,6 @@
 pub(crate) mod in_memory;
 pub(crate) mod persist;
+pub(crate) mod prefix_cht;
 mod tests;
 
 pub(crate) mod bit_span;
diff --git a/src/local_array/prefix_cht/cht.rs b/src/local_array/prefix_cht/cht.rs
new file mode 100644
index 00000000..2283f8b5
--- /dev/null
+++ b/src/local_array/prefix_cht/cht.rs
@@ -0,0 +1,329 @@
+use std::marker::PhantomData;
+
+use crossbeam_epoch::Guard;
+use crossbeam_utils::Backoff;
+use inetnum::addr::Prefix;
+use log::{debug, log_enabled, trace};
+use roaring::RoaringBitmap;
+
+use crate::{
+    local_array::in_memory::atomic_types::{MultiMapValue, StoredPrefix},
+    prelude::multi::{PrefixBuckets, PrefixId, PrefixSet, PrefixStoreError},
+    rib::UpsertReport,
+    AddressFamily, Meta, PublicRecord,
+};
+
+#[derive(Debug)]
+pub(crate) struct PrefixCHT<
+    AF: AddressFamily,
+    M: Meta,
+    PB: PrefixBuckets<AF, M>,
+> {
+    store: PB,
+    _af: PhantomData<AF>,
+    _m: PhantomData<M>,
+}
+
+impl<AF: AddressFamily, M: Meta, PB: PrefixBuckets<AF, M>>
+    PrefixCHT<AF, M, PB>
+{
+    pub(crate) fn new() -> Self {
+        Self {
+            store: PB::init(),
+            _af: PhantomData,
+            _m: PhantomData,
+        }
+    }
+
+    pub(crate) fn get_records_for_prefix(
+        &self,
+        prefix: PrefixId<AF>,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        bmin: &RoaringBitmap,
+    ) -> Option<Vec<PublicRecord<M>>> {
+        let mut prefix_set = self.store.get_root_prefix_set(prefix.get_len());
+        let mut level: u8 = 0;
+        let backoff = Backoff::new();
+
+        loop {
+            // The index of the prefix in this array (at this len and
+            // level) is calculated by performing the hash function
+            // over the prefix.
+
+            // HASHING FUNCTION
+            let index = Self::hash_prefix_id(prefix, level);
+
+            if let Some(stored_prefix) = prefix_set.0.get(index) {
+                if prefix == stored_prefix.get_prefix_id() {
+                    if log_enabled!(log::Level::Trace) {
+                        trace!(
+                            "found requested prefix {} ({:?})",
+                            Prefix::from(prefix),
+                            prefix
+                        );
+                    }
+
+                    return stored_prefix.record_map.get_filtered_records(
+                        mui,
+                        include_withdrawn,
+                        bmin,
+                    );
+                };
+
+                // Advance to the next level.
+                prefix_set = &stored_prefix.next_bucket;
+                level += 1;
+                backoff.spin();
+                continue;
+            }
+
+            trace!("no prefix found for {:?}", prefix);
+            return None;
+        }
+    }
+
+    pub(crate) fn upsert_prefix(
+        &self,
+        prefix: PrefixId<AF>,
+        record: PublicRecord<M>,
+        update_path_selections: Option<M::TBI>,
+        guard: &Guard,
+    ) -> Result<(UpsertReport, Option<MultiMapValue<M>>), PrefixStoreError>
+    {
+        let mut prefix_is_new = true;
+        let mut mui_is_new = true;
+
+        let (mui_count, cas_count) =
+            match self.non_recursive_retrieve_prefix_mut(prefix) {
+                // There's no StoredPrefix at this location yet. Create a new
+                // PrefixRecord and try to store it in the empty slot.
+                (stored_prefix, false) => {
+                    if log_enabled!(log::Level::Debug) {
+                        debug!(
+                            "{} store: Create new prefix record",
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread")
+                        );
+                    }
+
+                    let (mui_count, retry_count) =
+                        stored_prefix.record_map.upsert_record(record)?;
+
+                    // See if someone beat us to creating the record.
+                    if mui_count.is_some() {
+                        mui_is_new = false;
+                        prefix_is_new = false;
+                    }
+
+                    (mui_count, retry_count)
+                }
+                // There already is a StoredPrefix with a record at this
+                // location.
+                (stored_prefix, true) => {
+                    if log_enabled!(log::Level::Debug) {
+                        debug!(
+                        "{} store: Found existing prefix record for {}/{}",
+                        std::thread::current()
+                            .name()
+                            .unwrap_or("unnamed-thread"),
+                        prefix.get_net(),
+                        prefix.get_len()
+                    );
+                    }
+                    prefix_is_new = false;
+
+                    // Update the already existing record_map with our
+                    // caller's record.
+                    stored_prefix.set_ps_outdated(guard)?;
+
+                    let (mui_count, retry_count) =
+                        stored_prefix.record_map.upsert_record(record)?;
+                    mui_is_new = mui_count.is_none();
+
+                    if let Some(tbi) = update_path_selections {
+                        stored_prefix
+                            .calculate_and_store_best_backup(&tbi, guard)?;
+                    }
+
+                    (mui_count, retry_count)
+                }
+            };
+
+        let count = mui_count.as_ref().map(|m| m.1).unwrap_or(1);
+        Ok((
+            UpsertReport {
+                prefix_new: prefix_is_new,
+                cas_count,
+                mui_new: mui_is_new,
+                mui_count: count,
+            },
+            mui_count.map(|m| m.0),
+        ))
+    }
+    // This function is used by the upsert_prefix function above.
+    //
+    // We're using a Chained Hash Table and this function returns one of:
+    // - a StoredPrefix that already exists for this search_prefix_id
+    // - the Last StoredPrefix in the chain.
+    // - an error, if no StoredPrefix whatsoever can be found in the store.
+    //
+    // The error condition really shouldn't happen, because that basically
+    // means the root node for that particular prefix length doesn't exist.
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn non_recursive_retrieve_prefix_mut(
+        &self,
+        search_prefix_id: PrefixId<AF>,
+    ) -> (&StoredPrefix<AF, M>, bool) {
+        trace!("non_recursive_retrieve_prefix_mut_with_guard");
+        let mut prefix_set =
+            self.store.get_root_prefix_set(search_prefix_id.get_len());
+        let mut level: u8 = 0;
+
+        trace!("root prefix_set {:?}", prefix_set);
+        loop {
+            // HASHING FUNCTION
+            let index = Self::hash_prefix_id(search_prefix_id, level);
+
+            // probe the slot with the index that's the result of the hashing.
+            let stored_prefix = match prefix_set.0.get(index) {
+                Some(p) => {
+                    trace!("prefix set found.");
+                    (p, true)
+                }
+                None => {
+                    // We're at the end of the chain and haven't found our
+                    // search_prefix_id anywhere. Return the end-of-the-chain
+                    // StoredPrefix, so the caller can attach a new one.
+                    trace!(
+                        "no record. returning last found record in level
+                        {}, with index {}.",
+                        level,
+                        index
+                    );
+                    let index = Self::hash_prefix_id(search_prefix_id, level);
+                    trace!("calculate next index {}", index);
+                    let var_name = (
+                        prefix_set
+                            .0
+                            .get_or_init(index, || {
+                                StoredPrefix::new::<PB>(
+                                    PrefixId::new(
+                                        search_prefix_id.get_net(),
+                                        search_prefix_id.get_len(),
+                                    ),
+                                    level,
+                                )
+                            })
+                            .0,
+                        false,
+                    );
+                    var_name
+                }
+            };
+
+            if search_prefix_id == stored_prefix.0.prefix {
+                // GOTCHA!
+                // Our search-prefix is stored here, so we're returning
+                // it, so its PrefixRecord can be updated by the caller.
+                if log_enabled!(log::Level::Trace) {
+                    trace!(
+                        "found requested prefix {} ({:?})",
+                        Prefix::from(search_prefix_id),
+                        search_prefix_id
+                    );
+                }
+                return stored_prefix;
+            } else {
+                // A Collision. Follow the chain.
+                level += 1;
+                prefix_set = &stored_prefix.0.next_bucket;
+                continue;
+            }
+        }
+    }
+
+    // This function is used by the match_prefix, and [more|less]_specifics
+    // public methods on the TreeBitMap (indirectly).
+    #[allow(clippy::type_complexity)]
+    pub fn non_recursive_retrieve_prefix(
+        &self,
+        id: PrefixId<AF>,
+    ) -> (
+        Option<&StoredPrefix<AF, M>>,
+        Option<(
+            PrefixId<AF>,
+            u8,
+            &PrefixSet<AF, M>,
+            [Option<(&PrefixSet<AF, M>, usize)>; 32],
+            usize,
+        )>,
+    ) {
+        let mut prefix_set = self.store.get_root_prefix_set(id.get_len());
+        let mut parents = [None; 32];
+        let mut level: u8 = 0;
+        let backoff = Backoff::new();
+
+        loop {
+            // The index of the prefix in this array (at this len and
+            // level) is calculated by performing the hash function
+            // over the prefix.
+
+            // HASHING FUNCTION
+            let index = Self::hash_prefix_id(id, level);
+
+            if let Some(stored_prefix) = prefix_set.0.get(index) {
+                if id == stored_prefix.get_prefix_id() {
+                    if log_enabled!(log::Level::Trace) {
+                        trace!(
+                            "found requested prefix {} ({:?})",
+                            Prefix::from(id),
+                            id
+                        );
+                    }
+                    parents[level as usize] = Some((prefix_set, index));
+                    return (
+                        Some(stored_prefix),
+                        Some((id, level, prefix_set, parents, index)),
+                    );
+                };
+
+                // Advance to the next level.
+                prefix_set = &stored_prefix.next_bucket;
+                level += 1;
+                backoff.spin();
+                continue;
+            }
+
+            trace!("no prefix found for {:?}", id);
+            parents[level as usize] = Some((prefix_set, index));
+            return (None, Some((id, level, prefix_set, parents, index)));
+        }
+    }
+
+    pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
+        // And, this is all of our hashing function.
+        let last_level = if level > 0 {
+            <PB>::get_bits_for_len(id.get_len(), level - 1)
+        } else {
+            0
+        };
+        let this_level = <PB>::get_bits_for_len(id.get_len(), level);
+        // trace!(
+        //     "bits division {}; no of bits {}",
+        //     this_level,
+        //     this_level - last_level
+        // );
+        // trace!(
+        //     "calculated index ({} << {}) >> {}",
+        //     id.get_net(),
+        //     last_level,
+        //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
+        // );
+        // HASHING FUNCTION
+        ((id.get_net() << last_level)
+            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
+            .dangerously_truncate_to_u32() as usize
+    }
+}
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
new file mode 100644
index 00000000..1b1861e8
--- /dev/null
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -0,0 +1,244 @@
+use log::trace;
+use roaring::RoaringBitmap;
+
+use crate::{
+    local_array::{
+        bit_span::BitSpan,
+        in_memory::{node::SizedStrideRef, tree::TreeBitMap},
+    },
+    prelude::multi::{NodeBuckets, PrefixId},
+    rib::iterators::{SizedNodeMoreSpecificIter, SizedPrefixIter},
+    AddressFamily,
+};
+
+type Type<AF: AddressFamily> = SizedNodeMoreSpecificIter<AF>;
+
+pub(crate) struct MoreSpecificPrefixIter<
+    'a,
+    AF: AddressFamily,
+    // M: Meta,
+    NB: NodeBuckets<AF>,
+    // PB: PrefixBuckets<AF, M>,
+> {
+    store: &'a TreeBitMap<AF, NB>,
+    cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
+    cur_pfx_iter: SizedPrefixIter<AF>,
+    // start_bit_span: BitSpan,
+    // skip_self: bool,
+    parent_and_position: Vec<Type<AF>>,
+    // If specified, we're only iterating over records for this mui.
+    mui: Option<u32>,
+    // This is the tree-wide index of withdrawn muis, used to rewrite the
+    // statuses of these records, or filter them out.
+    global_withdrawn_bmin: &'a RoaringBitmap,
+    // Whether we should filter out the withdrawn records in the search result
+    include_withdrawn: bool,
+}
+
+impl<
+        'a,
+        AF: AddressFamily + 'a,
+        // M: Meta,
+        NB: NodeBuckets<AF>,
+        // PB: PrefixBuckets<AF, M>,
+    > Iterator for MoreSpecificPrefixIter<'a, AF, NB>
+{
+    type Item = PrefixId<AF>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        trace!("MoreSpecificsPrefixIter");
+
+        loop {
+            // first drain the current prefix iterator until empty.
+            let next_pfx = self.cur_pfx_iter.next();
+
+            if next_pfx.is_some() {
+                return next_pfx;
+                // If we have a mui, we have to deal slightly different with
+                // the records: There can only be one record for a (prefix,
+                // mui) combination, and the record may be filtered out by the
+                // global status of the mui, or its local status. In that case
+                // we don't return here (because that would result in a Prefix
+                // with an empty record vec).
+                // next_pfx
+                // if let Some(mui) = self.mui {
+                //     if let Some(p) = self
+                //         .store
+                //         .non_recursive_retrieve_prefix(
+                //             next_pfx.unwrap_or_else(|| {
+                //                 panic!(
+                //                 "BOOM! More-specific prefix {:?} disappeared \
+                //                 from the store",
+                //                 next_pfx
+                //             )
+                //             }),
+                //         )
+                //         .0
+                //     {
+                //         // We may either have to rewrite the local status with
+                //         // the provided global status OR we may have to omit
+                //         // all of the records with either global of local
+                //         // withdrawn status.
+                //         if self.include_withdrawn {
+                //             if let Some(rec) = p
+                //                 .record_map
+                //                 .get_record_for_mui_with_rewritten_status(
+                //                     mui,
+                //                     self.global_withdrawn_bmin,
+                //                     RouteStatus::Withdrawn,
+                //                 )
+                //             {
+                //                 return Some((p.prefix, vec![rec]));
+                //             }
+                //         } else if let Some(rec) = p
+                //             .record_map
+                //             .get_record_for_mui(mui, self.include_withdrawn)
+                //         {
+                //             return Some((p.prefix, vec![rec]));
+                //         }
+                //     };
+                // } else {
+                //     return self
+                //         .store
+                //         .non_recursive_retrieve_prefix(
+                //             next_pfx.unwrap_or_else(|| {
+                //                 panic!(
+                //                 "BOOM! More-specific prefix {:?} disappeared \
+                //                 from the store",
+                //                 next_pfx
+                //             )
+                //             }),
+                //             // self.guard,
+                //         )
+                //         .0
+                //         .map(|p| {
+                //             // Just like the mui specific records, we may have
+                //             // to either rewrite the local status (if the user
+                //             // wants the withdrawn records) or omit them.
+                //             if self.include_withdrawn {
+                //                 (
+                //                     p.prefix,
+                //                     p.record_map
+                //                         .as_records_with_rewritten_status(
+                //                             self.global_withdrawn_bmin,
+                //                             RouteStatus::Withdrawn,
+                //                         ),
+                //                 )
+                //             } else {
+                //                 (
+                //                     p.prefix,
+                //                     p.record_map
+                //                         .as_active_records_not_in_bmin(
+                //                             self.global_withdrawn_bmin,
+                //                         ),
+                //                 )
+                //             }
+                //         });
+                // }
+            }
+
+            // Our current prefix iterator for this node is done, look for
+            // the next pfx iterator of the next child node in the current
+            // ptr iterator.
+            trace!("resume ptr iterator {:?}", self.cur_ptr_iter);
+
+            let mut next_ptr = self.cur_ptr_iter.next();
+
+            // Our current ptr iterator is also done, maybe we have a parent
+            if next_ptr.is_none() {
+                trace!("try for parent");
+                if let Some(cur_ptr_iter) = self.parent_and_position.pop() {
+                    trace!("continue with parent");
+                    self.cur_ptr_iter = cur_ptr_iter;
+                    next_ptr = self.cur_ptr_iter.next();
+                } else {
+                    trace!("no more parents");
+                    return None;
+                }
+            }
+
+            if let Some(next_ptr) = next_ptr {
+                let node = if self.mui.is_none() {
+                    trace!("let's retriev node {}", next_ptr);
+                    self.store.retrieve_node(next_ptr)
+                } else {
+                    self.store
+                        .retrieve_node_for_mui(next_ptr, self.mui.unwrap())
+                };
+
+                match node {
+                    Some(SizedStrideRef::Stride3(next_node)) => {
+                        // copy the current iterator into the parent vec and create
+                        // a new ptr iterator for this node
+                        self.parent_and_position.push(self.cur_ptr_iter);
+                        let ptr_iter = next_node.more_specific_ptr_iter(
+                            next_ptr,
+                            BitSpan { bits: 0, len: 0 },
+                        );
+                        self.cur_ptr_iter = ptr_iter.wrap();
+
+                        // trace!(
+                        //     "next stride new iterator stride 3 {:?} start \
+                        // bit_span {:?}",
+                        //     self.cur_ptr_iter,
+                        //     self.start_bit_span
+                        // );
+                        self.cur_pfx_iter = next_node
+                            .more_specific_pfx_iter(
+                                next_ptr,
+                                BitSpan::new(0, 0),
+                            )
+                            .wrap();
+                    }
+                    Some(SizedStrideRef::Stride4(next_node)) => {
+                        // create new ptr iterator for this node.
+                        self.parent_and_position.push(self.cur_ptr_iter);
+                        let ptr_iter = next_node.more_specific_ptr_iter(
+                            next_ptr,
+                            BitSpan { bits: 0, len: 0 },
+                        );
+                        self.cur_ptr_iter = ptr_iter.wrap();
+
+                        trace!(
+                            "next stride new iterator stride 4 {:?} start \
+                        bit_span 0 0",
+                            self.cur_ptr_iter,
+                        );
+                        self.cur_pfx_iter = next_node
+                            .more_specific_pfx_iter(
+                                next_ptr,
+                                BitSpan::new(0, 0),
+                            )
+                            .wrap();
+                    }
+                    Some(SizedStrideRef::Stride5(next_node)) => {
+                        // create new ptr iterator for this node.
+                        self.parent_and_position.push(self.cur_ptr_iter);
+                        let ptr_iter = next_node.more_specific_ptr_iter(
+                            next_ptr,
+                            BitSpan { bits: 0, len: 0 },
+                        );
+                        self.cur_ptr_iter = ptr_iter.wrap();
+
+                        // trace!(
+                        //     "next stride new iterator stride 5 {:?} start \
+                        // bit_span {:?}",
+                        //     self.cur_ptr_iter,
+                        //     self.start_bit_span
+                        // );
+                        self.cur_pfx_iter = next_node
+                            .more_specific_pfx_iter(
+                                next_ptr,
+                                BitSpan::new(0, 0),
+                            )
+                            .wrap();
+                    }
+                    None => {
+                        println!("no node here.");
+                        return None;
+                    }
+                };
+            }
+        }
+    }
+}
diff --git a/src/local_array/prefix_cht/iterators_cp.rs b/src/local_array/prefix_cht/iterators_cp.rs
new file mode 100644
index 00000000..b1013e68
--- /dev/null
+++ b/src/local_array/prefix_cht/iterators_cp.rs
@@ -0,0 +1,879 @@
+// ----------- Store Iterators ----------------------------------------------
+//
+// This file hosts the iterators for the Rib and implementations for the
+// methods that start'em. There are 3 Iterators:
+//
+// 1. an iterator `PrefixIter` that iterates over ALL of the prefix buckets of
+// the CHT backing the TreeBitMap.
+//
+// 2. a MoreSpecificsIterator that starts from a prefix in the prefix buckets
+// for that particular prefix length, but uses the node in the TreeBitMap to
+// find its more specifics.
+//
+// 3. a LessSpecificIterator, that just reduces the prefix size bit-by-bit and
+// looks in the prefix buckets for the diminuishing prefix.
+//
+// The Iterators that start from the root node of the TreeBitMap (which
+// is the only option for the single-threaded TreeBitMap) live in the
+// deprecated_node.rs file. They theoretically should be slower and cause more
+// contention, since every lookup has to go through the levels near the root
+// in the TreeBitMap.
+
+use super::atomic_types::{NodeBuckets, PrefixBuckets, PrefixSet};
+use super::node::{SizedStrideRef, StrideNodeId};
+use super::tree::{Stride3, Stride4, Stride5, TreeBitMap};
+use crate::local_array::types::RouteStatus;
+use crate::PublicRecord;
+use crate::{
+    af::AddressFamily,
+    local_array::{
+        bit_span::BitSpan,
+        in_memory::node::{
+            NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter,
+        },
+        types::PrefixId,
+    },
+    prefix_record::Meta,
+};
+
+use crossbeam_epoch::Guard;
+use inetnum::addr::Prefix;
+use log::trace;
+use roaring::RoaringBitmap;
+
+// ----------- PrefixIter ---------------------------------------------------
+
+// Iterator over all the prefixes in the storage. This Iterator does *not* use
+// the tree, it iterates over all the length arrays in the CustomAllocStorage.
+
+pub(crate) struct PrefixIter<
+    'a,
+    AF: AddressFamily + 'a,
+    M: Meta + 'a,
+    PB: PrefixBuckets<AF, M>,
+> {
+    prefixes: &'a PB,
+    cur_len: u8,
+    cur_bucket: &'a PrefixSet<AF, M>,
+    cur_level: u8,
+    // level depth of IPv4 as defined in rotonda-macros/maps.rs Option(parent,
+    // cursor position at the parent) 32 is the max number of levels in IPv6,
+    // which is the max number of of both IPv4 and IPv6.
+    parents: [Option<(&'a PrefixSet<AF, M>, usize)>; 32],
+    cursor: usize,
+}
+
+impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
+    Iterator for PrefixIter<'a, AF, M, PB>
+{
+    type Item = (inetnum::addr::Prefix, Vec<PublicRecord<M>>);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        trace!(
+            "starting next loop for level {} cursor {} (len {})",
+            self.cur_level,
+            self.cursor,
+            self.cur_len
+        );
+
+        loop {
+            if self.cur_len > AF::BITS {
+                // This is the end, my friend
+                trace!("reached max length {}, returning None", self.cur_len);
+                return None;
+            }
+
+            if PB::get_bits_for_len(self.cur_len, self.cur_level) == 0 {
+                // END OF THE LENGTH
+
+                // This length is done too, go to the next length
+                // trace!("next length {}", self.cur_len + 1);
+                self.cur_len += 1;
+
+                // a new length, a new life reset the level depth and cursor,
+                // but also empty all the parents
+                self.cur_level = 0;
+                self.cursor = 0;
+                self.parents = [None; 32];
+
+                // let's continue, get the prefixes for the next length
+                self.cur_bucket =
+                    self.prefixes.get_root_prefix_set(self.cur_len);
+                continue;
+            }
+            let bucket_size = 1_usize
+                << (if self.cur_level > 0 {
+                    PB::get_bits_for_len(self.cur_len, self.cur_level)
+                        - PB::get_bits_for_len(
+                            self.cur_len,
+                            self.cur_level - 1,
+                        )
+                } else {
+                    PB::get_bits_for_len(self.cur_len, self.cur_level)
+                });
+
+            if self.cursor >= bucket_size {
+                if self.cur_level == 0 {
+                    // END OF THE LENGTH
+
+                    // This length is done too, go to the next length
+                    // trace!("next length {}", self.cur_len);
+                    self.cur_len += 1;
+
+                    // a new length, a new life reset the level depth and
+                    // cursor, but also empty all the parents
+                    self.cur_level = 0;
+                    self.cursor = 0;
+                    self.parents = [None; 32];
+
+                    if self.cur_len > AF::BITS {
+                        // This is the end, my friend
+                        return None;
+                    }
+
+                    // let's continue, get the prefixes for the next length
+                    self.cur_bucket =
+                        self.prefixes.get_root_prefix_set(self.cur_len);
+                } else {
+                    // END OF THIS BUCKET GO BACK UP ONE LEVEL
+
+                    // The level is done, but the length isn't Go back up one
+                    // level and continue
+                    match self.parents[self.cur_level as usize] {
+                        Some(parent) => {
+                            // There is a parent, go back up. Since we're
+                            // doing depth-first we have to check if there's a
+                            // prefix directly at the parent and return that.
+                            self.cur_level -= 1;
+
+                            // move the current bucket to the parent and move
+                            // the cursor position where we left off. The next
+                            // run of the loop will read it.
+                            self.cur_bucket = parent.0;
+                            self.cursor = parent.1 + 1;
+
+                            continue;
+                        }
+                        None => {
+                            trace!(
+                                "c {} lvl {} len {}",
+                                self.cursor,
+                                self.cur_level,
+                                self.cur_len
+                            );
+                            panic!(
+                                "Where do we belong? Where do we come from?"
+                            );
+                        }
+                    };
+                }
+            };
+
+            // we're somewhere in the PrefixSet iteration, read the next
+            // StoredPrefix. We are doing depth-first iteration, so we check
+            // for a child first and descend into that if it exists.
+
+            if let Some(s_pfx) = self.cur_bucket.get_by_index(self.cursor) {
+                // DEPTH FIRST ITERATION
+                match s_pfx.get_next_bucket() {
+                    Some(bucket) => {
+                        // DESCEND ONe LEVEL There's a child here, descend into
+                        // it, but... trace!("C. got next bucket {:?}", bucket);
+
+                        // save our parent and cursor position first, and then..
+                        self.parents[(self.cur_level + 1) as usize] =
+                            Some((self.cur_bucket, self.cursor));
+
+                        // move to the next bucket,
+                        self.cur_bucket = bucket;
+
+                        // increment the level and reset the cursor.
+                        self.cur_level += 1;
+                        self.cursor = 0;
+
+                        // If there's a child here there MUST be a prefix here,
+                        // as well.
+                        // if let Some(meta) =
+                        //     s_pfx.get_stored_prefix(self.guard).map(|p| {
+                        //         if log_enabled!(log::Level::Trace) {
+                        //             // There's a prefix here, that's the next one
+                        //             trace!("D. found prefix {:?}", p.prefix);
+                        //         }
+                        //         p.record_map.as_records()
+                        //     })
+                        // {
+                        return Some((
+                            s_pfx.get_prefix_id().into(),
+                            s_pfx.record_map.as_records(),
+                        ));
+                        // } else {
+                        //     panic!(
+                        //         "No prefix here, but there's a child here?"
+                        //     );
+                        // }
+                    }
+                    None => {
+                        // No reference to another PrefixSet, all that's left, is
+                        // checking for a prefix at the current cursor position.
+                        // if let Some(meta) =
+                        //     s_pfx.get_stored_prefix(self.guard).map(|p| {
+                        //         // There's a prefix here, that's the next one
+                        //         if log_enabled!(log::Level::Debug) {
+                        //             debug!("E. found prefix {:?}", p.prefix);
+                        //         }
+                        //         p.record_map.as_records()
+                        //     })
+                        // {
+                        self.cursor += 1;
+                        return Some((
+                            s_pfx.get_prefix_id().into(),
+                            s_pfx.record_map.as_records(),
+                        ));
+                        // }
+                    }
+                };
+            } else {
+                self.cursor += 1;
+            }
+        }
+    }
+}
+
+// ----------- Sized Wrappers -----------------------------------------------
+
+// These are enums to abstract over the Stride Size of the iterators. Each
+// iterator in here need to go over iterators that have different underlying
+// stride sizes. To facilitate this these wrapper enums exist.
+
+#[derive(Copy, Clone, Debug)]
+pub(crate) enum SizedNodeMoreSpecificIter<AF: AddressFamily> {
+    Stride3(NodeMoreSpecificChildIter<AF, Stride3>),
+    Stride4(NodeMoreSpecificChildIter<AF, Stride4>),
+    Stride5(NodeMoreSpecificChildIter<AF, Stride5>),
+}
+
+impl<AF: AddressFamily> SizedNodeMoreSpecificIter<AF> {
+    fn next(&mut self) -> Option<StrideNodeId<AF>> {
+        match self {
+            SizedNodeMoreSpecificIter::Stride3(iter) => iter.next(),
+            SizedNodeMoreSpecificIter::Stride4(iter) => iter.next(),
+            SizedNodeMoreSpecificIter::Stride5(iter) => iter.next(),
+        }
+    }
+}
+
+pub(crate) enum SizedPrefixIter<AF: AddressFamily> {
+    Stride3(NodeMoreSpecificsPrefixIter<AF, Stride3>),
+    Stride4(NodeMoreSpecificsPrefixIter<AF, Stride4>),
+    Stride5(NodeMoreSpecificsPrefixIter<AF, Stride5>),
+}
+
+impl<AF: AddressFamily> SizedPrefixIter<AF> {
+    fn next(&mut self) -> Option<PrefixId<AF>> {
+        match self {
+            SizedPrefixIter::Stride3(iter) => iter.next(),
+            SizedPrefixIter::Stride4(iter) => iter.next(),
+            SizedPrefixIter::Stride5(iter) => iter.next(),
+        }
+    }
+}
+
+// ----------- MoreSpecificPrefixIter ------------------------------------
+
+// A iterator over all the more-specifics for a given prefix.
+//
+// This iterator is somewhat different from the other *PrefixIterator types,
+// since it uses the Nodes to select the more specifics. An Iterator that
+// would only use the Prefixes in the store could exist, but iterating over
+// those in search of more specifics would be way more expensive.
+
+// The first iterator it goes over should have a bit_span that is the
+// difference between the requested prefix and the node that hosts that
+// prefix. See the method initializing this iterator (`get_node_for_id_prefix`
+// takes care of it in there). The consecutive iterators will all have a
+// bit_span of { bits: 0, len: 0 }. Yes, we could also use the PrefixIter
+// there (it iterates over all prefixes of a node), but then we would have to
+// deal with two different types of iterators. Note that the iterator is
+// neither depth- or breadth-first and the results are essentially unordered.
+
+pub(crate) struct MoreSpecificPrefixIter<
+    'a,
+    AF: AddressFamily,
+    M: Meta,
+    NB: NodeBuckets<AF>,
+    PB: PrefixBuckets<AF, M>,
+> {
+    store: &'a TreeBitMap<AF, M, NB, PB>,
+    cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
+    cur_pfx_iter: SizedPrefixIter<AF>,
+    // start_bit_span: BitSpan,
+    // skip_self: bool,
+    parent_and_position: Vec<SizedNodeMoreSpecificIter<AF>>,
+    // If specified, we're only iterating over records for this mui.
+    mui: Option<u32>,
+    // This is the tree-wide index of withdrawn muis, used to rewrite the
+    // statuses of these records, or filter them out.
+    global_withdrawn_bmin: &'a RoaringBitmap,
+    // Whether we should filter out the withdrawn records in the search result
+    include_withdrawn: bool,
+}
+
+impl<
+        'a,
+        AF: AddressFamily + 'a,
+        M: Meta,
+        NB: NodeBuckets<AF>,
+        PB: PrefixBuckets<AF, M>,
+    > Iterator for MoreSpecificPrefixIter<'a, AF, M, NB, PB>
+{
+    type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        trace!("MoreSpecificsPrefixIter");
+
+        loop {
+            // first drain the current prefix iterator until empty.
+            let next_pfx = self.cur_pfx_iter.next();
+
+            if next_pfx.is_some() {
+                // If we have a mui, we have to deal slightly different with
+                // the records: There can only be one record for a (prefix,
+                // mui) combination, and the record may be filtered out by the
+                // global status of the mui, or its local status. In that case
+                // we don't return here (because that would result in a Prefix
+                // with an empty record vec).
+                if let Some(mui) = self.mui {
+                    if let Some(p) = self
+                        .store
+                        .non_recursive_retrieve_prefix(
+                            next_pfx.unwrap_or_else(|| {
+                                panic!(
+                                "BOOM! More-specific prefix {:?} disappeared \
+                                from the store",
+                                next_pfx
+                            )
+                            }),
+                            // self.guard,
+                        )
+                        .0
+                    {
+                        // We may either have to rewrite the local status with
+                        // the provided global status OR we may have to omit
+                        // all of the records with either global of local
+                        // withdrawn status.
+                        if self.include_withdrawn {
+                            if let Some(rec) = p
+                                .record_map
+                                .get_record_for_mui_with_rewritten_status(
+                                    mui,
+                                    self.global_withdrawn_bmin,
+                                    RouteStatus::Withdrawn,
+                                )
+                            {
+                                return Some((p.prefix, vec![rec]));
+                            }
+                        } else if let Some(rec) = p
+                            .record_map
+                            .get_record_for_mui(mui, self.include_withdrawn)
+                        {
+                            return Some((p.prefix, vec![rec]));
+                        }
+                    };
+                } else {
+                    return self
+                        .store
+                        .non_recursive_retrieve_prefix(
+                            next_pfx.unwrap_or_else(|| {
+                                panic!(
+                                "BOOM! More-specific prefix {:?} disappeared \
+                                from the store",
+                                next_pfx
+                            )
+                            }),
+                            // self.guard,
+                        )
+                        .0
+                        .map(|p| {
+                            // Just like the mui specific records, we may have
+                            // to either rewrite the local status (if the user
+                            // wants the withdrawn records) or omit them.
+                            if self.include_withdrawn {
+                                (
+                                    p.prefix,
+                                    p.record_map
+                                        .as_records_with_rewritten_status(
+                                            self.global_withdrawn_bmin,
+                                            RouteStatus::Withdrawn,
+                                        ),
+                                )
+                            } else {
+                                (
+                                    p.prefix,
+                                    p.record_map
+                                        .as_active_records_not_in_bmin(
+                                            self.global_withdrawn_bmin,
+                                        ),
+                                )
+                            }
+                        });
+                }
+            }
+
+            // Our current prefix iterator for this node is done, look for
+            // the next pfx iterator of the next child node in the current
+            // ptr iterator.
+            trace!("resume ptr iterator {:?}", self.cur_ptr_iter);
+
+            let mut next_ptr = self.cur_ptr_iter.next();
+
+            // Our current ptr iterator is also done, maybe we have a parent
+            if next_ptr.is_none() {
+                trace!("try for parent");
+                if let Some(cur_ptr_iter) = self.parent_and_position.pop() {
+                    trace!("continue with parent");
+                    self.cur_ptr_iter = cur_ptr_iter;
+                    next_ptr = self.cur_ptr_iter.next();
+                } else {
+                    trace!("no more parents");
+                    return None;
+                }
+            }
+
+            if let Some(next_ptr) = next_ptr {
+                let node = if self.mui.is_none() {
+                    trace!("let's retriev node {}", next_ptr);
+                    self.store.retrieve_node(next_ptr)
+                } else {
+                    self.store
+                        .retrieve_node_for_mui(next_ptr, self.mui.unwrap())
+                };
+
+                match node {
+                    Some(SizedStrideRef::Stride3(next_node)) => {
+                        // copy the current iterator into the parent vec and create
+                        // a new ptr iterator for this node
+                        self.parent_and_position.push(self.cur_ptr_iter);
+                        let ptr_iter = next_node.more_specific_ptr_iter(
+                            next_ptr,
+                            BitSpan { bits: 0, len: 0 },
+                        );
+                        self.cur_ptr_iter = ptr_iter.wrap();
+
+                        // trace!(
+                        //     "next stride new iterator stride 3 {:?} start \
+                        // bit_span {:?}",
+                        //     self.cur_ptr_iter,
+                        //     self.start_bit_span
+                        // );
+                        self.cur_pfx_iter = next_node
+                            .more_specific_pfx_iter(
+                                next_ptr,
+                                BitSpan::new(0, 0),
+                            )
+                            .wrap();
+                    }
+                    Some(SizedStrideRef::Stride4(next_node)) => {
+                        // create new ptr iterator for this node.
+                        self.parent_and_position.push(self.cur_ptr_iter);
+                        let ptr_iter = next_node.more_specific_ptr_iter(
+                            next_ptr,
+                            BitSpan { bits: 0, len: 0 },
+                        );
+                        self.cur_ptr_iter = ptr_iter.wrap();
+
+                        trace!(
+                            "next stride new iterator stride 4 {:?} start \
+                        bit_span 0 0",
+                            self.cur_ptr_iter,
+                        );
+                        self.cur_pfx_iter = next_node
+                            .more_specific_pfx_iter(
+                                next_ptr,
+                                BitSpan::new(0, 0),
+                            )
+                            .wrap();
+                    }
+                    Some(SizedStrideRef::Stride5(next_node)) => {
+                        // create new ptr iterator for this node.
+                        self.parent_and_position.push(self.cur_ptr_iter);
+                        let ptr_iter = next_node.more_specific_ptr_iter(
+                            next_ptr,
+                            BitSpan { bits: 0, len: 0 },
+                        );
+                        self.cur_ptr_iter = ptr_iter.wrap();
+
+                        // trace!(
+                        //     "next stride new iterator stride 5 {:?} start \
+                        // bit_span {:?}",
+                        //     self.cur_ptr_iter,
+                        //     self.start_bit_span
+                        // );
+                        self.cur_pfx_iter = next_node
+                            .more_specific_pfx_iter(
+                                next_ptr,
+                                BitSpan::new(0, 0),
+                            )
+                            .wrap();
+                    }
+                    None => {
+                        println!("no node here.");
+                        return None;
+                    }
+                };
+            }
+        }
+    }
+}
+
+// ----------- LessSpecificPrefixIter ---------------------------------------
+
+// This iterator iterates over all the less-specifics for a given prefix. It
+// does *not* use the tree, it goes directly into the CustomAllocStorage and
+// retrieves the less-specifics by going from len to len, searching for the
+// prefixes.
+
+pub(crate) struct LessSpecificPrefixIter<
+    'a,
+    AF: AddressFamily + 'a,
+    M: Meta + 'a,
+    PB: PrefixBuckets<AF, M>,
+> {
+    prefixes: &'a PB,
+    cur_len: u8,
+    cur_bucket: &'a PrefixSet<AF, M>,
+    cur_level: u8,
+    cur_prefix_id: PrefixId<AF>,
+    mui: Option<u32>,
+    // Whether to include withdrawn records, both globally and local.
+    include_withdrawn: bool,
+    // This is the tree-wide index of withdrawn muis, used to filter out the
+    // records for those.
+    global_withdrawn_bmin: &'a RoaringBitmap,
+}
+
+impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
+    Iterator for LessSpecificPrefixIter<'a, AF, M, PB>
+{
+    type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
+
+    // This iterator moves down all prefix lengths, starting with the length
+    // of the (search prefix - 1), looking for shorter prefixes, where the
+    // its bits are the same as the bits of the search prefix.
+    fn next(&mut self) -> Option<Self::Item> {
+        trace!("search next less-specific for {:?}", self.cur_prefix_id);
+
+        loop {
+            if self.cur_len == 0 {
+                // This is the end, my friend
+                trace!("reached min length {}, returning None", self.cur_len);
+                return None;
+            }
+
+            // shave a bit of the current prefix.
+            trace!(
+                "truncate to len {} (level {})",
+                self.cur_len,
+                self.cur_level
+            );
+            self.cur_prefix_id = PrefixId::new(
+                self.cur_prefix_id.get_net().truncate_to_len(self.cur_len),
+                self.cur_len,
+            );
+
+            let last_level = if self.cur_level > 0 {
+                PB::get_bits_for_len(self.cur_len, self.cur_level - 1)
+            } else {
+                0
+            };
+
+            let this_level =
+                PB::get_bits_for_len(self.cur_len, self.cur_level);
+
+            // NOT THE HASHING FUNCTION
+            let index = ((self.cur_prefix_id.get_net() << last_level)
+                >> ((AF::BITS - (this_level - last_level)) % AF::BITS))
+                .dangerously_truncate_to_u32()
+                as usize;
+
+            if this_level == 0 {
+                // END OF THE LENGTH
+                // This length is done too, go to the next length
+                // trace!("next length {}", self.cur_len + 1);
+                self.cur_len -= 1;
+
+                // a new length, a new life
+                // reset the level depth and cursor,
+                // but also empty all the parents
+                self.cur_level = 0;
+                // self.parents = [None; 26];
+
+                // let's continue, get the prefixes for the next length
+                self.cur_bucket =
+                    self.prefixes.get_root_prefix_set(self.cur_len);
+                continue;
+            }
+
+            // LEVEL DEPTH ITERATION
+            if let Some(stored_prefix) = self.cur_bucket.get_by_index(index) {
+                // if let Some(stored_prefix) =
+                // s_pfx.get_stored_prefix(self.guard)
+                // {
+                trace!("get_record {:?}", stored_prefix.record_map);
+                let pfx_rec = if let Some(mui) = self.mui {
+                    // We don't have to check for the appearance of We may
+                    // either have to rewrite the local status with the
+                    // provided global status OR we may have to omit all of
+                    // the records with either global of local withdrawn
+                    // status.
+                    if self.include_withdrawn {
+                        stored_prefix
+                            .record_map
+                            .get_record_for_mui_with_rewritten_status(
+                                mui,
+                                self.global_withdrawn_bmin,
+                                RouteStatus::Withdrawn,
+                            )
+                            .into_iter()
+                            .collect()
+                    } else {
+                        stored_prefix
+                            .record_map
+                            .get_record_for_mui(mui, self.include_withdrawn)
+                            .into_iter()
+                            .collect()
+                    }
+                } else {
+                    // Just like the mui specific records, we may have
+                    // to either rewrite the local status (if the user
+                    // wants the withdrawn records) or omit them.
+                    if self.include_withdrawn {
+                        stored_prefix
+                            .record_map
+                            .as_records_with_rewritten_status(
+                                self.global_withdrawn_bmin,
+                                RouteStatus::Withdrawn,
+                            )
+                    } else {
+                        stored_prefix
+                            .record_map
+                            .as_active_records_not_in_bmin(
+                                self.global_withdrawn_bmin,
+                            )
+                    }
+                };
+                // There is a prefix here, but we need to check if it's
+                // the right one.
+                if self.cur_prefix_id == stored_prefix.prefix {
+                    trace!("found requested prefix {:?}", self.cur_prefix_id);
+                    self.cur_len -= 1;
+                    self.cur_level = 0;
+                    self.cur_bucket =
+                        self.prefixes.get_root_prefix_set(self.cur_len);
+                    return if !pfx_rec.is_empty() {
+                        Some((stored_prefix.prefix, pfx_rec))
+                    } else {
+                        None
+                    };
+                };
+                // Advance to the next level or the next len.
+                match stored_prefix
+                    .next_bucket.is_empty()
+                    // .0
+                    // .load(Ordering::SeqCst, self.guard)
+                    // .is_null()
+                {
+                    // No child here, move one length down.
+                    true => {
+                        self.cur_len -= 1;
+                        self.cur_level = 0;
+                        self.cur_bucket =
+                            self.prefixes.get_root_prefix_set(self.cur_len);
+                    }
+                    // There's a child, move a level up and set the child
+                    // as current. Length remains the same.
+                    false => {
+                        self.cur_bucket = &stored_prefix.next_bucket;
+                        self.cur_level += 1;
+                    }
+                };
+            } else {
+                trace!("no prefix at this level. Move one down.");
+                self.cur_len -= 1;
+                self.cur_level = 0;
+                self.cur_bucket =
+                    self.prefixes.get_root_prefix_set(self.cur_len);
+            }
+            // }
+        }
+    }
+}
+
+// ----------- Iterator initialization methods for Rib -----------------------
+
+// These are only the methods that are starting the iterations. All other
+// methods for Rib are in the main rib.rs file.
+
+impl<
+        'a,
+        AF: AddressFamily,
+        M: crate::prefix_record::Meta,
+        NB: NodeBuckets<AF>,
+        PB: PrefixBuckets<AF, M>,
+    > TreeBitMap<AF, M, NB, PB>
+{
+    // Iterator over all more-specific prefixes, starting from the given
+    // prefix at the given level and cursor.
+    pub fn more_specific_prefix_iter_from(
+        &'a self,
+        start_prefix_id: PrefixId<AF>,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        trace!("more specifics for {:?}", start_prefix_id);
+
+        // A v4 /32 or a v6 /128 doesn't have more specific prefixes 🤓.
+        if start_prefix_id.get_len() >= AF::BITS {
+            None
+        } else {
+            // calculate the node start_prefix_id lives in.
+            let (start_node_id, start_bs) =
+                self.get_node_id_for_prefix(&start_prefix_id);
+            trace!("start node {}", start_node_id);
+            trace!(
+                "start prefix id {:032b} (len {})",
+                start_prefix_id.get_net(),
+                start_prefix_id.get_len()
+            );
+            trace!(
+                "start node id   {:032b} (bits {} len {})",
+                start_node_id.get_id().0,
+                start_node_id.get_id().0,
+                start_node_id.get_len()
+            );
+            trace!(
+                "start pfx bit span  {:08b} {} len {}",
+                start_bs.bits,
+                start_bs.bits,
+                start_bs.len
+            );
+            trace!(
+                "start ptr bit span  {:08b} {} len {}",
+                start_bs.bits,
+                start_bs.bits,
+                start_bs.len
+            );
+
+            let cur_pfx_iter: SizedPrefixIter<AF>;
+            let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
+
+            let node = if let Some(mui) = mui {
+                self.retrieve_node_for_mui(start_node_id, mui)
+            } else {
+                self.retrieve_node(start_node_id)
+            };
+
+            if let Some(node) = node {
+                match node {
+                    SizedStrideRef::Stride3(n) => {
+                        cur_pfx_iter = SizedPrefixIter::Stride3(
+                            n.more_specific_pfx_iter(start_node_id, start_bs),
+                        );
+                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
+                            n.more_specific_ptr_iter(start_node_id, start_bs),
+                        );
+                    }
+                    SizedStrideRef::Stride4(n) => {
+                        cur_pfx_iter = SizedPrefixIter::Stride4(
+                            n.more_specific_pfx_iter(start_node_id, start_bs),
+                        );
+                        trace!("---------------------");
+                        trace!("start iterating nodes");
+                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride4(
+                            n.more_specific_ptr_iter(start_node_id, start_bs),
+                        );
+                    }
+                    SizedStrideRef::Stride5(n) => {
+                        cur_pfx_iter = SizedPrefixIter::Stride5(
+                            n.more_specific_pfx_iter(start_node_id, start_bs),
+                        );
+                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
+                            n.more_specific_ptr_iter(start_node_id, start_bs),
+                        );
+                    }
+                };
+
+                let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
+
+                Some(MoreSpecificPrefixIter {
+                    store: self,
+                    cur_pfx_iter,
+                    cur_ptr_iter,
+                    // start_bit_span,
+                    parent_and_position: vec![],
+                    global_withdrawn_bmin,
+                    include_withdrawn,
+                    mui,
+                })
+            } else {
+                None
+            }
+        }
+        .into_iter()
+        .flatten()
+    }
+
+    // Iterator over all less-specific prefixes, starting from the given
+    // prefix at the given level and cursor.
+    pub fn less_specific_prefix_iter(
+        &'a self,
+        start_prefix_id: PrefixId<AF>,
+        // Indicate whether we want to return only records for a specific mui,
+        // None indicates returning all records for a prefix.
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        trace!("less specifics for {:?}", start_prefix_id);
+        trace!("level {}, len {}", 0, start_prefix_id.get_len());
+
+        // We could just let the /0 prefix search the tree and have it return
+        // an empty iterator, but to avoid having to read out the root node
+        // for this prefix, we'll just return an empty iterator. None can be
+        // turned into an Iterator!
+        if start_prefix_id.get_len() < 1 {
+            None
+        } else {
+            let cur_len = start_prefix_id.get_len() - 1;
+            let cur_bucket = self.prefix_buckets.get_root_prefix_set(cur_len);
+            let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
+
+            Some(LessSpecificPrefixIter {
+                prefixes: &self.prefix_buckets,
+                cur_len,
+                cur_bucket,
+                cur_level: 0,
+                cur_prefix_id: start_prefix_id,
+                mui,
+                global_withdrawn_bmin,
+                include_withdrawn,
+            })
+        }
+        .into_iter()
+        .flatten()
+    }
+
+    // Iterator over all the prefixes in the in_memory store.
+    pub fn prefixes_iter(
+        &'a self,
+    ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
+        PrefixIter {
+            prefixes: &self.prefix_buckets,
+            cur_bucket: self.prefix_buckets.get_root_prefix_set(0),
+            cur_len: 0,
+            cur_level: 0,
+            cursor: 0,
+            parents: [None; 32],
+        }
+    }
+}
diff --git a/src/local_array/prefix_cht/mod.rs b/src/local_array/prefix_cht/mod.rs
new file mode 100644
index 00000000..dbb286d0
--- /dev/null
+++ b/src/local_array/prefix_cht/mod.rs
@@ -0,0 +1,2 @@
+pub(crate) mod cht;
+pub(crate) mod iterators;
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 42b92f17..3cd108d1 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -1,12 +1,13 @@
 use crossbeam_epoch::{self as epoch};
 use epoch::Guard;
+use log::{log_enabled, trace};
 
 use crate::af::AddressFamily;
 use crate::local_array::in_memory::atomic_types::{
     NodeBuckets, PrefixBuckets,
 };
 use crate::rib::{PersistStrategy, Rib};
-use crate::PublicRecord;
+use crate::{PublicRecord, RecordSet};
 use inetnum::addr::Prefix;
 
 use crate::{IncludeHistory, Meta, QueryResult};
@@ -34,19 +35,6 @@ where
         guard: &'a Guard,
     ) -> Option<Vec<PublicRecord<M>>> {
         match self.persist_strategy() {
-            PersistStrategy::WriteAhead => todo!(),
-            PersistStrategy::PersistHistory => todo!(),
-            PersistStrategy::MemoryOnly => self
-                .in_memory_tree
-                .non_recursive_retrieve_prefix(prefix_id)
-                .0
-                .map(|pfx| {
-                    pfx.record_map.get_filtered_records(
-                        mui,
-                        include_withdrawn,
-                        self.in_memory_tree.withdrawn_muis_bmin(guard),
-                    )
-                }),
             PersistStrategy::PersistOnly => {
                 self.persist_tree.as_ref().map(|tree| {
                     tree.get_records_for_prefix(
@@ -57,6 +45,12 @@ where
                     )
                 })
             }
+            _ => self.prefix_cht.get_records_for_prefix(
+                prefix_id,
+                mui,
+                include_withdrawn,
+                self.in_memory_tree.withdrawn_muis_bmin(guard),
+            ),
         }
     }
 
@@ -72,13 +66,14 @@ where
         } else {
             None
         };
-        let more_specifics_vec =
-            self.in_memory_tree.more_specific_prefix_iter_from(
-                prefix_id,
-                mui,
-                include_withdrawn,
-                guard,
-            );
+        let more_specifics = self
+            .in_memory_tree
+            .more_specific_prefix_iter_from(prefix_id)
+            .map(|p| {
+                self.get_value(prefix_id, mui, include_withdrawn, guard)
+                    .map(|v| (p, v))
+            })
+            .collect();
 
         QueryResult {
             prefix,
@@ -90,7 +85,7 @@ where
                 .unwrap_or(vec![]),
             match_type: MatchType::EmptyMatch,
             less_specifics: None,
-            more_specifics: Some(more_specifics_vec.collect()),
+            more_specifics,
         }
     }
 
@@ -107,31 +102,22 @@ where
             None
         };
 
-        // let less_specifics_vec = result.1.map(
-        //     |(prefix_id, _level, _cur_set, _parents, _index)| {
-        let less_specifics_vec =
-            self.in_memory_tree.less_specific_prefix_iter(
-                prefix_id,
-                mui,
-                include_withdrawn,
-                guard,
-            );
+        let less_specifics = self
+            .in_memory_tree
+            .less_specific_prefix_iter(prefix_id)
+            .map(|p| {
+                self.get_value(prefix_id, mui, include_withdrawn, guard)
+                    .map(|v| (p, v))
+            })
+            .collect();
 
         QueryResult {
             prefix,
-            // prefix_meta: prefix
-            //     .map(|r| {
-            //         r.record_map.get_filtered_records(
-            //             mui,
-            //             self.in_memory_tree.withdrawn_muis_bmin(guard),
-            //         )
-            //     })
-            //     .unwrap_or_default(),
             prefix_meta: self
                 .get_value(prefix_id, mui, include_withdrawn, guard)
                 .unwrap_or_default(),
             match_type: MatchType::EmptyMatch,
-            less_specifics: Some(less_specifics_vec.collect()),
+            less_specifics,
             more_specifics: None,
         }
     }
@@ -159,12 +145,14 @@ where
         }) {
             None
         } else {
-            Some(self.in_memory_tree.more_specific_prefix_iter_from(
-                prefix_id,
-                mui,
-                include_withdrawn,
-                guard,
-            ))
+            Some(
+                self.in_memory_tree
+                    .more_specific_prefix_iter_from(prefix_id)
+                    .filter_map(move |p| {
+                        self.get_value(p, mui, include_withdrawn, guard)
+                            .map(|v| (p, v))
+                    }),
+            )
         })
         .into_iter()
         .flatten()
@@ -203,18 +191,42 @@ where
             PersistStrategy::MemoryOnly
             | PersistStrategy::WriteAhead
             | PersistStrategy::PersistHistory => {
-                (if mui.is_some_and(|m| self.mui_is_withdrawn(m, guard)) {
-                    None
-                } else {
-                    Some(self.in_memory_tree.less_specific_prefix_iter(
-                        prefix_id,
-                        mui,
-                        include_withdrawn,
-                        guard,
-                    ))
-                })
-                .into_iter()
-                .flatten()
+                // if mui.is_some_and(|m| self.mui_is_withdrawn(m, guard)) {
+                //     return None;
+                // }
+
+                self.in_memory_tree
+                    .less_specific_prefix_iter(prefix_id)
+                    .filter_map(move |p| {
+                        self.get_value(p, mui, include_withdrawn, guard)
+                            .map(|v| (p, v))
+                    })
+
+                // (if mui.is_some_and(|m| self.mui_is_withdrawn(m, guard)) {
+                //     None
+                // } else {
+                //     Some(
+                //         self.in_memory_tree
+                //             .less_specific_prefix_iter(
+                //                 prefix_id,
+                //                 // mui,
+                //                 // include_withdrawn,
+                //                 // guard,
+                //             )
+                //             .map(|p| {
+                //                 self.get_value(
+                //                     p,
+                //                     mui.clone(),
+                //                     include_withdrawn,
+                //                     guard,
+                //                 )
+                //                 .map(|v| (p, v))
+                //                 .unwrap()
+                //             }),
+                //     )
+                // })
+                // .into_iter()
+                // .flatten()
             }
             PersistStrategy::PersistOnly => unimplemented!(),
         }
@@ -226,16 +238,110 @@ where
         options: &MatchOptions,
         guard: &'a Guard,
     ) -> QueryResult<M> {
-        println!("match_prefix rib");
+        trace!("match_prefix rib {:?} {:?}", search_pfx, options);
+        let res = self.in_memory_tree.match_prefix(search_pfx, options);
+
+        trace!("res {:?}", res);
+        let mut res = QueryResult::from(res);
+
+        if log_enabled!(log::Level::Trace) {
+            let ms = self
+                .in_memory_tree
+                .more_specific_prefix_iter_from(search_pfx)
+                .collect::<Vec<_>>();
+            trace!("more specifics!!! {:?}", ms);
+            trace!("len {}", ms.len());
+        }
+
+        if let Some(Some(m)) = res.prefix.map(|p| {
+            self.get_value(
+                p.into(),
+                options.mui,
+                options.include_withdrawn,
+                guard,
+            )
+            .and_then(|v| if v.is_empty() { None } else { Some(v) })
+        }) {
+            trace!("got value {:?}", m);
+            res.prefix_meta = m;
+        } else {
+            res.prefix = None;
+            res.match_type = MatchType::EmptyMatch;
+        }
+
+        if options.include_more_specifics {
+            res.more_specifics = res.more_specifics.map(|p| {
+                p.iter()
+                    .filter_map(|mut r| {
+                        if let Some(m) = self.get_value(
+                            r.prefix.into(),
+                            options.mui,
+                            options.include_withdrawn,
+                            guard,
+                        ) {
+                            r.meta = m;
+                            Some(r)
+                        } else {
+                            None
+                        }
+                    })
+                    .collect()
+            });
+        }
+        if options.include_less_specifics {
+            res.less_specifics = res.less_specifics.map(|p| {
+                p.iter()
+                    .filter_map(|mut r| {
+                        if let Some(m) = self.get_value(
+                            r.prefix.into(),
+                            options.mui,
+                            options.include_withdrawn,
+                            guard,
+                        ) {
+                            r.meta = m;
+                            Some(r)
+                        } else {
+                            None
+                        }
+                    })
+                    .collect()
+            });
+        }
+
+        res
+    }
+
+    pub fn match_prefix_legacy(
+        &'a self,
+        search_pfx: PrefixId<AF>,
+        options: &MatchOptions,
+        guard: &'a Guard,
+    ) -> QueryResult<M> {
         match self.config.persist_strategy() {
             // Everything is in memory only, so look there. There can be no
             // historical records for this variant, so we just return the
             // stuff found in memeory. A request for  historical records
             // willbe ignored.
-            PersistStrategy::MemoryOnly => self
-                .in_memory_tree
-                .match_prefix(search_pfx, options, guard)
-                .into(),
+            PersistStrategy::MemoryOnly => {
+                let mut res: QueryResult<M> = self
+                    .in_memory_tree
+                    .match_prefix_by_tree_traversal(search_pfx, options)
+                    .into();
+
+                res.prefix_meta = res
+                    .prefix
+                    .map(|p| {
+                        self.get_value(
+                            p.into(),
+                            options.mui,
+                            options.include_withdrawn,
+                            guard,
+                        )
+                        .unwrap_or_default()
+                    })
+                    .unwrap_or_default();
+                res
+            }
             // All the records are persisted, they have never been committed
             // to memory. However the in-memory-tree is still used to indicate
             // which (prefix, mui) tuples have been created.
@@ -284,9 +390,10 @@ where
             // We have the current records in memory, additionally they may be
             // encriched with historical data from the persisted data.
             PersistStrategy::WriteAhead => {
-                let mut res = self
+                let mut res: FamilyQueryResult<AF, M> = self
                     .in_memory_tree
-                    .match_prefix(search_pfx, options, guard);
+                    .match_prefix_by_tree_traversal(search_pfx, options)
+                    .into();
                 if let Some(persist_tree) = &self.persist_tree {
                     match options.include_history {
                         IncludeHistory::All => {
@@ -367,9 +474,10 @@ where
             // requested historical records, we will ask the persist_tree to
             // add those to the intermedidate result of the in-memory query.
             PersistStrategy::PersistHistory => {
-                let mut res = self
+                let mut res: FamilyQueryResult<AF, M> = self
                     .in_memory_tree
-                    .match_prefix(search_pfx, options, guard);
+                    .match_prefix_by_tree_traversal(search_pfx, options)
+                    .into();
 
                 if let Some(persist_tree) = &self.persist_tree {
                     match options.include_history {
@@ -457,7 +565,7 @@ where
         search_pfx: PrefixId<AF>,
         guard: &Guard,
     ) -> Option<Result<PublicRecord<M>, PrefixStoreError>> {
-        self.in_memory_tree
+        self.prefix_cht
             .non_recursive_retrieve_prefix(search_pfx)
             .0
             .map(|p_rec| {
@@ -479,7 +587,7 @@ where
         tbi: &<M as Meta>::TBI,
         guard: &Guard,
     ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
-        self.in_memory_tree
+        self.prefix_cht
             .non_recursive_retrieve_prefix(search_pfx)
             .0
             .map_or(Err(PrefixStoreError::StoreNotReadyError), |p_rec| {
@@ -492,7 +600,7 @@ where
         search_pfx: PrefixId<AF>,
         guard: &Guard,
     ) -> Result<bool, PrefixStoreError> {
-        self.in_memory_tree
+        self.prefix_cht
             .non_recursive_retrieve_prefix(search_pfx)
             .0
             .map_or(Err(PrefixStoreError::StoreNotReadyError), |p| {
@@ -509,6 +617,37 @@ pub(crate) struct TreeQueryResult<AF: AddressFamily> {
     pub more_specifics: Option<Vec<PrefixId<AF>>>,
 }
 
+impl<AF: AddressFamily, M: Meta> From<TreeQueryResult<AF>>
+    for QueryResult<M>
+{
+    fn from(value: TreeQueryResult<AF>) -> Self {
+        Self {
+            match_type: value.match_type,
+            prefix: value.prefix.map(|p| p.into()),
+            prefix_meta: vec![],
+            less_specifics: value
+                .less_specifics
+                .map(|ls| ls.into_iter().map(|p| (p, vec![])).collect()),
+            more_specifics: value
+                .more_specifics
+                .map(|ms| ms.into_iter().map(|p| (p, vec![])).collect()),
+        }
+    }
+}
+
+impl<AF: AddressFamily, M: Meta> From<TreeQueryResult<AF>>
+    for FamilyQueryResult<AF, M>
+{
+    fn from(value: TreeQueryResult<AF>) -> Self {
+        Self {
+            match_type: value.match_type,
+            prefix: value.prefix,
+            prefix_meta: vec![],
+            less_specifics: None,
+            more_specifics: None,
+        }
+    }
+}
 pub(crate) type FamilyRecord<AF, M> =
     Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>;
 
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index cc3cac5d..c8297f0f 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -3,12 +3,13 @@ use std::path::Path;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use inetnum::addr::Prefix;
-use log::info;
+use log::{info, trace};
 
 use crossbeam_epoch::{self as epoch};
 use epoch::{Guard, Owned};
 
 use crate::local_array::in_memory::tree::TreeBitMap;
+use crate::local_array::prefix_cht::cht::PrefixCHT;
 use crate::local_array::types::PrefixId;
 use crate::prelude::multi::RouteStatus;
 use crate::stats::CreatedNodes;
@@ -225,7 +226,8 @@ pub struct Rib<
     const KEY_SIZE: usize,
 > {
     pub config: StoreConfig,
-    pub in_memory_tree: TreeBitMap<AF, M, NB, PB>,
+    pub(crate) in_memory_tree: TreeBitMap<AF, NB>,
+    pub(crate) prefix_cht: PrefixCHT<AF, M, PB>,
     #[cfg(feature = "persist")]
     pub(in crate::local_array) persist_tree:
         Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
@@ -265,10 +267,11 @@ impl<
 
         let store = Rib {
             config,
-            in_memory_tree: TreeBitMap::<AF, M, NB, PB>::new()?,
+            in_memory_tree: TreeBitMap::<AF, NB>::new()?,
             persist_tree,
             // withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
+            prefix_cht: PrefixCHT::<AF, M, PB>::new(),
         };
 
         Ok(store)
@@ -316,7 +319,7 @@ impl<
                 if let Some(persist_tree) = &self.persist_tree {
                     persist_tree.persist_record(prefix, &record);
 
-                    self.in_memory_tree
+                    self.prefix_cht
                         .upsert_prefix(
                             prefix,
                             record,
@@ -329,7 +332,7 @@ impl<
                 }
             }
             PersistStrategy::PersistHistory => self
-                .in_memory_tree
+                .prefix_cht
                 .upsert_prefix(prefix, record, update_path_selections, guard)
                 .map(|(report, old_rec)| {
                     if let Some(rec) = old_rec {
@@ -343,7 +346,7 @@ impl<
                     report
                 }),
             PersistStrategy::MemoryOnly => self
-                .in_memory_tree
+                .prefix_cht
                 .upsert_prefix(prefix, record, update_path_selections, guard)
                 .map(|(report, _)| report),
             PersistStrategy::PersistOnly => {
@@ -387,9 +390,8 @@ impl<
     ) -> Result<(), PrefixStoreError> {
         match self.persist_strategy() {
             PersistStrategy::WriteAhead | PersistStrategy::MemoryOnly => {
-                let (stored_prefix, exists) = self
-                    .in_memory_tree
-                    .non_recursive_retrieve_prefix_mut(prefix);
+                let (stored_prefix, exists) =
+                    self.prefix_cht.non_recursive_retrieve_prefix_mut(prefix);
 
                 if !exists {
                     return Err(PrefixStoreError::PrefixNotFound);
@@ -428,9 +430,8 @@ impl<
             }
             PersistStrategy::PersistHistory => {
                 // First do the in-memory part
-                let (stored_prefix, exists) = self
-                    .in_memory_tree
-                    .non_recursive_retrieve_prefix_mut(prefix);
+                let (stored_prefix, exists) =
+                    self.prefix_cht.non_recursive_retrieve_prefix_mut(prefix);
 
                 if !exists {
                     return Err(PrefixStoreError::StoreNotReadyError);
@@ -482,9 +483,8 @@ impl<
     ) -> Result<(), PrefixStoreError> {
         match self.persist_strategy() {
             PersistStrategy::WriteAhead | PersistStrategy::MemoryOnly => {
-                let (stored_prefix, exists) = self
-                    .in_memory_tree
-                    .non_recursive_retrieve_prefix_mut(prefix);
+                let (stored_prefix, exists) =
+                    self.prefix_cht.non_recursive_retrieve_prefix_mut(prefix);
 
                 if !exists {
                     return Err(PrefixStoreError::PrefixNotFound);
@@ -528,9 +528,8 @@ impl<
             }
             PersistStrategy::PersistHistory => {
                 // First do the in-memory part
-                let (stored_prefix, exists) = self
-                    .in_memory_tree
-                    .non_recursive_retrieve_prefix_mut(prefix);
+                let (stored_prefix, exists) =
+                    self.prefix_cht.non_recursive_retrieve_prefix_mut(prefix);
 
                 if !exists {
                     return Err(PrefixStoreError::PrefixNotFound);
@@ -688,19 +687,28 @@ impl<
         }
     }
 
-    pub fn prefixes_iter(
-        &self,
-    ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + '_ {
-        let pt_iter =
-            if self.config.persist_strategy == PersistStrategy::WriteAhead {
-                None
-            } else {
-                self.persist_tree.as_ref().map(|t| t.prefixes_iter())
-            }
-            .into_iter()
-            .flatten();
+    pub fn prefixes_iter<'a>(
+        &'a self,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
+        self.in_memory_tree.prefixes_iter().map(|p| {
+            (
+                p,
+                self.get_value(p.into(), None, true, guard)
+                    .unwrap_or_default(),
+            )
+        })
+
+        // let pt_iter = match self.config.persist_strategy {
+        //     PersistStrategy::PersistOnly => {
+        //         self.persist_tree.as_ref().map(|t| t.prefixes_iter())
+        //     }
+        //     _ => None,
+        // }
+        // .into_iter()
+        // .flatten();
 
-        self.in_memory_tree.prefixes_iter().chain(pt_iter)
+        // in_mem_iter.chain(pt_iter)
     }
 
     //-------- Persistence ---------------------------------------------------
@@ -715,16 +723,30 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
     ) -> Vec<PublicRecord<M>> {
-        if let Some(p) = &self.persist_tree {
-            let guard = epoch::pin();
-            p.get_records_for_prefix(
-                PrefixId::from(*prefix),
-                mui,
-                include_withdrawn,
-                self.in_memory_tree.withdrawn_muis_bmin(&guard),
-            )
-        } else {
-            vec![]
+        trace!("get records for prefix in the right store");
+        let guard = epoch::pin();
+        match self.persist_strategy() {
+            PersistStrategy::PersistOnly => self
+                .persist_tree
+                .as_ref()
+                .map(|tree| {
+                    tree.get_records_for_prefix(
+                        PrefixId::from(*prefix),
+                        mui,
+                        include_withdrawn,
+                        self.in_memory_tree.withdrawn_muis_bmin(&guard),
+                    )
+                })
+                .unwrap_or_default(),
+            _ => self
+                .prefix_cht
+                .get_records_for_prefix(
+                    PrefixId::from(*prefix),
+                    mui,
+                    include_withdrawn,
+                    self.in_memory_tree.withdrawn_muis_bmin(&guard),
+                )
+                .unwrap_or_default(),
         }
     }
 
diff --git a/src/local_array/types.rs b/src/local_array/types.rs
index 731b85a2..48e48bd9 100644
--- a/src/local_array/types.rs
+++ b/src/local_array/types.rs
@@ -1,3 +1,5 @@
+use log::trace;
+
 use crate::AddressFamily;
 
 use super::errors::PrefixStoreError;
@@ -74,6 +76,24 @@ impl<AF: AddressFamily> PrefixId<AF> {
         }
     }
 
+    pub fn truncate_to_len(self, len: u8) -> Self {
+        trace!("orig {:032b}", self.net);
+        trace!(
+            "new  {:032b}",
+            self.net >> (AF::BITS - len) << (AF::BITS - len)
+        );
+        trace!(
+            "truncate to net {} len {}",
+            self.net >> (AF::BITS - len) << (AF::BITS - len),
+            len
+        );
+        Self {
+            // net: (self.net >> (AF::BITS - len)) << (AF::BITS - len),
+            net: self.net.truncate_to_len(len),
+            len,
+        }
+    }
+
     // The lsm tree, used for persistence, stores the prefix in the key with
     // len first, so that key range lookups can be made for more-specifics in
     // each prefix length.

From 3a24a08b241d23d0b8eda7ecb253a9db01f492e2 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 7 Feb 2025 15:28:46 +0100
Subject: [PATCH 068/147] grand cleanup #1

---
 src/local_array/in_memory/iterators.rs  |  492 +--------
 src/local_array/in_memory/node.rs       |  922 ++++++++---------
 src/local_array/in_memory/query.rs      | 1224 ++++++++++-------------
 src/local_array/in_memory/tree.rs       |  345 ++++---
 src/local_array/prefix_cht/iterators.rs |   81 --
 src/local_array/query.rs                |  341 +------
 src/local_array/rib/rib.rs              |   77 +-
 src/local_array/tests.rs                |    5 +-
 8 files changed, 1234 insertions(+), 2253 deletions(-)

diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 80a0e7db..2047c18e 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -38,204 +38,6 @@ use crate::{
 use inetnum::addr::Prefix;
 use log::{log_enabled, trace};
 
-// ----------- PrefixIter ---------------------------------------------------
-
-// Iterator over all the prefixes in the storage. This Iterator does *not* use
-// the tree, it iterates over all the length arrays in the CustomAllocStorage.
-
-// pub(crate) struct PrefixIter<
-//     'a,
-//     AF: AddressFamily + 'a,
-//     M: Meta + 'a,
-//     PB: PrefixBuckets<AF, M>,
-// > {
-//     prefixes: &'a PB,
-//     cur_len: u8,
-//     cur_bucket: &'a PrefixSet<AF, M>,
-//     cur_level: u8,
-//     // level depth of IPv4 as defined in rotonda-macros/maps.rs Option(parent,
-//     // cursor position at the parent) 32 is the max number of levels in IPv6,
-//     // which is the max number of of both IPv4 and IPv6.
-//     parents: [Option<(&'a PrefixSet<AF, M>, usize)>; 32],
-//     cursor: usize,
-// }
-
-// impl<'a, AF: AddressFamily + 'a, M: Meta + 'a, PB: PrefixBuckets<AF, M>>
-//     Iterator for PrefixIter<'a, AF, M, PB>
-// {
-//     type Item = (inetnum::addr::Prefix, Vec<PublicRecord<M>>);
-
-//     fn next(&mut self) -> Option<Self::Item> {
-//         trace!(
-//             "starting next loop for level {} cursor {} (len {})",
-//             self.cur_level,
-//             self.cursor,
-//             self.cur_len
-//         );
-
-//         loop {
-//             if self.cur_len > AF::BITS {
-//                 // This is the end, my friend
-//                 trace!("reached max length {}, returning None", self.cur_len);
-//                 return None;
-//             }
-
-//             if PB::get_bits_for_len(self.cur_len, self.cur_level) == 0 {
-//                 // END OF THE LENGTH
-
-//                 // This length is done too, go to the next length
-//                 // trace!("next length {}", self.cur_len + 1);
-//                 self.cur_len += 1;
-
-//                 // a new length, a new life reset the level depth and cursor,
-//                 // but also empty all the parents
-//                 self.cur_level = 0;
-//                 self.cursor = 0;
-//                 self.parents = [None; 32];
-
-//                 // let's continue, get the prefixes for the next length
-//                 self.cur_bucket =
-//                     self.prefixes.get_root_prefix_set(self.cur_len);
-//                 continue;
-//             }
-//             let bucket_size = 1_usize
-//                 << (if self.cur_level > 0 {
-//                     PB::get_bits_for_len(self.cur_len, self.cur_level)
-//                         - PB::get_bits_for_len(
-//                             self.cur_len,
-//                             self.cur_level - 1,
-//                         )
-//                 } else {
-//                     PB::get_bits_for_len(self.cur_len, self.cur_level)
-//                 });
-
-//             if self.cursor >= bucket_size {
-//                 if self.cur_level == 0 {
-//                     // END OF THE LENGTH
-
-//                     // This length is done too, go to the next length
-//                     // trace!("next length {}", self.cur_len);
-//                     self.cur_len += 1;
-
-//                     // a new length, a new life reset the level depth and
-//                     // cursor, but also empty all the parents
-//                     self.cur_level = 0;
-//                     self.cursor = 0;
-//                     self.parents = [None; 32];
-
-//                     if self.cur_len > AF::BITS {
-//                         // This is the end, my friend
-//                         return None;
-//                     }
-
-//                     // let's continue, get the prefixes for the next length
-//                     self.cur_bucket =
-//                         self.prefixes.get_root_prefix_set(self.cur_len);
-//                 } else {
-//                     // END OF THIS BUCKET GO BACK UP ONE LEVEL
-
-//                     // The level is done, but the length isn't Go back up one
-//                     // level and continue
-//                     match self.parents[self.cur_level as usize] {
-//                         Some(parent) => {
-//                             // There is a parent, go back up. Since we're
-//                             // doing depth-first we have to check if there's a
-//                             // prefix directly at the parent and return that.
-//                             self.cur_level -= 1;
-
-//                             // move the current bucket to the parent and move
-//                             // the cursor position where we left off. The next
-//                             // run of the loop will read it.
-//                             self.cur_bucket = parent.0;
-//                             self.cursor = parent.1 + 1;
-
-//                             continue;
-//                         }
-//                         None => {
-//                             trace!(
-//                                 "c {} lvl {} len {}",
-//                                 self.cursor,
-//                                 self.cur_level,
-//                                 self.cur_len
-//                             );
-//                             panic!(
-//                                 "Where do we belong? Where do we come from?"
-//                             );
-//                         }
-//                     };
-//                 }
-//             };
-
-//             // we're somewhere in the PrefixSet iteration, read the next
-//             // StoredPrefix. We are doing depth-first iteration, so we check
-//             // for a child first and descend into that if it exists.
-
-//             if let Some(s_pfx) = self.cur_bucket.get_by_index(self.cursor) {
-//                 // DEPTH FIRST ITERATION
-//                 match s_pfx.get_next_bucket() {
-//                     Some(bucket) => {
-//                         // DESCEND ONe LEVEL There's a child here, descend into
-//                         // it, but... trace!("C. got next bucket {:?}", bucket);
-
-//                         // save our parent and cursor position first, and then..
-//                         self.parents[(self.cur_level + 1) as usize] =
-//                             Some((self.cur_bucket, self.cursor));
-
-//                         // move to the next bucket,
-//                         self.cur_bucket = bucket;
-
-//                         // increment the level and reset the cursor.
-//                         self.cur_level += 1;
-//                         self.cursor = 0;
-
-//                         // If there's a child here there MUST be a prefix here,
-//                         // as well.
-//                         // if let Some(meta) =
-//                         //     s_pfx.get_stored_prefix(self.guard).map(|p| {
-//                         //         if log_enabled!(log::Level::Trace) {
-//                         //             // There's a prefix here, that's the next one
-//                         //             trace!("D. found prefix {:?}", p.prefix);
-//                         //         }
-//                         //         p.record_map.as_records()
-//                         //     })
-//                         // {
-//                         return Some((
-//                             s_pfx.get_prefix_id().into(),
-//                             s_pfx.record_map.as_records(),
-//                         ));
-//                         // } else {
-//                         //     panic!(
-//                         //         "No prefix here, but there's a child here?"
-//                         //     );
-//                         // }
-//                     }
-//                     None => {
-//                         // No reference to another PrefixSet, all that's left, is
-//                         // checking for a prefix at the current cursor position.
-//                         // if let Some(meta) =
-//                         //     s_pfx.get_stored_prefix(self.guard).map(|p| {
-//                         //         // There's a prefix here, that's the next one
-//                         //         if log_enabled!(log::Level::Debug) {
-//                         //             debug!("E. found prefix {:?}", p.prefix);
-//                         //         }
-//                         //         p.record_map.as_records()
-//                         //     })
-//                         // {
-//                         self.cursor += 1;
-//                         return Some((
-//                             s_pfx.get_prefix_id().into(),
-//                             s_pfx.record_map.as_records(),
-//                         ));
-//                         // }
-//                     }
-//                 };
-//             } else {
-//                 self.cursor += 1;
-//             }
-//         }
-//     }
-// }
-
 // ----------- Sized Wrappers -----------------------------------------------
 
 // These are enums to abstract over the Stride Size of the iterators. Each
@@ -296,32 +98,16 @@ impl<AF: AddressFamily> SizedPrefixIter<AF> {
 pub(crate) struct MoreSpecificPrefixIter<
     'a,
     AF: AddressFamily,
-    // M: Meta,
     NB: NodeBuckets<AF>,
-    // PB: PrefixBuckets<AF, M>,
 > {
     tree: &'a TreeBitMap<AF, NB>,
     cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
     cur_pfx_iter: SizedPrefixIter<AF>,
-    // start_bit_span: BitSpan,
-    // skip_self: bool,
     parent_and_position: Vec<SizedNodeMoreSpecificIter<AF>>,
-    // If specified, we're only iterating over records for this mui.
-    // mui: Option<u32>,
-    // This is the tree-wide index of withdrawn muis, used to rewrite the
-    // statuses of these records, or filter them out.
-    // global_withdrawn_bmin: &'a RoaringBitmap,
-    // Whether we should filter out the withdrawn records in the search result
-    // include_withdrawn: bool,
 }
 
-impl<
-        'a,
-        AF: AddressFamily + 'a,
-        // M: Meta,
-        NB: NodeBuckets<AF>,
-        // PB: PrefixBuckets<AF, M>,
-    > Iterator for MoreSpecificPrefixIter<'a, AF, NB>
+impl<'a, AF: AddressFamily + 'a, NB: NodeBuckets<AF>> Iterator
+    for MoreSpecificPrefixIter<'a, AF, NB>
 {
     type Item = PrefixId<AF>;
 
@@ -334,87 +120,6 @@ impl<
 
             if next_pfx.is_some() {
                 return next_pfx;
-                // If we have a mui, we have to deal slightly different with
-                // the records: There can only be one record for a (prefix,
-                // mui) combination, and the record may be filtered out by the
-                // global status of the mui, or its local status. In that case
-                // we don't return here (because that would result in a Prefix
-                // with an empty record vec).
-                // next_pfx
-                // if let Some(mui) = self.mui {
-                //     if let Some(p) = self
-                //         .store
-                //         .non_recursive_retrieve_prefix(
-                //             next_pfx.unwrap_or_else(|| {
-                //                 panic!(
-                //                 "BOOM! More-specific prefix {:?} disappeared \
-                //                 from the store",
-                //                 next_pfx
-                //             )
-                //             }),
-                //         )
-                //         .0
-                //     {
-                //         // We may either have to rewrite the local status with
-                //         // the provided global status OR we may have to omit
-                //         // all of the records with either global of local
-                //         // withdrawn status.
-                //         if self.include_withdrawn {
-                //             if let Some(rec) = p
-                //                 .record_map
-                //                 .get_record_for_mui_with_rewritten_status(
-                //                     mui,
-                //                     self.global_withdrawn_bmin,
-                //                     RouteStatus::Withdrawn,
-                //                 )
-                //             {
-                //                 return Some((p.prefix, vec![rec]));
-                //             }
-                //         } else if let Some(rec) = p
-                //             .record_map
-                //             .get_record_for_mui(mui, self.include_withdrawn)
-                //         {
-                //             return Some((p.prefix, vec![rec]));
-                //         }
-                //     };
-                // } else {
-                //     return self
-                //         .store
-                //         .non_recursive_retrieve_prefix(
-                //             next_pfx.unwrap_or_else(|| {
-                //                 panic!(
-                //                 "BOOM! More-specific prefix {:?} disappeared \
-                //                 from the store",
-                //                 next_pfx
-                //             )
-                //             }),
-                //             // self.guard,
-                //         )
-                //         .0
-                //         .map(|p| {
-                //             // Just like the mui specific records, we may have
-                //             // to either rewrite the local status (if the user
-                //             // wants the withdrawn records) or omit them.
-                //             if self.include_withdrawn {
-                //                 (
-                //                     p.prefix,
-                //                     p.record_map
-                //                         .as_records_with_rewritten_status(
-                //                             self.global_withdrawn_bmin,
-                //                             RouteStatus::Withdrawn,
-                //                         ),
-                //                 )
-                //             } else {
-                //                 (
-                //                     p.prefix,
-                //                     p.record_map
-                //                         .as_active_records_not_in_bmin(
-                //                             self.global_withdrawn_bmin,
-                //                         ),
-                //                 )
-                //             }
-                //         });
-                // }
             }
 
             // Our current prefix iterator for this node is done, look for
@@ -438,13 +143,7 @@ impl<
             }
 
             if let Some(next_ptr) = next_ptr {
-                // let node = if self.mui.is_none() {
-                //     trace!("let's retriev node {}", next_ptr);
                 let node = self.tree.retrieve_node(next_ptr);
-                // } else {
-                //     self.store
-                //         .retrieve_node_for_mui(next_ptr, self.mui.unwrap())
-                // };
 
                 match node {
                     Some(SizedStrideRef::Stride3(next_node)) => {
@@ -594,151 +293,6 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> Iterator
         }
     }
 }
-// loop {
-//     if self.cur_len == 0 {
-//         // This is the end, my friend
-//         trace!("reached min length {}, returning None", self.cur_len);
-//         return None;
-//     }
-
-//     // shave a bit of the current prefix.
-//     trace!(
-//         "truncate to len {} (level {})",
-//         self.cur_len,
-//         self.cur_level
-//     );
-//     self.cur_prefix_id = PrefixId::new(
-//         self.cur_prefix_id.get_net().truncate_to_len(self.cur_len),
-//         self.cur_len,
-//     );
-
-//     let last_level = if self.cur_level > 0 {
-//         PB::get_bits_for_len(self.cur_len, self.cur_level - 1)
-//     } else {
-//         0
-//     };
-
-//     let this_level =
-//         PB::get_bits_for_len(self.cur_len, self.cur_level);
-
-//     // NOT THE HASHING FUNCTION
-//     let index = ((self.cur_prefix_id.get_net() << last_level)
-//         >> ((AF::BITS - (this_level - last_level)) % AF::BITS))
-//         .dangerously_truncate_to_u32()
-//         as usize;
-
-//     if this_level == 0 {
-//         // END OF THE LENGTH
-//         // This length is done too, go to the next length
-//         // trace!("next length {}", self.cur_len + 1);
-//         self.cur_len -= 1;
-
-//         // a new length, a new life
-//         // reset the level depth and cursor,
-//         // but also empty all the parents
-//         self.cur_level = 0;
-//         // self.parents = [None; 26];
-
-//         // let's continue, get the prefixes for the next length
-//         self.cur_bucket =
-//             self.prefixes.get_root_prefix_set(self.cur_len);
-//         continue;
-//     }
-
-//     // LEVEL DEPTH ITERATION
-//     if let Some(stored_prefix) = self.cur_bucket.get_by_index(index) {
-//         // if let Some(stored_prefix) =
-//         // s_pfx.get_stored_prefix(self.guard)
-//         // {
-//         // trace!("get_record {:?}", stored_prefix.record_map);
-//         // let pfx_rec = if let Some(mui) = self.mui {
-//         // We don't have to check for the appearance of We may
-//         // either have to rewrite the local status with the
-//         // provided global status OR we may have to omit all of
-//         // the records with either global of local withdrawn
-//         // status.
-//         // if self.include_withdrawn {
-//         // let pfx_rec = stored_prefix
-//         //     .record_map
-//         //     .get_record_for_mui_with_rewritten_status(
-//         //         mui,
-//         //         self.global_withdrawn_bmin,
-//         //         RouteStatus::Withdrawn,
-//         //     )
-//         //     .into_iter()
-//         //     .collect();
-//         // } else {
-//         let pfx_rec = stored_prefix
-//             .record_map
-//             .get_record_for_mui(mui, self.include_withdrawn)
-//             .into_iter()
-//             .collect();
-//         // }
-//         // } else {
-//         // Just like the mui specific records, we may have
-//         // to either rewrite the local status (if the user
-//         // wants the withdrawn records) or omit them.
-//         //     if self.include_withdrawn {
-//         //         stored_prefix
-//         //             .record_map
-//         //             .as_records_with_rewritten_status(
-//         //                 self.global_withdrawn_bmin,
-//         //                 RouteStatus::Withdrawn,
-//         //             )
-//         //     } else {
-//         //         stored_prefix
-//         //             .record_map
-//         //             .as_active_records_not_in_bmin(
-//         //                 self.global_withdrawn_bmin,
-//         //             )
-//         //     }
-//         // };
-//         // There is a prefix here, but we need to check if it's
-//         // the right one.
-//         if self.cur_prefix_id == stored_prefix.prefix {
-//             trace!("found requested prefix {:?}", self.cur_prefix_id);
-//             self.cur_len -= 1;
-//             self.cur_level = 0;
-//             self.cur_bucket =
-//                 self.prefixes.get_root_prefix_set(self.cur_len);
-//             return if !pfx_rec.is_empty() {
-//                 Some((stored_prefix.prefix, pfx_rec))
-//             } else {
-//                 None
-//             };
-//         };
-//         // Advance to the next level or the next len.
-//         match stored_prefix
-//             .next_bucket.is_empty()
-//             // .0
-//             // .load(Ordering::SeqCst, self.guard)
-//             // .is_null()
-//         {
-//             // No child here, move one length down.
-//             true => {
-//                 self.cur_len -= 1;
-//                 self.cur_level = 0;
-//                 self.cur_bucket =
-//                     self.prefixes.get_root_prefix_set(self.cur_len);
-//             }
-//             // There's a child, move a level up and set the child
-//             // as current. Length remains the same.
-//             false => {
-//                 self.cur_bucket = &stored_prefix.next_bucket;
-//                 self.cur_level += 1;
-//             }
-//         };
-//     } else {
-//         trace!("no prefix at this level. Move one down.");
-//         self.cur_len -= 1;
-//         self.cur_level = 0;
-//         self.cur_bucket =
-//             self.prefixes.get_root_prefix_set(self.cur_len);
-//     }
-//     // }
-// }
-//     }
-// }
 
 // ----------- Iterator initialization methods for Rib -----------------------
 
@@ -758,9 +312,6 @@ impl<
     pub fn more_specific_prefix_iter_from(
         &'a self,
         start_prefix_id: PrefixId<AF>,
-        // mui: Option<u32>,
-        // include_withdrawn: bool,
-        // guard: &'a Guard,
     ) -> impl Iterator<Item = PrefixId<AF>> + 'a {
         trace!("more specifics for {:?}", start_prefix_id);
 
@@ -798,12 +349,7 @@ impl<
 
             let cur_pfx_iter: SizedPrefixIter<AF>;
             let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
-
-            // let node = if let Some(mui) = mui {
-            // self.retrieve_node_for_mui(start_node_id, mui)
-            // } else {
             let node = self.retrieve_node(start_node_id);
-            // };
 
             if let Some(node) = node {
                 match node {
@@ -835,16 +381,11 @@ impl<
                     }
                 };
 
-                // let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
-
                 Some(MoreSpecificPrefixIter {
                     tree: self,
                     cur_pfx_iter,
                     cur_ptr_iter,
                     parent_and_position: vec![],
-                    // global_withdrawn_bmin,
-                    // include_withdrawn,
-                    // mui,
                 })
             } else {
                 None
@@ -859,46 +400,17 @@ impl<
     pub fn less_specific_prefix_iter(
         &'a self,
         start_prefix_id: PrefixId<AF>,
-        // Indicate whether we want to return only records for a specific mui,
-        // None indicates returning all records for a prefix.
-        // mui: Option<u32>,
-        // include_withdrawn: bool,
-        // guard: &'a Guard,
     ) -> impl Iterator<Item = PrefixId<AF>> + 'a {
         if log_enabled!(log::Level::Trace) {
             trace!("less specifics for {}", Prefix::from(start_prefix_id));
             trace!("level {}, len {}", 0, start_prefix_id.get_len());
         }
 
-        // We could just let the /0 prefix search the tree and have it return
-        // an empty iterator, but to avoid having to read out the root node
-        // for this prefix, we'll just return an empty iterator. None can be
-        // turned into an Iterator!
-        // if start_prefix_id.get_len() < 1 {
-        //     None
-        // } else {
-        // let cur_len = start_prefix_id.get_len() - 1;
-        // let cur_bucket = self.prefix_store.get_root_prefix_set(cur_len);
-        // let global_withdrawn_bmin = self.withdrawn_muis_bmin(guard);
-
-        // Some(
-
         LessSpecificPrefixIter {
-            // prefixes: &self.prefix_buckets,
             tree: self,
             prefix: start_prefix_id,
-            // cur_len,
-            // cur_bucket,
             cur_level: start_prefix_id.get_len(),
-            // cur_prefix_id: start_prefix_id,
-            // mui,
-            // global_withdrawn_bmin,
-            // include_withdrawn,
         }
-        //     })
-        // }
-        // .into_iter()
-        // .flatten()
     }
 
     pub fn longest_matching_prefix(
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 784e2282..cc7b50fd 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -86,46 +86,16 @@ where
 
     // ------- Iterators ----------------------------------------------------
 
-    // Iterate over all the child node of this node
-    pub(crate) fn ptr_iter(
-        &self,
-        base_prefix: StrideNodeId<AF>,
-    ) -> NodeChildIter<AF, S> {
-        NodeChildIter::<AF, S> {
-            base_prefix,
-            ptrbitarr: self.ptrbitarr.load(),
-            bit_span: BitSpan::new(0, 1),
-            _af: PhantomData,
-        }
-    }
-
-    // Iterate over all the prefix ids contained in this node.
-    // Note that this function is *not* used by the iterator that iterates
-    // over all prefixes. That one doesn't have to use the tree at all, but
-    // uses the store directly.
-    pub(crate) fn pfx_iter(
-        &self,
-        base_prefix: StrideNodeId<AF>,
-    ) -> NodePrefixIter<AF, S> {
-        NodePrefixIter::<AF, S> {
-            pfxbitarr: self.pfxbitarr.load(),
-            base_prefix,
-            cursor: 1,
-            _af: PhantomData,
-            _s: PhantomData,
-        }
-    }
-
     // Iterate over the more specific prefixes ids contained in this node
     pub(crate) fn more_specific_pfx_iter(
         &self,
         base_prefix: StrideNodeId<AF>,
         start_bs: BitSpan,
     ) -> NodeMoreSpecificsPrefixIter<AF, S> {
-        assert!(start_bs.check());
+        debug_assert!(start_bs.check());
         let mut pfxbitarr = self.pfxbitarr.load();
-        trace!("pfxbitarr {:032b}", pfxbitarr);
         pfxbitarr = S::ms_pfx_mask(pfxbitarr, start_bs);
+
         NodeMoreSpecificsPrefixIter::<AF, S> {
             pfxbitarr,
             base_prefix,
@@ -139,12 +109,10 @@ where
         base_prefix: StrideNodeId<AF>,
         start_bs: BitSpan,
     ) -> NodeMoreSpecificChildIter<AF, S> {
+        debug_assert!(start_bs.check());
         let ptrbitarr = self.ptrbitarr.load();
         let (bitrange, start_cursor) = S::ptr_range(ptrbitarr, start_bs);
 
-        trace!("now we're really starting!");
-        trace!("start_bs {:?}", start_bs);
-        trace!("ptrbitarr {:032b}", ptrbitarr);
         NodeMoreSpecificChildIter::<AF, S> {
             bitrange,
             base_prefix,
@@ -335,157 +303,157 @@ where
     // the appropriate bytes from the requested prefix. It mutates the
     // `less_specifics_vec` that was passed in to hold all the prefixes found
     // along the way.
-    pub(crate) fn search_stride_for_longest_match_at(
-        &self,
-        search_pfx: PrefixId<AF>,
-        mut nibble: u32,
-        nibble_len: u8,
-        start_bit: u8,
-        less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
-    ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
-        let pfxbitarr = self.pfxbitarr.load();
-        let ptrbitarr = self.ptrbitarr.load();
-        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_pfx = None;
-
-        trace!("start longest_match search");
-        for n_l in 1..(nibble_len + 1) {
-            // Move the bit in the right position.
-            nibble = AddressFamily::get_nibble(
-                search_pfx.get_net(),
-                start_bit,
-                n_l,
-            );
-            bit_pos = S::get_bit_pos(nibble, n_l);
-
-            // Check if the prefix has been set, if so select the prefix.
-            // This is not necessarily the final prefix that will be
-            // returned.
-
-            // Check it there's a prefix matching in this bitmap for this
-            // nibble.
-            trace!("pfxbitarr {:032b}", pfxbitarr);
-
-            if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize
-                as AtomicBitmap>::InnerType::zero() {
-                let f_pfx = PrefixId::new(search_pfx.get_net()
-                    .truncate_to_len(start_bit + n_l), start_bit + n_l);
-                // f_pfx.set_serial(self.get_pfx_serial(f_pfx, nibble, n_l, guard).load(Ordering::Relaxed));
-
-                // Receiving a less_specifics_vec means that the user wants
-                // to have all the last-specific prefixes returned, so add
-                // the found prefix.
-                trace!("gather pfx in less_specifics {:?}", f_pfx);
-                trace!("ls_vec {:?}", less_specifics_vec);
-                if let Some(ls_vec) = less_specifics_vec {
-                    trace!("len {}", search_pfx.get_len());
-                    trace!("start_bit {}", start_bit);
-                    trace!("n_l {}", n_l);
-                    trace!("smaller length? {}",
-                        search_pfx.get_len() > start_bit + n_l);
-                    trace!("{}", (S::into_stride_size(ptrbitarr)
-                            & bit_pos)
-                            == <<S as Stride>::AtomicPfxSize
-                            as AtomicBitmap>::InnerType::zero());
-                    if search_pfx.get_len() > start_bit + n_l
-                        && (S::into_stride_size(ptrbitarr)
-                            & bit_pos)
-                            == <<S as Stride>::AtomicPfxSize
-                                as AtomicBitmap>::InnerType::zero()
-                    {
-                        ls_vec.push(f_pfx);
-                    }
-                }
-
-                found_pfx = Some(f_pfx);
-            }
-        }
-
-        let base_prefix = StrideNodeId::new_with_cleaned_id(
-            search_pfx.get_net(),
-            start_bit,
-        );
-
-        // Check if this the last stride, or if they're no more children to
-        // go to, if so return what we found up until now.
-        if search_pfx.get_len() <= start_bit + nibble_len
-            || (S::into_stride_size(ptrbitarr) & bit_pos) ==
-                <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType
-                    ::zero()
-        // No children or at the end, return the definitive LMP we found.
-        {
-            return (
-                None,      /* no more children */
-                found_pfx, /* The definitive LMP if any */
-            );
-        }
-
-        // There's another child, return it together with the preliminary LMP
-        // we found.
-        (
-            // The identifier of the node that has children of the next
-            // stride.
-            Some(base_prefix.add_bit_span(BitSpan {
-                bits: nibble,
-                len: nibble_len,
-            })),
-            found_pfx,
-        )
-    }
+    // pub(crate) fn search_stride_for_longest_match_at(
+    //     &self,
+    //     search_pfx: PrefixId<AF>,
+    //     mut nibble: u32,
+    //     nibble_len: u8,
+    //     start_bit: u8,
+    //     less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
+    // ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
+    //     let pfxbitarr = self.pfxbitarr.load();
+    //     let ptrbitarr = self.ptrbitarr.load();
+    //     let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
+    //     let mut found_pfx = None;
+
+    //     trace!("start longest_match search");
+    //     for n_l in 1..(nibble_len + 1) {
+    //         // Move the bit in the right position.
+    //         nibble = AddressFamily::get_nibble(
+    //             search_pfx.get_net(),
+    //             start_bit,
+    //             n_l,
+    //         );
+    //         bit_pos = S::get_bit_pos(nibble, n_l);
+
+    //         // Check if the prefix has been set, if so select the prefix.
+    //         // This is not necessarily the final prefix that will be
+    //         // returned.
+
+    //         // Check it there's a prefix matching in this bitmap for this
+    //         // nibble.
+    //         trace!("pfxbitarr {:032b}", pfxbitarr);
+
+    //         if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize
+    //             as AtomicBitmap>::InnerType::zero() {
+    //             let f_pfx = PrefixId::new(search_pfx.get_net()
+    //                 .truncate_to_len(start_bit + n_l), start_bit + n_l);
+    //             // f_pfx.set_serial(self.get_pfx_serial(f_pfx, nibble, n_l, guard).load(Ordering::Relaxed));
+
+    //             // Receiving a less_specifics_vec means that the user wants
+    //             // to have all the last-specific prefixes returned, so add
+    //             // the found prefix.
+    //             trace!("gather pfx in less_specifics {:?}", f_pfx);
+    //             trace!("ls_vec {:?}", less_specifics_vec);
+    //             if let Some(ls_vec) = less_specifics_vec {
+    //                 trace!("len {}", search_pfx.get_len());
+    //                 trace!("start_bit {}", start_bit);
+    //                 trace!("n_l {}", n_l);
+    //                 trace!("smaller length? {}",
+    //                     search_pfx.get_len() > start_bit + n_l);
+    //                 trace!("{}", (S::into_stride_size(ptrbitarr)
+    //                         & bit_pos)
+    //                         == <<S as Stride>::AtomicPfxSize
+    //                         as AtomicBitmap>::InnerType::zero());
+    //                 if search_pfx.get_len() > start_bit + n_l
+    //                     && (S::into_stride_size(ptrbitarr)
+    //                         & bit_pos)
+    //                         == <<S as Stride>::AtomicPfxSize
+    //                             as AtomicBitmap>::InnerType::zero()
+    //                 {
+    //                     ls_vec.push(f_pfx);
+    //                 }
+    //             }
+
+    //             found_pfx = Some(f_pfx);
+    //         }
+    //     }
+
+    //     let base_prefix = StrideNodeId::new_with_cleaned_id(
+    //         search_pfx.get_net(),
+    //         start_bit,
+    //     );
+
+    //     // Check if this the last stride, or if they're no more children to
+    //     // go to, if so return what we found up until now.
+    //     if search_pfx.get_len() <= start_bit + nibble_len
+    //         || (S::into_stride_size(ptrbitarr) & bit_pos) ==
+    //             <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType
+    //                 ::zero()
+    //     // No children or at the end, return the definitive LMP we found.
+    //     {
+    //         return (
+    //             None,      /* no more children */
+    //             found_pfx, /* The definitive LMP if any */
+    //         );
+    //     }
+
+    //     // There's another child, return it together with the preliminary LMP
+    //     // we found.
+    //     (
+    //         // The identifier of the node that has children of the next
+    //         // stride.
+    //         Some(base_prefix.add_bit_span(BitSpan {
+    //             bits: nibble,
+    //             len: nibble_len,
+    //         })),
+    //         found_pfx,
+    //     )
+    // }
 
     // This function looks for the exactly matching prefix in the provided
     // nibble. It doesn't need to iterate over anything it just compares
     // the complete nibble, with the appropriate bits in the requested
     // prefix. Although this is rather efficient, there's no way to collect
     // less-specific prefixes from the search prefix.
-    pub(crate) fn search_stride_for_exact_match_at(
-        &'_ self,
-        search_pfx: PrefixId<AF>,
-        nibble: u32,
-        nibble_len: u8,
-        start_bit: u8,
-        _: &mut Option<Vec<PrefixId<AF>>>,
-    ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
-        let pfxbitarr = self.pfxbitarr.load();
-        let ptrbitarr = self.ptrbitarr.load();
-        // This is an exact match, so we're only considering the position of
-        // the full nibble.
-        let bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_pfx = None;
-        let mut found_child = None;
-
-        // Is this the last nibble?
-        // Otherwise we're not looking for a prefix (exact matching only
-        // lives at last nibble)
-        match search_pfx.get_len() <= start_bit + nibble_len {
-            // We're at the last nibble.
-            true => {
-                // Check for an actual prefix at the right position, i.e.
-                // consider the complete nibble.
-                if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-                    let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + nibble_len), start_bit + nibble_len);
-                    found_pfx = Some(f_pfx);
-                }
-            }
-            // We're not at the last nibble.
-            false => {
-                // Check for a child node at the right position, i.e.
-                // consider the complete nibble.
-                if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-                {
-                    found_child = Some(
-                        StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)
-                    );
-                }
-            }
-        }
-
-        (
-            found_child, /* The node that has children in the next stride, if
-                         any */
-            found_pfx, /* The exactly matching prefix, if any */
-        )
-    }
+    // pub(crate) fn search_stride_for_exact_match_at(
+    //     &'_ self,
+    //     search_pfx: PrefixId<AF>,
+    //     nibble: u32,
+    //     nibble_len: u8,
+    //     start_bit: u8,
+    //     _: &mut Option<Vec<PrefixId<AF>>>,
+    // ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
+    //     let pfxbitarr = self.pfxbitarr.load();
+    //     let ptrbitarr = self.ptrbitarr.load();
+    //     // This is an exact match, so we're only considering the position of
+    //     // the full nibble.
+    //     let bit_pos = S::get_bit_pos(nibble, nibble_len);
+    //     let mut found_pfx = None;
+    //     let mut found_child = None;
+
+    //     // Is this the last nibble?
+    //     // Otherwise we're not looking for a prefix (exact matching only
+    //     // lives at last nibble)
+    //     match search_pfx.get_len() <= start_bit + nibble_len {
+    //         // We're at the last nibble.
+    //         true => {
+    //             // Check for an actual prefix at the right position, i.e.
+    //             // consider the complete nibble.
+    //             if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
+    //                 let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + nibble_len), start_bit + nibble_len);
+    //                 found_pfx = Some(f_pfx);
+    //             }
+    //         }
+    //         // We're not at the last nibble.
+    //         false => {
+    //             // Check for a child node at the right position, i.e.
+    //             // consider the complete nibble.
+    //             if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+    //             {
+    //                 found_child = Some(
+    //                     StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)
+    //                 );
+    //             }
+    //         }
+    //     }
+
+    //     (
+    //         found_child, /* The node that has children in the next stride, if
+    //                      any */
+    //         found_pfx, /* The exactly matching prefix, if any */
+    //     )
+    // }
 
     pub(crate) fn prefix_exists_in_stride(
         &'_ self,
@@ -539,195 +507,195 @@ where
     // bytes in the nibble to collect the less-specific prefixes of the the
     // search prefix. This is of course slower, so it should only be used
     // when the user explicitly requests less-specifics.
-    pub(crate) fn search_stride_for_exact_match_with_less_specifics_at(
-        &self,
-        search_pfx: PrefixId<AF>,
-        mut nibble: u32,
-        nibble_len: u8,
-        start_bit: u8,
-        less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
-    ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
-        let pfxbitarr = self.pfxbitarr.load();
-        let ptrbitarr = self.ptrbitarr.load();
-        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_pfx = None;
-
-        let ls_vec = less_specifics_vec.as_mut().expect(concat!(
-            "You shouldn't call this function without",
-            "a `less_specifics_vec` buffer. Supply one when calling this
-             function",
-            "or use `search_stride_for_exact_match_at`"
-        ));
-
-        for n_l in 1..(nibble_len + 1) {
-            // Move the bit in the right position.
-            nibble = AddressFamily::get_nibble(
-                search_pfx.get_net(),
-                start_bit,
-                n_l,
-            );
-            bit_pos = S::get_bit_pos(nibble, n_l);
-
-            // Check if the prefix has been set, if so select the prefix.
-            // This is not necessarily the final prefix that will be
-            // returned.
-
-            // Check it there's a prefix matching in this bitmap for this
-            // nibble.
-            if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as
-                AtomicBitmap>::InnerType::zero() {
-                // since we want an exact match only, we will fill the prefix
-                // field only if we're exactly at the last bit of the nibble
-                if n_l == nibble_len {
-                    let f_pfx =
-                        PrefixId::new(
-                            search_pfx.get_net().truncate_to_len(start_bit +
-                                n_l), start_bit + n_l);
-                        found_pfx = Some(f_pfx);
-                }
-
-                // Receiving a less_specifics_vec means that the user wants to
-                // have all the last-specific prefixes returned, so add the
-                // found prefix.
-                let f_pfx = PrefixId::new(search_pfx.get_net().
-                    truncate_to_len(start_bit + n_l), start_bit + n_l);
-                ls_vec.push(f_pfx);
-            }
-        }
-
-        if found_pfx.is_none() {
-            // no prefix here, clear out all of the prefixes we found along
-            // the way, since it doesn't make sense to return less-specifics
-            // if we don't have a exact match.
-            ls_vec.clear();
-        }
-
-        // Check if this the last stride, or if they're no more children to
-        // go to, if so return what we found up until now.
-        match search_pfx.get_len() <= start_bit + nibble_len
-            || (S::into_stride_size(ptrbitarr) & bit_pos)
-                == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType
-                    as std::ops::BitAnd>::Output::zero()
-        {
-            // No children or at the end, return the definitive LMP we found.
-            true => {
-                println!("found {:?}", found_pfx); 
-                (
-                None,      /* no more children */
-                found_pfx, /* The definitive LMP if any */
-                )
-            },
-            // There's another child, we won't return the found_pfx, since
-            // we're not at the last nibble and we want an exact match only.
-            false => (
-                Some(StrideNodeId::new_with_cleaned_id(search_pfx.get_net(),
-                    start_bit + nibble_len)),
-                None,
-            ),
-        }
-    }
+    // pub(crate) fn search_stride_for_exact_match_with_less_specifics_at(
+    //     &self,
+    //     search_pfx: PrefixId<AF>,
+    //     mut nibble: u32,
+    //     nibble_len: u8,
+    //     start_bit: u8,
+    //     less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
+    // ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
+    //     let pfxbitarr = self.pfxbitarr.load();
+    //     let ptrbitarr = self.ptrbitarr.load();
+    //     let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
+    //     let mut found_pfx = None;
+
+    //     let ls_vec = less_specifics_vec.as_mut().expect(concat!(
+    //         "You shouldn't call this function without",
+    //         "a `less_specifics_vec` buffer. Supply one when calling this
+    //          function",
+    //         "or use `search_stride_for_exact_match_at`"
+    //     ));
+
+    //     for n_l in 1..(nibble_len + 1) {
+    //         // Move the bit in the right position.
+    //         nibble = AddressFamily::get_nibble(
+    //             search_pfx.get_net(),
+    //             start_bit,
+    //             n_l,
+    //         );
+    //         bit_pos = S::get_bit_pos(nibble, n_l);
+
+    //         // Check if the prefix has been set, if so select the prefix.
+    //         // This is not necessarily the final prefix that will be
+    //         // returned.
+
+    //         // Check it there's a prefix matching in this bitmap for this
+    //         // nibble.
+    //         if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as
+    //             AtomicBitmap>::InnerType::zero() {
+    //             // since we want an exact match only, we will fill the prefix
+    //             // field only if we're exactly at the last bit of the nibble
+    //             if n_l == nibble_len {
+    //                 let f_pfx =
+    //                     PrefixId::new(
+    //                         search_pfx.get_net().truncate_to_len(start_bit +
+    //                             n_l), start_bit + n_l);
+    //                     found_pfx = Some(f_pfx);
+    //             }
+
+    //             // Receiving a less_specifics_vec means that the user wants to
+    //             // have all the last-specific prefixes returned, so add the
+    //             // found prefix.
+    //             let f_pfx = PrefixId::new(search_pfx.get_net().
+    //                 truncate_to_len(start_bit + n_l), start_bit + n_l);
+    //             ls_vec.push(f_pfx);
+    //         }
+    //     }
+
+    //     if found_pfx.is_none() {
+    //         // no prefix here, clear out all of the prefixes we found along
+    //         // the way, since it doesn't make sense to return less-specifics
+    //         // if we don't have a exact match.
+    //         ls_vec.clear();
+    //     }
+
+    //     // Check if this the last stride, or if they're no more children to
+    //     // go to, if so return what we found up until now.
+    //     match search_pfx.get_len() <= start_bit + nibble_len
+    //         || (S::into_stride_size(ptrbitarr) & bit_pos)
+    //             == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType
+    //                 as std::ops::BitAnd>::Output::zero()
+    //     {
+    //         // No children or at the end, return the definitive LMP we found.
+    //         true => {
+    //             println!("found {:?}", found_pfx);
+    //             (
+    //             None,      /* no more children */
+    //             found_pfx, /* The definitive LMP if any */
+    //             )
+    //         },
+    //         // There's another child, we won't return the found_pfx, since
+    //         // we're not at the last nibble and we want an exact match only.
+    //         false => (
+    //             Some(StrideNodeId::new_with_cleaned_id(search_pfx.get_net(),
+    //                 start_bit + nibble_len)),
+    //             None,
+    //         ),
+    //     }
+    // }
 
     // Search a stride for more-specific prefixes and child nodes containing
     // more specifics for `search_prefix`.
-    pub(crate) fn add_more_specifics_at(
-        &self,
-        nibble: u32,
-        nibble_len: u8,
-        base_prefix: StrideNodeId<AF>,
-    ) -> (
-        Vec<StrideNodeId<AF>>, /* child nodes with more more-specifics in
-                               this stride */
-        Vec<PrefixId<AF>>, /* more-specific prefixes in this stride */
-    ) {
-        trace!("start adding more specifics");
-        let pfxbitarr = self.pfxbitarr.load();
-        let ptrbitarr = self.ptrbitarr.load();
-        trace!("ptrbitarr {:032b}", ptrbitarr);
-        trace!("pfxbitarr {:032b}", pfxbitarr);
-        let mut found_children_with_more_specifics = vec![];
-        let mut found_more_specifics_vec: Vec<PrefixId<AF>> = vec![];
-
-        // This is an exact match, so we're only considering the position of
-        // the full nibble.
-        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_child = None;
-
-        // Is there also a child node here?
-        // Note that even without a child node, there may be more specifics
-        // further up in this pfxbitarr or children in this ptrbitarr.
-        if (S::into_stride_size(ptrbitarr) & bit_pos)
-            > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
-            )
-        {
-            found_child = Some(base_prefix.add_bit_span(BitSpan {
-                bits: nibble,
-                len: nibble_len,
-            }));
-        }
-
-        if let Some(child) = found_child {
-            found_children_with_more_specifics.push(child);
-        }
-
-        // We're expanding the search for more-specifics bit-by-bit.
-        // `ms_nibble_len` is the number of bits including the original
-        // nibble we're considering, e.g. if our prefix has a length of 25
-        // and we've all strides sized 4, we would end up with a last
-        // nibble_len of 1. `ms_nibble_len` will expand then from 2 up and
-        // till 4.
-        //
-        // ex.:
-        // nibble: 1 , (nibble_len: 1)
-        // Iteration:
-        // ms_nibble_len=1,n_l=0: 10, n_l=1: 11
-        // ms_nibble_len=2,n_l=0: 100, n_l=1: 101, n_l=2: 110, n_l=3: 111
-        // ms_nibble_len=3,n_l=0: 1000, n_l=1: 1001, n_l=2: 1010, ...,
-        // n_l=7: 1111
-
-        for ms_nibble_len in nibble_len + 1..=S::STRIDE_LEN {
-            // iterate over all the possible values for this `ms_nibble_len`,
-            // e.g. two bits can have 4 different values.
-            for n_l in 0..(1 << (ms_nibble_len - nibble_len)) {
-                // move the nibble left with the amount of bits we're going
-                // to loop over. e.g. a stride of size 4 with a nibble 0000
-                // 0000 0000 0011 becomes 0000 0000 0000 1100, then it will
-                // iterate over ...1100,...1101,...1110,...1111
-                let ms_nibble =
-                    (nibble << (ms_nibble_len - nibble_len)) + n_l as u32;
-                bit_pos = S::get_bit_pos(ms_nibble, ms_nibble_len);
-
-                if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-                {
-                    found_children_with_more_specifics.push(
-                    base_prefix.add_bit_span(BitSpan {
-                        bits: ms_nibble, len: ms_nibble_len})
-                    );
-                }
-
-                if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-                    found_more_specifics_vec.push(
-                        base_prefix.add_bit_span(BitSpan {
-                            bits: ms_nibble,
-                            len: ms_nibble_len
-                        }).into())
-                }
-            }
-        }
-
-        trace!(
-            "found_children_with_more_specifics {:?}",
-            found_children_with_more_specifics
-        );
-        trace!("found_more_specifics_vec {:?}", found_more_specifics_vec);
-
-        (
-            // We're done here, the caller should now go over all nodes in
-            // found_children_with_more_specifics vec and add ALL prefixes
-            // found in there.
-            found_children_with_more_specifics,
-            found_more_specifics_vec,
-        )
-    }
+    // pub(crate) fn add_more_specifics_at(
+    //     &self,
+    //     nibble: u32,
+    //     nibble_len: u8,
+    //     base_prefix: StrideNodeId<AF>,
+    // ) -> (
+    //     Vec<StrideNodeId<AF>>, /* child nodes with more more-specifics in
+    //                            this stride */
+    //     Vec<PrefixId<AF>>, /* more-specific prefixes in this stride */
+    // ) {
+    //     trace!("start adding more specifics");
+    //     let pfxbitarr = self.pfxbitarr.load();
+    //     let ptrbitarr = self.ptrbitarr.load();
+    //     trace!("ptrbitarr {:032b}", ptrbitarr);
+    //     trace!("pfxbitarr {:032b}", pfxbitarr);
+    //     let mut found_children_with_more_specifics = vec![];
+    //     let mut found_more_specifics_vec: Vec<PrefixId<AF>> = vec![];
+
+    //     // This is an exact match, so we're only considering the position of
+    //     // the full nibble.
+    //     let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
+    //     let mut found_child = None;
+
+    //     // Is there also a child node here?
+    //     // Note that even without a child node, there may be more specifics
+    //     // further up in this pfxbitarr or children in this ptrbitarr.
+    //     if (S::into_stride_size(ptrbitarr) & bit_pos)
+    //         > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
+    //         )
+    //     {
+    //         found_child = Some(base_prefix.add_bit_span(BitSpan {
+    //             bits: nibble,
+    //             len: nibble_len,
+    //         }));
+    //     }
+
+    //     if let Some(child) = found_child {
+    //         found_children_with_more_specifics.push(child);
+    //     }
+
+    //     // We're expanding the search for more-specifics bit-by-bit.
+    //     // `ms_nibble_len` is the number of bits including the original
+    //     // nibble we're considering, e.g. if our prefix has a length of 25
+    //     // and we've all strides sized 4, we would end up with a last
+    //     // nibble_len of 1. `ms_nibble_len` will expand then from 2 up and
+    //     // till 4.
+    //     //
+    //     // ex.:
+    //     // nibble: 1 , (nibble_len: 1)
+    //     // Iteration:
+    //     // ms_nibble_len=1,n_l=0: 10, n_l=1: 11
+    //     // ms_nibble_len=2,n_l=0: 100, n_l=1: 101, n_l=2: 110, n_l=3: 111
+    //     // ms_nibble_len=3,n_l=0: 1000, n_l=1: 1001, n_l=2: 1010, ...,
+    //     // n_l=7: 1111
+
+    //     for ms_nibble_len in nibble_len + 1..=S::STRIDE_LEN {
+    //         // iterate over all the possible values for this `ms_nibble_len`,
+    //         // e.g. two bits can have 4 different values.
+    //         for n_l in 0..(1 << (ms_nibble_len - nibble_len)) {
+    //             // move the nibble left with the amount of bits we're going
+    //             // to loop over. e.g. a stride of size 4 with a nibble 0000
+    //             // 0000 0000 0011 becomes 0000 0000 0000 1100, then it will
+    //             // iterate over ...1100,...1101,...1110,...1111
+    //             let ms_nibble =
+    //                 (nibble << (ms_nibble_len - nibble_len)) + n_l as u32;
+    //             bit_pos = S::get_bit_pos(ms_nibble, ms_nibble_len);
+
+    //             if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+    //             {
+    //                 found_children_with_more_specifics.push(
+    //                 base_prefix.add_bit_span(BitSpan {
+    //                     bits: ms_nibble, len: ms_nibble_len})
+    //                 );
+    //             }
+
+    //             if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
+    //                 found_more_specifics_vec.push(
+    //                     base_prefix.add_bit_span(BitSpan {
+    //                         bits: ms_nibble,
+    //                         len: ms_nibble_len
+    //                     }).into())
+    //             }
+    //         }
+    //     }
+
+    //     trace!(
+    //         "found_children_with_more_specifics {:?}",
+    //         found_children_with_more_specifics
+    //     );
+    //     trace!("found_more_specifics_vec {:?}", found_more_specifics_vec);
+
+    //     (
+    //         // We're done here, the caller should now go over all nodes in
+    //         // found_children_with_more_specifics vec and add ALL prefixes
+    //         // found in there.
+    //         found_children_with_more_specifics,
+    //         found_more_specifics_vec,
+    //     )
+    // }
 }
 
 // ------------ Iterator methods --------------------------------------------
@@ -763,41 +731,41 @@ where
 type PtrBitArr<S> = <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
 type PfxBitArr<S> = <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
 
-#[derive(Debug, Copy, Clone)]
-pub(crate) struct NodeChildIter<AF: AddressFamily, S: Stride> {
-    base_prefix: StrideNodeId<AF>,
-    ptrbitarr: PtrBitArr<S>,
-    bit_span: BitSpan, // start with 0
-    _af: PhantomData<AF>,
-}
-
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator
-    for NodeChildIter<AF, S>
-{
-    type Item = StrideNodeId<AF>;
-    fn next(&mut self) -> Option<Self::Item> {
-        // iterate over all the possible values for this stride length, e.g.
-        // two bits can have 4 different values.
-        for cursor in self.bit_span.bits..(1 << S::STRIDE_LEN) {
-            // move the bit_span left with the amount of bits we're going to
-            // loop over.
-            // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
-            // becomes 0000 0000 0000 1100, then it will iterate over
-            // ...1100,...1101,...1110,...1111
-            let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
-            if (S::into_stride_size(self.ptrbitarr) & bit_pos)
-                > PfxBitArr::<S>::zero()
-            {
-                self.bit_span.bits = cursor + 1;
-                return Some(self.base_prefix.add_bit_span(BitSpan {
-                    bits: cursor,
-                    len: S::STRIDE_LEN,
-                }));
-            }
-        }
-        None
-    }
-}
+// #[derive(Debug, Copy, Clone)]
+// pub(crate) struct NodeChildIter<AF: AddressFamily, S: Stride> {
+//     base_prefix: StrideNodeId<AF>,
+//     ptrbitarr: PtrBitArr<S>,
+//     bit_span: BitSpan, // start with 0
+//     _af: PhantomData<AF>,
+// }
+
+// impl<AF: AddressFamily, S: Stride> std::iter::Iterator
+//     for NodeChildIter<AF, S>
+// {
+//     type Item = StrideNodeId<AF>;
+//     fn next(&mut self) -> Option<Self::Item> {
+//         // iterate over all the possible values for this stride length, e.g.
+//         // two bits can have 4 different values.
+//         for cursor in self.bit_span.bits..(1 << S::STRIDE_LEN) {
+//             // move the bit_span left with the amount of bits we're going to
+//             // loop over.
+//             // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
+//             // becomes 0000 0000 0000 1100, then it will iterate over
+//             // ...1100,...1101,...1110,...1111
+//             let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
+//             if (S::into_stride_size(self.ptrbitarr) & bit_pos)
+//                 > PfxBitArr::<S>::zero()
+//             {
+//                 self.bit_span.bits = cursor + 1;
+//                 return Some(self.base_prefix.add_bit_span(BitSpan {
+//                     bits: cursor,
+//                     len: S::STRIDE_LEN,
+//                 }));
+//             }
+//         }
+//         None
+//     }
+// }
 
 // ----------- NodeMoreSpecificChildIter ------------------------------------
 
@@ -967,82 +935,82 @@ impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride5> {
 // variable.
 //
 // The `bit_span` variable starts counting at every new prefix length.
-pub(crate) struct NodePrefixIter<AF: AddressFamily, S: Stride> {
-    base_prefix: StrideNodeId<AF>,
-    pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
-    cursor: u8,
-    _af: PhantomData<AF>,
-    _s: PhantomData<S>,
-}
-
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator
-    for NodePrefixIter<AF, S>
-{
-    type Item = PrefixId<AF>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.pfxbitarr == PfxBitArr::<S>::zero() {
-            return None;
-        }
-        if log_enabled!(log::Level::Trace) {
-            trace!("---- NodePrefixIter -----");
-            match S::STRIDE_LEN {
-                3 => {
-                    trace!(
-                        "pfxbitarr  {:016b} ({})",
-                        self.pfxbitarr,
-                        S::STRIDE_LEN
-                    );
-                }
-                4 => {
-                    trace!(
-                        "pfxbitarr  {:032b} ({})",
-                        self.pfxbitarr,
-                        S::STRIDE_LEN
-                    );
-                }
-                5 => {
-                    trace!(
-                        "pfxbitarr  {:064b} ({})",
-                        self.pfxbitarr,
-                        S::STRIDE_LEN
-                    );
-                }
-                _ => {
-                    trace!("UNKNOWN STRIDE SIZE ENCOUNTERED.");
-                }
-            };
-        }
-
-        let cursor = self.cursor;
-
-        for i in cursor..S::BITS - 1 {
-            let bit_pos = S::bit_pos_from_index(i);
-            self.cursor += 1;
-
-            if log_enabled!(log::Level::Trace) {
-                let bs = BitSpan::from_bit_pos_index(i);
-                trace!(
-                    "{:02}: {:06b} {:064b} bit_span: {:05b} (len: {})",
-                    i,
-                    i - 1,
-                    bit_pos,
-                    bs.bits,
-                    bs.len
-                );
-            }
-
-            if self.pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize
-                as AtomicBitmap>::InnerType::zero() {
-                let bs = BitSpan::from_bit_pos_index(i);
-                return Some(self.base_prefix.add_bit_span(bs).into());
-            }
-        }
-
-        trace!("end of iterator NodePrefixIter for this node");
-        None
-    }
-}
+// pub(crate) struct NodePrefixIter<AF: AddressFamily, S: Stride> {
+//     base_prefix: StrideNodeId<AF>,
+//     pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
+//     cursor: u8,
+//     _af: PhantomData<AF>,
+//     _s: PhantomData<S>,
+// }
+
+// impl<AF: AddressFamily, S: Stride> std::iter::Iterator
+//     for NodePrefixIter<AF, S>
+// {
+//     type Item = PrefixId<AF>;
+
+//     fn next(&mut self) -> Option<Self::Item> {
+//         if self.pfxbitarr == PfxBitArr::<S>::zero() {
+//             return None;
+//         }
+//         if log_enabled!(log::Level::Trace) {
+//             trace!("---- NodePrefixIter -----");
+//             match S::STRIDE_LEN {
+//                 3 => {
+//                     trace!(
+//                         "pfxbitarr  {:016b} ({})",
+//                         self.pfxbitarr,
+//                         S::STRIDE_LEN
+//                     );
+//                 }
+//                 4 => {
+//                     trace!(
+//                         "pfxbitarr  {:032b} ({})",
+//                         self.pfxbitarr,
+//                         S::STRIDE_LEN
+//                     );
+//                 }
+//                 5 => {
+//                     trace!(
+//                         "pfxbitarr  {:064b} ({})",
+//                         self.pfxbitarr,
+//                         S::STRIDE_LEN
+//                     );
+//                 }
+//                 _ => {
+//                     trace!("UNKNOWN STRIDE SIZE ENCOUNTERED.");
+//                 }
+//             };
+//         }
+
+//         let cursor = self.cursor;
+
+//         for i in cursor..S::BITS - 1 {
+//             let bit_pos = S::bit_pos_from_index(i);
+//             self.cursor += 1;
+
+//             if log_enabled!(log::Level::Trace) {
+//                 let bs = BitSpan::from_bit_pos_index(i);
+//                 trace!(
+//                     "{:02}: {:06b} {:064b} bit_span: {:05b} (len: {})",
+//                     i,
+//                     i - 1,
+//                     bit_pos,
+//                     bs.bits,
+//                     bs.len
+//                 );
+//             }
+
+//             if self.pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize
+//                 as AtomicBitmap>::InnerType::zero() {
+//                 let bs = BitSpan::from_bit_pos_index(i);
+//                 return Some(self.base_prefix.add_bit_span(bs).into());
+//             }
+//         }
+
+//         trace!("end of iterator NodePrefixIter for this node");
+//         None
+//     }
+// }
 
 pub const fn ms_prefix_mask_arr(bs: BitSpan) -> u32 {
     [
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index 1a40c263..c9a91956 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -1,19 +1,10 @@
-use crossbeam_epoch::Guard;
-use log::trace;
-
 use crate::af::AddressFamily;
-use crate::prelude::multi::RouteStatus;
-use crate::rib::query::{FamilyQueryResult, TreeQueryResult};
-
-use crate::{Meta, PublicRecord};
 
-use crate::local_array::in_memory::node::{SizedStrideRef, TreeBitMapNode};
+use crate::rib::query::TreeQueryResult;
 use crate::{MatchOptions, MatchType};
 
-use super::super::in_memory::tree::Stride;
 use super::super::types::PrefixId;
 use super::atomic_types::NodeBuckets;
-use super::node::StrideNodeId;
 use super::tree::TreeBitMap;
 
 impl<AF, NB> TreeBitMap<AF, NB>
@@ -68,197 +59,80 @@ where
         }
     }
 
-    // pub(crate) fn match_prefix<'a>(
-    //     &'a self,
-    //     search_pfx: PrefixId<AF>,
-    //     options: &MatchOptions,
-    //     guard: &'a Guard,
-    // ) -> FamilyQueryResult<AF, M> {
-    //     // `non_recursive_retrieve_prefix` returns an exact match only, so no
-    //     // longest matching prefix!
-    //     let withdrawn_muis_bmin = self.withdrawn_muis_bmin(guard);
-    //     let mut stored_prefix =
-    //         self.non_recursive_retrieve_prefix(search_pfx).0.map(|pfx| {
-    //             (
-    //                 pfx.prefix,
-    //                 if !options.include_withdrawn {
-    //                     // Filter out all the withdrawn records, both
-    //                     // with globally withdrawn muis, and with local
-    //                     // statuses
-    //                     // set to Withdrawn.
-    //                     pfx.record_map
-    //                         .get_filtered_records(
-    //                             options.mui,
-    //                             options.include_withdrawn,
-    //                             withdrawn_muis_bmin,
-    //                         )
-    //                         .into_iter()
-    //                         .collect()
-    //                 } else {
-    //                     // Do no filter out any records, but do rewrite
-    //                     // the local statuses of the records with muis
-    //                     // that appear in the specified bitmap index.
-    //                     pfx.record_map.as_records_with_rewritten_status(
-    //                         withdrawn_muis_bmin,
-    //                         RouteStatus::Withdrawn,
-    //                     )
-    //                 },
-    //             )
-    //         });
-
-    //     // Check if we have an actual exact match, if not then fetch the
-    //     // first lesser-specific with the greatest length, that's the Longest
-    //     // matching prefix, but only if the user requested a longest match or
-    //     // empty match.
-    //     let match_type = match (&options.match_type, &stored_prefix) {
-    //         // we found an exact match, we don't need to do anything.
-    //         (_, Some((_pfx, meta))) if !meta.is_empty() => {
-    //             MatchType::ExactMatch
+    // This function assembles all entries in the `pfx_vec` of all child nodes
+    // of the `start_node` into one vec, starting from itself and then
+    // recursively assembling adding all `pfx_vec`s of its children.
+    // fn get_all_more_specifics_for_node(
+    //     &self,
+    //     start_node_id: StrideNodeId<AF>,
+    //     found_pfx_vec: &mut Vec<PrefixId<AF>>,
+    // ) {
+    //     trace!("{:?}", self.retrieve_node(start_node_id));
+    //     match self.retrieve_node(start_node_id) {
+    //         Some(SizedStrideRef::Stride3(n)) => {
+    //             found_pfx_vec.extend(
+    //                 n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
+    //             );
+
+    //             for child_node in n.ptr_iter(start_node_id) {
+    //                 self.get_all_more_specifics_for_node(
+    //                     child_node,
+    //                     found_pfx_vec,
+    //                 );
+    //             }
     //         }
-    //         // we didn't find an exact match, but the user requested it
-    //         // so we need to find the longest matching prefix.
-    //         (MatchType::LongestMatch | MatchType::EmptyMatch, _) => {
-    //             stored_prefix = self
-    //                 .less_specific_prefix_iter(
-    //                     search_pfx,
-    //                     options.mui,
-    //                     options.include_withdrawn,
-    //                     guard,
-    //                 )
-    //                 .max_by(|p0, p1| p0.0.get_len().cmp(&p1.0.get_len()));
-    //             if stored_prefix.is_some() {
-    //                 MatchType::LongestMatch
-    //             } else {
-    //                 MatchType::EmptyMatch
+    //         Some(SizedStrideRef::Stride4(n)) => {
+    //             found_pfx_vec.extend(
+    //                 n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
+    //             );
+
+    //             for child_node in n.ptr_iter(start_node_id) {
+    //                 self.get_all_more_specifics_for_node(
+    //                     child_node,
+    //                     found_pfx_vec,
+    //                 );
     //             }
     //         }
-    //         // We got an empty match, but the user requested an exact match,
-    //         // even so, we're going to look for more and/or less specifics if
-    //         // the user asked for it.
-    //         (MatchType::ExactMatch, _) => MatchType::EmptyMatch,
-    //     };
-
-    //     FamilyQueryResult {
-    //         prefix: stored_prefix.as_ref().map(|sp| sp.0),
-    //         prefix_meta: stored_prefix
-    //             .as_ref()
-    //             .map(|pfx| pfx.1.clone())
-    //             .unwrap_or_default(),
-    //         less_specifics: if options.include_less_specifics {
-    //             Some(
-    //                 self.less_specific_prefix_iter(
-    //                     if let Some(ref pfx) = stored_prefix {
-    //                         pfx.0
-    //                     } else {
-    //                         search_pfx
-    //                     },
-    //                     options.mui,
-    //                     options.include_withdrawn,
-    //                     guard,
-    //                 )
-    //                 .collect::<Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>>(),
-    //             )
-    //         } else {
-    //             None
-    //         },
-    //         more_specifics: if options.include_more_specifics {
-    //             trace!("more specifics requested. acquiring...");
-    //             Some(
-    //                 self.more_specific_prefix_iter_from(
-    //                     if let Some(pfx) = stored_prefix {
-    //                         pfx.0
-    //                     } else {
-    //                         search_pfx
-    //                     },
-    //                     options.mui,
-    //                     options.include_withdrawn,
-    //                     guard,
-    //                 )
-    //                 .collect::<Vec<_>>(),
-    //             )
-    //             // The user requested more specifics, but there aren't any, so
-    //             // we need to return an empty vec, not a None.
-    //         } else {
-    //             None
-    //         },
-    //         match_type,
+    //         Some(SizedStrideRef::Stride5(n)) => {
+    //             found_pfx_vec.extend(
+    //                 n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
+    //             );
+
+    //             for child_node in n.ptr_iter(start_node_id) {
+    //                 self.get_all_more_specifics_for_node(
+    //                     child_node,
+    //                     found_pfx_vec,
+    //                 );
+    //             }
+    //         }
+    //         _ => {
+    //             panic!("can't find node {}", start_node_id);
+    //         }
     //     }
     // }
 
-    // This function assembles all entries in the `pfx_vec` of all child nodes
-    // of the `start_node` into one vec, starting from itself and then
-    // recursively assembling adding all `pfx_vec`s of its children.
-    fn get_all_more_specifics_for_node(
-        &self,
-        start_node_id: StrideNodeId<AF>,
-        found_pfx_vec: &mut Vec<PrefixId<AF>>,
-    ) {
-        trace!("{:?}", self.retrieve_node(start_node_id));
-        match self.retrieve_node(start_node_id) {
-            Some(SizedStrideRef::Stride3(n)) => {
-                found_pfx_vec.extend(
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
-                );
-
-                for child_node in n.ptr_iter(start_node_id) {
-                    self.get_all_more_specifics_for_node(
-                        child_node,
-                        found_pfx_vec,
-                    );
-                }
-            }
-            Some(SizedStrideRef::Stride4(n)) => {
-                found_pfx_vec.extend(
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
-                );
-
-                for child_node in n.ptr_iter(start_node_id) {
-                    self.get_all_more_specifics_for_node(
-                        child_node,
-                        found_pfx_vec,
-                    );
-                }
-            }
-            Some(SizedStrideRef::Stride5(n)) => {
-                found_pfx_vec.extend(
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
-                );
-
-                for child_node in n.ptr_iter(start_node_id) {
-                    self.get_all_more_specifics_for_node(
-                        child_node,
-                        found_pfx_vec,
-                    );
-                }
-            }
-            _ => {
-                panic!("can't find node {}", start_node_id);
-            }
-        }
-    }
-
     // This function assembles the prefixes of a child node starting on a
     // specified bit position in a ptr_vec of `current_node` into a vec,
     // then adds all prefixes of these children recursively into a vec and
     // returns that.
-    fn get_all_more_specifics_from_nibble<S: Stride>(
-        &self,
-        current_node: &TreeBitMapNode<AF, S>,
-        nibble: u32,
-        nibble_len: u8,
-        base_prefix: StrideNodeId<AF>,
-    ) -> Option<Vec<PrefixId<AF>>> {
-        let (cnvec, mut msvec) = current_node.add_more_specifics_at(
-            nibble,
-            nibble_len,
-            base_prefix,
-        );
-
-        for child_node in cnvec.iter() {
-            self.get_all_more_specifics_for_node(*child_node, &mut msvec);
-        }
-        Some(msvec)
-    }
+    // fn get_all_more_specifics_from_nibble<S: Stride>(
+    //     &self,
+    //     current_node: &TreeBitMapNode<AF, S>,
+    //     nibble: u32,
+    //     nibble_len: u8,
+    //     base_prefix: StrideNodeId<AF>,
+    // ) -> Option<Vec<PrefixId<AF>>> {
+    //     let (cnvec, mut msvec) = current_node.add_more_specifics_at(
+    //         nibble,
+    //         nibble_len,
+    //         base_prefix,
+    //     );
+
+    //     for child_node in cnvec.iter() {
+    //         self.get_all_more_specifics_for_node(*child_node, &mut msvec);
+    //     }
+    //     Some(msvec)
+    // }
 
     // In a LMP search we have to go over all the nibble lengths in the
     // stride up until the value of the actual nibble length were looking for
@@ -281,499 +155,499 @@ where
     // nibble              1010 1011 1100 1101 1110 1111    x
     // nibble len offset      4(contd.)
 
-    pub(crate) fn match_prefix_by_tree_traversal(
-        &self,
-        search_pfx: PrefixId<AF>,
-        options: &MatchOptions,
-        // guard: &'a Guard,
-    ) -> TreeQueryResult<AF> {
-        // --- The Default Route Prefix -------------------------------------
-
-        // The Default Route Prefix unfortunately does not fit in tree as we
-        // have it. There's no room for it in the pfxbitarr of the root node,
-        // since that can only contain serial numbers for prefixes that are
-        // children of the root node. We, however, want the default prefix
-        // which lives on the root node itself! We are *not* going to return
-        // all of the prefixes in the tree as more-specifics.
-        if search_pfx.get_len() == 0 {
-            // match self.load_default_route_prefix_serial() {
-            //     0 => {
-            //         return QueryResult {
-            //             prefix: None,
-            //             prefix_meta: vec![],
-            //             match_type: MatchType::EmptyMatch,
-            //             less_specifics: None,
-            //             more_specifics: None,
-            //         };
-            //     }
-
-            //     _serial => {
-            return TreeQueryResult {
-                prefix: None,
-                match_type: MatchType::EmptyMatch,
-                less_specifics: None,
-                more_specifics: None,
-            };
-            // }
-            // }
-        }
+    // pub(crate) fn match_prefix_by_tree_traversal(
+    //     &self,
+    //     search_pfx: PrefixId<AF>,
+    //     options: &MatchOptions,
+    //     // guard: &'a Guard,
+    // ) -> TreeQueryResult<AF> {
+    //     // --- The Default Route Prefix -------------------------------------
+
+    //     // The Default Route Prefix unfortunately does not fit in tree as we
+    //     // have it. There's no room for it in the pfxbitarr of the root node,
+    //     // since that can only contain serial numbers for prefixes that are
+    //     // children of the root node. We, however, want the default prefix
+    //     // which lives on the root node itself! We are *not* going to return
+    //     // all of the prefixes in the tree as more-specifics.
+    //     if search_pfx.get_len() == 0 {
+    //         // match self.load_default_route_prefix_serial() {
+    //         //     0 => {
+    //         //         return QueryResult {
+    //         //             prefix: None,
+    //         //             prefix_meta: vec![],
+    //         //             match_type: MatchType::EmptyMatch,
+    //         //             less_specifics: None,
+    //         //             more_specifics: None,
+    //         //         };
+    //         //     }
+
+    //         //     _serial => {
+    //         return TreeQueryResult {
+    //             prefix: None,
+    //             match_type: MatchType::EmptyMatch,
+    //             less_specifics: None,
+    //             more_specifics: None,
+    //         };
+    //         // }
+    //         // }
+    //     }
 
-        let mut stride_end = 0;
+    //     let mut stride_end = 0;
 
-        let root_node_id = self.get_root_node_id();
-        let mut node = match self.get_stride_for_id(root_node_id) {
-            3 => self.retrieve_node(root_node_id).unwrap(),
-            4 => self.retrieve_node(root_node_id).unwrap(),
-            _ => self.retrieve_node(root_node_id).unwrap(),
-        };
+    //     let root_node_id = self.get_root_node_id();
+    //     let mut node = match self.get_stride_for_id(root_node_id) {
+    //         3 => self.retrieve_node(root_node_id).unwrap(),
+    //         4 => self.retrieve_node(root_node_id).unwrap(),
+    //         _ => self.retrieve_node(root_node_id).unwrap(),
+    //     };
 
-        let mut nibble;
-        let mut nibble_len;
+    //     let mut nibble;
+    //     let mut nibble_len;
 
-        //---- result values ------------------------------------------------
+    //     //---- result values ------------------------------------------------
 
-        // These result values are kept in mutable variables, and assembled
-        // at the end into a QueryResult struct. This proved to result in the
-        // most efficient code, where we don't have to match on
-        // SizedStrideNode over and over. The `match_type` field in the
-        // QueryResult is computed at the end.
+    //     // These result values are kept in mutable variables, and assembled
+    //     // at the end into a QueryResult struct. This proved to result in the
+    //     // most efficient code, where we don't have to match on
+    //     // SizedStrideNode over and over. The `match_type` field in the
+    //     // QueryResult is computed at the end.
 
-        // The final prefix
-        let mut match_prefix_idx: Option<PrefixId<AF>> = None;
+    //     // The final prefix
+    //     let mut match_prefix_idx: Option<PrefixId<AF>> = None;
 
-        // The indexes of the less-specifics
-        let mut less_specifics_vec = if options.include_less_specifics {
-            Some(Vec::<PrefixId<AF>>::new())
-        } else {
-            None
-        };
+    //     // The indexes of the less-specifics
+    //     let mut less_specifics_vec = if options.include_less_specifics {
+    //         Some(Vec::<PrefixId<AF>>::new())
+    //     } else {
+    //         None
+    //     };
 
-        // The indexes of the more-specifics.
-        let mut more_specifics_vec = if options.include_more_specifics {
-            Some(Vec::<PrefixId<AF>>::new())
-        } else {
-            None
-        };
+    //     // The indexes of the more-specifics.
+    //     let mut more_specifics_vec = if options.include_more_specifics {
+    //         Some(Vec::<PrefixId<AF>>::new())
+    //     } else {
+    //         None
+    //     };
 
-        //---- Stride Processing --------------------------------------------
-
-        // We're going to iterate over all the strides in the treebitmap (so
-        // up to the last bit in the max prefix length for that tree). When
-        // a final prefix is found or we get to the end of the strides,
-        // depending on the options.match_type (the type requested by the
-        // user). we ALWAYS break out of the loop. WE ALWAYS BREAK OUT OF THE
-        // LOOP. Just before breaking some processing is done inside the loop
-        // before the break (looking up more-specifics mainly), which looks a
-        // bit repetitious, but again it's been done like that to avoid
-        // having to match over a SizedStrideNode again in the
-        // `post-processing` section.
-
-        for stride in self.get_stride_sizes() {
-            stride_end += stride;
-
-            let last_stride = search_pfx.get_len() < stride_end;
-
-            nibble_len = if last_stride {
-                stride + search_pfx.get_len() - stride_end
-            } else {
-                *stride
-            };
-
-            // Shift left and right to set the bits to zero that are not
-            // in the nibble we're handling here.
-            nibble = AddressFamily::get_nibble(
-                search_pfx.get_net(),
-                stride_end - stride,
-                nibble_len,
-            );
+    //     //---- Stride Processing --------------------------------------------
 
-            match node {
-                SizedStrideRef::Stride3(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-
-                    // This whole match assumes that:
-                    // - if the first value in the return tuple of
-                    //   `search_fn` holds a value, then we need to continue
-                    //   searching by following the node contained in the
-                    //   value.
-                    // - The second value in the tuple holds the prefix that
-                    //   was found.
-                    // The less_specifics_vec is mutated by `search_fn` to
-                    // hold the prefixes found along the way, in the cases
-                    // where `include_less_specifics` was requested by the
-                    // user.
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        // This and the next match will handle all
-                        // intermediary nodes, but they might also handle
-                        // exit nodes.
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx);
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        // This handles exact and longest matches: there are
-                        // no more children, but there is a prefix on this
-                        // node.
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                        StrideNodeId::new_with_cleaned_id(
-                                            search_pfx.get_net(),
-                                            stride_end - stride,
-                                        ),
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx);
-                            break;
-                        }
-                        // This handles cases where there's no prefix (and no
-                        // child) for exact match or longest match, the empty
-                        // match - which doesn't care about actually finding
-                        // a prefix - just continues in search of
-                        // more-specifics.
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    // To make sure we don't process this
-                                    // match arm more then once, we return
-                                    // early here.
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                //---- From here only repetitions for all strides -----------
-                // For comments see the code above for the Stride3 arm.
-                SizedStrideRef::Stride4(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx);
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                        StrideNodeId::new_with_cleaned_id(
-                                            search_pfx.get_net(),
-                                            stride_end - stride,
-                                        ),
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx);
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    // To make sure we don't process this
-                                    // match arm more then once, we return
-                                    // early here.
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                SizedStrideRef::Stride5(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx);
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                        StrideNodeId::new_with_cleaned_id(
-                                            search_pfx.get_net(),
-                                            stride_end - stride,
-                                        ),
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx);
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-            }
-        }
-        //------------------ end of Stride branch arm repetition ------------
-
-        //------------------ post-processing --------------------------------
-
-        // If the above loop finishes (so not hitting a break) we have
-        // processed all strides and have found a child node and maybe a
-        // prefix. Now we will look up more-specifics for longest-matching
-        // prefixes that were found in the last stride only. Note that still
-        // any of the match_types (as specified by the user, not the return
-        // type) may end up here.
-
-        let match_type = if let Some(prefix) = match_prefix_idx {
-            if prefix.get_len() == search_pfx.get_len() {
-                MatchType::ExactMatch
-            } else {
-                MatchType::LongestMatch
-            }
-        } else {
-            MatchType::EmptyMatch
-        };
+    //     // We're going to iterate over all the strides in the treebitmap (so
+    //     // up to the last bit in the max prefix length for that tree). When
+    //     // a final prefix is found or we get to the end of the strides,
+    //     // depending on the options.match_type (the type requested by the
+    //     // user). we ALWAYS break out of the loop. WE ALWAYS BREAK OUT OF THE
+    //     // LOOP. Just before breaking some processing is done inside the loop
+    //     // before the break (looking up more-specifics mainly), which looks a
+    //     // bit repetitious, but again it's been done like that to avoid
+    //     // having to match over a SizedStrideNode again in the
+    //     // `post-processing` section.
 
-        TreeQueryResult {
-            prefix: match_prefix_idx,
-            match_type,
-            less_specifics: if options.include_less_specifics {
-                less_specifics_vec.map(|lsv| {
-                    lsv.into_iter()
-                        .filter(|r| r != &search_pfx)
-                        .collect::<Vec<_>>()
-                })
-            } else {
-                None
-            },
-            more_specifics: if options.include_more_specifics {
-                more_specifics_vec
-            } else {
-                None
-            },
-        }
-    }
+    //     for stride in self.get_stride_sizes() {
+    //         stride_end += stride;
+
+    //         let last_stride = search_pfx.get_len() < stride_end;
+
+    //         nibble_len = if last_stride {
+    //             stride + search_pfx.get_len() - stride_end
+    //         } else {
+    //             *stride
+    //         };
+
+    //         // Shift left and right to set the bits to zero that are not
+    //         // in the nibble we're handling here.
+    //         nibble = AddressFamily::get_nibble(
+    //             search_pfx.get_net(),
+    //             stride_end - stride,
+    //             nibble_len,
+    //         );
+
+    //         match node {
+    //             SizedStrideRef::Stride3(current_node) => {
+    //                 let search_fn = match options.match_type {
+    //                     MatchType::ExactMatch => {
+    //                         if options.include_less_specifics {
+    //                             TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
+    //                         } else {
+    //                             TreeBitMapNode::search_stride_for_exact_match_at
+    //                         }
+    //                     }
+    //                     MatchType::LongestMatch => {
+    //                         TreeBitMapNode::search_stride_for_longest_match_at
+    //                     }
+    //                     MatchType::EmptyMatch => {
+    //                         TreeBitMapNode::search_stride_for_longest_match_at
+    //                     }
+    //                 };
+
+    //                 // This whole match assumes that:
+    //                 // - if the first value in the return tuple of
+    //                 //   `search_fn` holds a value, then we need to continue
+    //                 //   searching by following the node contained in the
+    //                 //   value.
+    //                 // - The second value in the tuple holds the prefix that
+    //                 //   was found.
+    //                 // The less_specifics_vec is mutated by `search_fn` to
+    //                 // hold the prefixes found along the way, in the cases
+    //                 // where `include_less_specifics` was requested by the
+    //                 // user.
+    //                 match search_fn(
+    //                     current_node,
+    //                     search_pfx,
+    //                     nibble,
+    //                     nibble_len,
+    //                     stride_end - stride,
+    //                     &mut less_specifics_vec,
+    //                 ) {
+    //                     // This and the next match will handle all
+    //                     // intermediary nodes, but they might also handle
+    //                     // exit nodes.
+    //                     (Some(n), Some(pfx_idx)) => {
+    //                         match_prefix_idx = Some(pfx_idx);
+    //                         node = self.retrieve_node(n).unwrap();
+
+    //                         if last_stride {
+    //                             if options.include_more_specifics {
+    //                                 more_specifics_vec = self
+    //                                     .get_all_more_specifics_from_nibble(
+    //                                         current_node,
+    //                                         nibble,
+    //                                         nibble_len,
+    //                                         StrideNodeId::new_with_cleaned_id(
+    //                                             search_pfx.get_net(),
+    //                                             stride_end - stride,
+    //                                         ),
+    //                                     );
+    //                             }
+    //                             break;
+    //                         }
+    //                     }
+    //                     (Some(n), None) => {
+    //                         node = self.retrieve_node(n).unwrap();
+
+    //                         if last_stride {
+    //                             if options.include_more_specifics {
+    //                                 more_specifics_vec = self
+    //                                     .get_all_more_specifics_from_nibble(
+    //                                         current_node,
+    //                                         nibble,
+    //                                         nibble_len,
+    //                                         StrideNodeId::new_with_cleaned_id(
+    //                                             search_pfx.get_net(),
+    //                                             stride_end - stride,
+    //                                         ),
+    //                                     );
+    //                             }
+    //                             break;
+    //                         }
+    //                     }
+    //                     // This handles exact and longest matches: there are
+    //                     // no more children, but there is a prefix on this
+    //                     // node.
+    //                     (None, Some(pfx_idx)) => {
+    //                         if options.include_more_specifics {
+    //                             more_specifics_vec = self
+    //                                 .get_all_more_specifics_from_nibble(
+    //                                     current_node,
+    //                                     nibble,
+    //                                     nibble_len,
+    //                                     StrideNodeId::new_with_cleaned_id(
+    //                                         search_pfx.get_net(),
+    //                                         stride_end - stride,
+    //                                     ),
+    //                                 );
+    //                         }
+    //                         match_prefix_idx = Some(pfx_idx);
+    //                         break;
+    //                     }
+    //                     // This handles cases where there's no prefix (and no
+    //                     // child) for exact match or longest match, the empty
+    //                     // match - which doesn't care about actually finding
+    //                     // a prefix - just continues in search of
+    //                     // more-specifics.
+    //                     (None, None) => {
+    //                         match options.match_type {
+    //                             MatchType::EmptyMatch => {
+    //                                 // To make sure we don't process this
+    //                                 // match arm more then once, we return
+    //                                 // early here.
+    //                                 more_specifics_vec = self
+    //                                     .get_all_more_specifics_from_nibble(
+    //                                         current_node,
+    //                                         nibble,
+    //                                         nibble_len,
+    //                                         StrideNodeId::new_with_cleaned_id(
+    //                                             search_pfx.get_net(),
+    //                                             stride_end - stride,
+    //                                         ),
+    //                                     );
+
+    //                                 match_prefix_idx = None;
+    //                                 break;
+    //                             }
+    //                             MatchType::LongestMatch => {}
+    //                             MatchType::ExactMatch => {
+    //                                 match_prefix_idx = None;
+    //                             }
+    //                         }
+    //                         break;
+    //                     }
+    //                 }
+    //             }
+    //             //---- From here only repetitions for all strides -----------
+    //             // For comments see the code above for the Stride3 arm.
+    //             SizedStrideRef::Stride4(current_node) => {
+    //                 let search_fn = match options.match_type {
+    //                     MatchType::ExactMatch => {
+    //                         if options.include_less_specifics {
+    //                             TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
+    //                         } else {
+    //                             TreeBitMapNode::search_stride_for_exact_match_at
+    //                         }
+    //                     }
+    //                     MatchType::LongestMatch => {
+    //                         TreeBitMapNode::search_stride_for_longest_match_at
+    //                     }
+    //                     MatchType::EmptyMatch => {
+    //                         TreeBitMapNode::search_stride_for_longest_match_at
+    //                     }
+    //                 };
+    //                 match search_fn(
+    //                     current_node,
+    //                     search_pfx,
+    //                     nibble,
+    //                     nibble_len,
+    //                     stride_end - stride,
+    //                     &mut less_specifics_vec,
+    //                 ) {
+    //                     (Some(n), Some(pfx_idx)) => {
+    //                         match_prefix_idx = Some(pfx_idx);
+    //                         node = self.retrieve_node(n).unwrap();
+
+    //                         if last_stride {
+    //                             if options.include_more_specifics {
+    //                                 more_specifics_vec = self
+    //                                     .get_all_more_specifics_from_nibble(
+    //                                         current_node,
+    //                                         nibble,
+    //                                         nibble_len,
+    //                                         StrideNodeId::new_with_cleaned_id(
+    //                                             search_pfx.get_net(),
+    //                                             stride_end - stride,
+    //                                         ),
+    //                                     );
+    //                             }
+    //                             break;
+    //                         }
+    //                     }
+    //                     (Some(n), None) => {
+    //                         node = self.retrieve_node(n).unwrap();
+
+    //                         if last_stride {
+    //                             if options.include_more_specifics {
+    //                                 more_specifics_vec = self
+    //                                     .get_all_more_specifics_from_nibble(
+    //                                         current_node,
+    //                                         nibble,
+    //                                         nibble_len,
+    //                                         StrideNodeId::new_with_cleaned_id(
+    //                                             search_pfx.get_net(),
+    //                                             stride_end - stride,
+    //                                         ),
+    //                                     );
+    //                             }
+    //                             break;
+    //                         }
+    //                     }
+    //                     (None, Some(pfx_idx)) => {
+    //                         if options.include_more_specifics {
+    //                             more_specifics_vec = self
+    //                                 .get_all_more_specifics_from_nibble(
+    //                                     current_node,
+    //                                     nibble,
+    //                                     nibble_len,
+    //                                     StrideNodeId::new_with_cleaned_id(
+    //                                         search_pfx.get_net(),
+    //                                         stride_end - stride,
+    //                                     ),
+    //                                 );
+    //                         }
+    //                         match_prefix_idx = Some(pfx_idx);
+    //                         break;
+    //                     }
+    //                     (None, None) => {
+    //                         match options.match_type {
+    //                             MatchType::EmptyMatch => {
+    //                                 // To make sure we don't process this
+    //                                 // match arm more then once, we return
+    //                                 // early here.
+    //                                 more_specifics_vec = self
+    //                                     .get_all_more_specifics_from_nibble(
+    //                                         current_node,
+    //                                         nibble,
+    //                                         nibble_len,
+    //                                         StrideNodeId::new_with_cleaned_id(
+    //                                             search_pfx.get_net(),
+    //                                             stride_end - stride,
+    //                                         ),
+    //                                     );
+
+    //                                 match_prefix_idx = None;
+    //                                 break;
+    //                             }
+    //                             MatchType::LongestMatch => {}
+    //                             MatchType::ExactMatch => {
+    //                                 match_prefix_idx = None;
+    //                             }
+    //                         }
+    //                         break;
+    //                     }
+    //                 }
+    //             }
+    //             SizedStrideRef::Stride5(current_node) => {
+    //                 let search_fn = match options.match_type {
+    //                     MatchType::ExactMatch => {
+    //                         if options.include_less_specifics {
+    //                             TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
+    //                         } else {
+    //                             TreeBitMapNode::search_stride_for_exact_match_at
+    //                         }
+    //                     }
+    //                     MatchType::LongestMatch => {
+    //                         TreeBitMapNode::search_stride_for_longest_match_at
+    //                     }
+    //                     MatchType::EmptyMatch => {
+    //                         TreeBitMapNode::search_stride_for_longest_match_at
+    //                     }
+    //                 };
+    //                 match search_fn(
+    //                     current_node,
+    //                     search_pfx,
+    //                     nibble,
+    //                     nibble_len,
+    //                     stride_end - stride,
+    //                     &mut less_specifics_vec,
+    //                 ) {
+    //                     (Some(n), Some(pfx_idx)) => {
+    //                         match_prefix_idx = Some(pfx_idx);
+    //                         node = self.retrieve_node(n).unwrap();
+
+    //                         if last_stride {
+    //                             if options.include_more_specifics {
+    //                                 more_specifics_vec = self
+    //                                     .get_all_more_specifics_from_nibble(
+    //                                         current_node,
+    //                                         nibble,
+    //                                         nibble_len,
+    //                                         StrideNodeId::new_with_cleaned_id(
+    //                                             search_pfx.get_net(),
+    //                                             stride_end - stride,
+    //                                         ),
+    //                                     );
+    //                             }
+    //                             break;
+    //                         }
+    //                     }
+    //                     (Some(n), None) => {
+    //                         node = self.retrieve_node(n).unwrap();
+
+    //                         if last_stride {
+    //                             if options.include_more_specifics {
+    //                                 more_specifics_vec = self
+    //                                     .get_all_more_specifics_from_nibble(
+    //                                         current_node,
+    //                                         nibble,
+    //                                         nibble_len,
+    //                                         StrideNodeId::new_with_cleaned_id(
+    //                                             search_pfx.get_net(),
+    //                                             stride_end - stride,
+    //                                         ),
+    //                                     );
+    //                             }
+    //                             break;
+    //                         }
+    //                     }
+    //                     (None, Some(pfx_idx)) => {
+    //                         if options.include_more_specifics {
+    //                             more_specifics_vec = self
+    //                                 .get_all_more_specifics_from_nibble(
+    //                                     current_node,
+    //                                     nibble,
+    //                                     nibble_len,
+    //                                     StrideNodeId::new_with_cleaned_id(
+    //                                         search_pfx.get_net(),
+    //                                         stride_end - stride,
+    //                                     ),
+    //                                 );
+    //                         }
+    //                         match_prefix_idx = Some(pfx_idx);
+    //                         break;
+    //                     }
+    //                     (None, None) => {
+    //                         match options.match_type {
+    //                             MatchType::EmptyMatch => {
+    //                                 more_specifics_vec = self
+    //                                     .get_all_more_specifics_from_nibble(
+    //                                         current_node,
+    //                                         nibble,
+    //                                         nibble_len,
+    //                                         StrideNodeId::new_with_cleaned_id(
+    //                                             search_pfx.get_net(),
+    //                                             stride_end - stride,
+    //                                         ),
+    //                                     );
+
+    //                                 match_prefix_idx = None;
+    //                                 break;
+    //                             }
+    //                             MatchType::LongestMatch => {}
+    //                             MatchType::ExactMatch => {
+    //                                 match_prefix_idx = None;
+    //                             }
+    //                         }
+    //                         break;
+    //                     }
+    //                 }
+    //             }
+    //         }
+    //     }
+    //     //------------------ end of Stride branch arm repetition ------------
+
+    //     //------------------ post-processing --------------------------------
+
+    //     // If the above loop finishes (so not hitting a break) we have
+    //     // processed all strides and have found a child node and maybe a
+    //     // prefix. Now we will look up more-specifics for longest-matching
+    //     // prefixes that were found in the last stride only. Note that still
+    //     // any of the match_types (as specified by the user, not the return
+    //     // type) may end up here.
+
+    //     let match_type = if let Some(prefix) = match_prefix_idx {
+    //         if prefix.get_len() == search_pfx.get_len() {
+    //             MatchType::ExactMatch
+    //         } else {
+    //             MatchType::LongestMatch
+    //         }
+    //     } else {
+    //         MatchType::EmptyMatch
+    //     };
+
+    //     TreeQueryResult {
+    //         prefix: match_prefix_idx,
+    //         match_type,
+    //         less_specifics: if options.include_less_specifics {
+    //             less_specifics_vec.map(|lsv| {
+    //                 lsv.into_iter()
+    //                     .filter(|r| r != &search_pfx)
+    //                     .collect::<Vec<_>>()
+    //             })
+    //         } else {
+    //             None
+    //         },
+    //         more_specifics: if options.include_more_specifics {
+    //             more_specifics_vec
+    //         } else {
+    //             None
+    //         },
+    //     }
+    // }
 }
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 4ae9ac51..fb16966c 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -187,7 +187,6 @@ use crate::local_array::bit_span::BitSpan;
 use crate::local_array::in_memory::atomic_types::{NodeSet, StoredNode};
 use crate::prelude::multi::PrefixId;
 use crossbeam_epoch::{Atomic, Guard};
-use inetnum::addr::Prefix;
 use log::{debug, error, log_enabled, trace};
 use roaring::RoaringBitmap;
 
@@ -390,176 +389,198 @@ impl<
         }
     }
 
-    pub fn prefix_exists_legacy(&self, prefix_id: PrefixId<AF>) -> bool {
-        // if prefix_id.get_len() == 0 {
-        //     return self.update_default_route_prefix_meta(mui);
-        // }
-        trace!("exists {}?", Prefix::from(prefix_id));
-        if prefix_id.get_len() == 0 {
-            return false;
-        }
-
-        let mut stride_end: u8 = 0;
-        let mut cur_i = self.get_root_node_id();
-        let mut level: u8 = 0;
-
-        loop {
-            let stride = self.get_stride_sizes()[level as usize];
-            stride_end += stride;
-            trace!("stride end {}", stride_end);
-            assert!(stride_end <= AF::BITS);
-            let nibble_len = if prefix_id.get_len() < stride_end {
-                stride + prefix_id.get_len() - stride_end
-            } else {
-                stride
-            };
-
-            let nibble = AF::get_nibble(
-                prefix_id.get_net(),
-                stride_end - stride,
-                nibble_len,
-            );
-            let stride_start = stride_end - stride;
-
-            let node_result = match self.retrieve_node(cur_i) {
-                Some(node) => match node {
-                    SizedStrideRef::Stride3(n) => {
-                        let (child_node, found) = n.prefix_exists_in_stride(
-                            prefix_id,
-                            nibble,
-                            nibble_len,
-                            stride_start,
-                        );
-                        if found {
-                            return true;
-                        } else {
-                            child_node
-                        }
-                    }
-                    SizedStrideRef::Stride4(n) => {
-                        let (child_node, found) = n.prefix_exists_in_stride(
-                            prefix_id,
-                            nibble,
-                            nibble_len,
-                            stride_start,
-                        );
-                        if found {
-                            return true;
-                        } else {
-                            child_node
-                        }
-                    }
-                    SizedStrideRef::Stride5(n) => {
-                        let (child_node, found) = n.prefix_exists_in_stride(
-                            prefix_id,
-                            nibble,
-                            nibble_len,
-                            stride_start,
-                        );
-                        if found {
-                            return true;
-                        } else {
-                            child_node
-                        }
-                    }
-                },
-                None => {
-                    return false;
-                }
-            };
-
-            if let Some(next_id) = node_result {
-                cur_i = next_id;
-                level += 1;
-                continue;
-            }
-
-            return false;
-        }
-    }
-
     pub fn prefix_exists_for_mui(
         &self,
         prefix_id: PrefixId<AF>,
         mui: u32,
     ) -> bool {
-        // if prefix_id.get_len() == 0 {
-        //     return self.update_default_route_prefix_meta(mui);
-        // }
-
-        let mut stride_end: u8 = 0;
-        let mut cur_i = self.get_root_node_id();
-        let mut level: u8 = 0;
-
-        loop {
-            let stride = self.get_stride_sizes()[level as usize];
-            stride_end += stride;
-            let nibble_len = if prefix_id.get_len() < stride_end {
-                stride + prefix_id.get_len() - stride_end
-            } else {
-                stride
-            };
-
-            let nibble = AF::get_nibble(
-                prefix_id.get_net(),
-                stride_end - stride,
-                nibble_len,
-            );
-            let stride_start = stride_end - stride;
-
-            let node_result = match self.retrieve_node_for_mui(cur_i, mui) {
-                Some(node) => match node {
-                    SizedStrideRef::Stride3(n) => {
-                        let (child_node, found) = n.prefix_exists_in_stride(
-                            prefix_id,
-                            nibble,
-                            nibble_len,
-                            stride_start,
-                        );
-                        if found {
-                            return true;
-                        } else {
-                            child_node
-                        }
-                    }
-                    SizedStrideRef::Stride4(n) => {
-                        let (child_node, found) = n.prefix_exists_in_stride(
-                            prefix_id,
-                            nibble,
-                            nibble_len,
-                            stride_start,
-                        );
-                        if found {
-                            return true;
-                        } else {
-                            child_node
-                        }
-                    }
-                    SizedStrideRef::Stride5(n) => {
-                        let (child_node, found) = n.prefix_exists_in_stride(
-                            prefix_id,
-                            nibble,
-                            nibble_len,
-                            stride_start,
-                        );
-                        if found {
-                            return true;
-                        } else {
-                            child_node
-                        }
-                    }
-                },
-                None => {
-                    return false;
-                }
-            };
+        trace!("pe exists {:?}?", prefix_id);
+        let (node_id, bs) = self.get_node_id_for_prefix(&prefix_id);
 
-            if let Some(next_id) = node_result {
-                cur_i = next_id;
-                level += 1;
+        match self.retrieve_node_for_mui(node_id, mui) {
+            Some(SizedStrideRef::Stride4(n)) => {
+                let pfxbitarr = n.pfxbitarr.load();
+                pfxbitarr & Stride4::get_bit_pos(bs.bits, bs.len) > 0
+            }
+            None => false,
+            Some(n) => {
+                panic!(
+                    "unsupported stride length: {:?} node_id {}",
+                    n, node_id
+                );
             }
         }
     }
+    // pub fn prefix_exists_legacy(&self, prefix_id: PrefixId<AF>) -> bool {
+    //     // if prefix_id.get_len() == 0 {
+    //     //     return self.update_default_route_prefix_meta(mui);
+    //     // }
+    //     trace!("exists {}?", Prefix::from(prefix_id));
+    //     if prefix_id.get_len() == 0 {
+    //         return false;
+    //     }
+
+    //     let mut stride_end: u8 = 0;
+    //     let mut cur_i = self.get_root_node_id();
+    //     let mut level: u8 = 0;
+
+    //     loop {
+    //         let stride = self.get_stride_sizes()[level as usize];
+    //         stride_end += stride;
+    //         trace!("stride end {}", stride_end);
+    //         assert!(stride_end <= AF::BITS);
+    //         let nibble_len = if prefix_id.get_len() < stride_end {
+    //             stride + prefix_id.get_len() - stride_end
+    //         } else {
+    //             stride
+    //         };
+
+    //         let nibble = AF::get_nibble(
+    //             prefix_id.get_net(),
+    //             stride_end - stride,
+    //             nibble_len,
+    //         );
+    //         let stride_start = stride_end - stride;
+
+    //         let node_result = match self.retrieve_node(cur_i) {
+    //             Some(node) => match node {
+    //                 SizedStrideRef::Stride3(n) => {
+    //                     let (child_node, found) = n.prefix_exists_in_stride(
+    //                         prefix_id,
+    //                         nibble,
+    //                         nibble_len,
+    //                         stride_start,
+    //                     );
+    //                     if found {
+    //                         return true;
+    //                     } else {
+    //                         child_node
+    //                     }
+    //                 }
+    //                 SizedStrideRef::Stride4(n) => {
+    //                     let (child_node, found) = n.prefix_exists_in_stride(
+    //                         prefix_id,
+    //                         nibble,
+    //                         nibble_len,
+    //                         stride_start,
+    //                     );
+    //                     if found {
+    //                         return true;
+    //                     } else {
+    //                         child_node
+    //                     }
+    //                 }
+    //                 SizedStrideRef::Stride5(n) => {
+    //                     let (child_node, found) = n.prefix_exists_in_stride(
+    //                         prefix_id,
+    //                         nibble,
+    //                         nibble_len,
+    //                         stride_start,
+    //                     );
+    //                     if found {
+    //                         return true;
+    //                     } else {
+    //                         child_node
+    //                     }
+    //                 }
+    //             },
+    //             None => {
+    //                 return false;
+    //             }
+    //         };
+
+    //         if let Some(next_id) = node_result {
+    //             cur_i = next_id;
+    //             level += 1;
+    //             continue;
+    //         }
+
+    //         return false;
+    //     }
+    // }
+
+    // pub fn prefix_exists_for_mui(
+    //     &self,
+    //     prefix_id: PrefixId<AF>,
+    //     mui: u32,
+    // ) -> bool {
+    //     // if prefix_id.get_len() == 0 {
+    //     //     return self.update_default_route_prefix_meta(mui);
+    //     // }
+
+    //     let mut stride_end: u8 = 0;
+    //     let mut cur_i = self.get_root_node_id();
+    //     let mut level: u8 = 0;
+
+    //     loop {
+    //         let stride = self.get_stride_sizes()[level as usize];
+    //         stride_end += stride;
+    //         let nibble_len = if prefix_id.get_len() < stride_end {
+    //             stride + prefix_id.get_len() - stride_end
+    //         } else {
+    //             stride
+    //         };
+
+    //         let nibble = AF::get_nibble(
+    //             prefix_id.get_net(),
+    //             stride_end - stride,
+    //             nibble_len,
+    //         );
+    //         let stride_start = stride_end - stride;
+
+    //         let node_result = match self.retrieve_node_for_mui(cur_i, mui) {
+    //             Some(node) => match node {
+    //                 SizedStrideRef::Stride3(n) => {
+    //                     let (child_node, found) = n.prefix_exists_in_stride(
+    //                         prefix_id,
+    //                         nibble,
+    //                         nibble_len,
+    //                         stride_start,
+    //                     );
+    //                     if found {
+    //                         return true;
+    //                     } else {
+    //                         child_node
+    //                     }
+    //                 }
+    //                 SizedStrideRef::Stride4(n) => {
+    //                     let (child_node, found) = n.prefix_exists_in_stride(
+    //                         prefix_id,
+    //                         nibble,
+    //                         nibble_len,
+    //                         stride_start,
+    //                     );
+    //                     if found {
+    //                         return true;
+    //                     } else {
+    //                         child_node
+    //                     }
+    //                 }
+    //                 SizedStrideRef::Stride5(n) => {
+    //                     let (child_node, found) = n.prefix_exists_in_stride(
+    //                         prefix_id,
+    //                         nibble,
+    //                         nibble_len,
+    //                         stride_start,
+    //                     );
+    //                     if found {
+    //                         return true;
+    //                     } else {
+    //                         child_node
+    //                     }
+    //                 }
+    //             },
+    //             None => {
+    //                 return false;
+    //             }
+    //         };
+
+    //         if let Some(next_id) = node_result {
+    //             cur_i = next_id;
+    //             level += 1;
+    //         }
+    //     }
+    // }
 
     // Yes, we're hating this. But, the root node has no room for a serial of
     // the prefix 0/0 (the default route), which doesn't even matter, unless,
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
index 1b1861e8..2fec77f9 100644
--- a/src/local_array/prefix_cht/iterators.rs
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -54,87 +54,6 @@ impl<
 
             if next_pfx.is_some() {
                 return next_pfx;
-                // If we have a mui, we have to deal slightly different with
-                // the records: There can only be one record for a (prefix,
-                // mui) combination, and the record may be filtered out by the
-                // global status of the mui, or its local status. In that case
-                // we don't return here (because that would result in a Prefix
-                // with an empty record vec).
-                // next_pfx
-                // if let Some(mui) = self.mui {
-                //     if let Some(p) = self
-                //         .store
-                //         .non_recursive_retrieve_prefix(
-                //             next_pfx.unwrap_or_else(|| {
-                //                 panic!(
-                //                 "BOOM! More-specific prefix {:?} disappeared \
-                //                 from the store",
-                //                 next_pfx
-                //             )
-                //             }),
-                //         )
-                //         .0
-                //     {
-                //         // We may either have to rewrite the local status with
-                //         // the provided global status OR we may have to omit
-                //         // all of the records with either global of local
-                //         // withdrawn status.
-                //         if self.include_withdrawn {
-                //             if let Some(rec) = p
-                //                 .record_map
-                //                 .get_record_for_mui_with_rewritten_status(
-                //                     mui,
-                //                     self.global_withdrawn_bmin,
-                //                     RouteStatus::Withdrawn,
-                //                 )
-                //             {
-                //                 return Some((p.prefix, vec![rec]));
-                //             }
-                //         } else if let Some(rec) = p
-                //             .record_map
-                //             .get_record_for_mui(mui, self.include_withdrawn)
-                //         {
-                //             return Some((p.prefix, vec![rec]));
-                //         }
-                //     };
-                // } else {
-                //     return self
-                //         .store
-                //         .non_recursive_retrieve_prefix(
-                //             next_pfx.unwrap_or_else(|| {
-                //                 panic!(
-                //                 "BOOM! More-specific prefix {:?} disappeared \
-                //                 from the store",
-                //                 next_pfx
-                //             )
-                //             }),
-                //             // self.guard,
-                //         )
-                //         .0
-                //         .map(|p| {
-                //             // Just like the mui specific records, we may have
-                //             // to either rewrite the local status (if the user
-                //             // wants the withdrawn records) or omit them.
-                //             if self.include_withdrawn {
-                //                 (
-                //                     p.prefix,
-                //                     p.record_map
-                //                         .as_records_with_rewritten_status(
-                //                             self.global_withdrawn_bmin,
-                //                             RouteStatus::Withdrawn,
-                //                         ),
-                //                 )
-                //             } else {
-                //                 (
-                //                     p.prefix,
-                //                     p.record_map
-                //                         .as_active_records_not_in_bmin(
-                //                             self.global_withdrawn_bmin,
-                //                         ),
-                //                 )
-                //             }
-                //         });
-                // }
             }
 
             // Our current prefix iterator for this node is done, look for
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 3cd108d1..cf20aa05 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -1,16 +1,16 @@
 use crossbeam_epoch::{self as epoch};
 use epoch::Guard;
-use log::{log_enabled, trace};
+use log::trace;
 
 use crate::af::AddressFamily;
 use crate::local_array::in_memory::atomic_types::{
     NodeBuckets, PrefixBuckets,
 };
 use crate::rib::{PersistStrategy, Rib};
-use crate::{PublicRecord, RecordSet};
+use crate::PublicRecord;
 use inetnum::addr::Prefix;
 
-use crate::{IncludeHistory, Meta, QueryResult};
+use crate::{Meta, QueryResult};
 
 use crate::{MatchOptions, MatchType};
 
@@ -27,7 +27,7 @@ where
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
 {
-    pub fn get_value(
+    pub(crate) fn get_value(
         &'a self,
         prefix_id: PrefixId<AF>,
         mui: Option<u32>,
@@ -54,7 +54,7 @@ where
         }
     }
 
-    pub fn more_specifics_from(
+    pub(crate) fn more_specifics_from(
         &'a self,
         prefix_id: PrefixId<AF>,
         mui: Option<u32>,
@@ -89,7 +89,7 @@ where
         }
     }
 
-    pub fn less_specifics_from(
+    pub(crate) fn less_specifics_from(
         &'a self,
         prefix_id: PrefixId<AF>,
         mui: Option<u32>,
@@ -122,15 +122,7 @@ where
         }
     }
 
-    // pub fn more_specifics_keys_from(
-    //     &self,
-    //     prefix_id: PrefixId<AF>,
-    // ) -> impl Iterator<Item = PrefixId<AF>> + '_ {
-    //     self.in_memory_tree
-    //         .more_specific_prefix_iter_from(prefix_id)
-    // }
-
-    pub fn more_specifics_iter_from(
+    pub(crate) fn more_specifics_iter_from(
         &'a self,
         prefix_id: PrefixId<AF>,
         mui: Option<u32>,
@@ -180,59 +172,22 @@ where
         )
     }
 
-    pub fn less_specifics_iter_from(
+    pub(crate) fn less_specifics_iter_from(
         &'a self,
         prefix_id: PrefixId<AF>,
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-        match self.config.persist_strategy() {
-            PersistStrategy::MemoryOnly
-            | PersistStrategy::WriteAhead
-            | PersistStrategy::PersistHistory => {
-                // if mui.is_some_and(|m| self.mui_is_withdrawn(m, guard)) {
-                //     return None;
-                // }
-
-                self.in_memory_tree
-                    .less_specific_prefix_iter(prefix_id)
-                    .filter_map(move |p| {
-                        self.get_value(p, mui, include_withdrawn, guard)
-                            .map(|v| (p, v))
-                    })
-
-                // (if mui.is_some_and(|m| self.mui_is_withdrawn(m, guard)) {
-                //     None
-                // } else {
-                //     Some(
-                //         self.in_memory_tree
-                //             .less_specific_prefix_iter(
-                //                 prefix_id,
-                //                 // mui,
-                //                 // include_withdrawn,
-                //                 // guard,
-                //             )
-                //             .map(|p| {
-                //                 self.get_value(
-                //                     p,
-                //                     mui.clone(),
-                //                     include_withdrawn,
-                //                     guard,
-                //                 )
-                //                 .map(|v| (p, v))
-                //                 .unwrap()
-                //             }),
-                //     )
-                // })
-                // .into_iter()
-                // .flatten()
-            }
-            PersistStrategy::PersistOnly => unimplemented!(),
-        }
+        self.in_memory_tree
+            .less_specific_prefix_iter(prefix_id)
+            .filter_map(move |p| {
+                self.get_value(p, mui, include_withdrawn, guard)
+                    .map(|v| (p, v))
+            })
     }
 
-    pub fn match_prefix(
+    pub(crate) fn match_prefix(
         &'a self,
         search_pfx: PrefixId<AF>,
         options: &MatchOptions,
@@ -244,15 +199,6 @@ where
         trace!("res {:?}", res);
         let mut res = QueryResult::from(res);
 
-        if log_enabled!(log::Level::Trace) {
-            let ms = self
-                .in_memory_tree
-                .more_specific_prefix_iter_from(search_pfx)
-                .collect::<Vec<_>>();
-            trace!("more specifics!!! {:?}", ms);
-            trace!("len {}", ms.len());
-        }
-
         if let Some(Some(m)) = res.prefix.map(|p| {
             self.get_value(
                 p.into(),
@@ -262,7 +208,6 @@ where
             )
             .and_then(|v| if v.is_empty() { None } else { Some(v) })
         }) {
-            trace!("got value {:?}", m);
             res.prefix_meta = m;
         } else {
             res.prefix = None;
@@ -311,256 +256,7 @@ where
         res
     }
 
-    pub fn match_prefix_legacy(
-        &'a self,
-        search_pfx: PrefixId<AF>,
-        options: &MatchOptions,
-        guard: &'a Guard,
-    ) -> QueryResult<M> {
-        match self.config.persist_strategy() {
-            // Everything is in memory only, so look there. There can be no
-            // historical records for this variant, so we just return the
-            // stuff found in memeory. A request for  historical records
-            // willbe ignored.
-            PersistStrategy::MemoryOnly => {
-                let mut res: QueryResult<M> = self
-                    .in_memory_tree
-                    .match_prefix_by_tree_traversal(search_pfx, options)
-                    .into();
-
-                res.prefix_meta = res
-                    .prefix
-                    .map(|p| {
-                        self.get_value(
-                            p.into(),
-                            options.mui,
-                            options.include_withdrawn,
-                            guard,
-                        )
-                        .unwrap_or_default()
-                    })
-                    .unwrap_or_default();
-                res
-            }
-            // All the records are persisted, they have never been committed
-            // to memory. However the in-memory-tree is still used to indicate
-            // which (prefix, mui) tuples have been created.
-            PersistStrategy::PersistOnly => {
-                // If no withdawn should be included and a specific mui was
-                // requested, then return early, if the mui lives in the
-                // global withdrawn muis.
-                if !options.include_withdrawn {
-                    if let Some(mui) = options.mui {
-                        let withdrawn_muis_bmin =
-                            self.in_memory_tree.withdrawn_muis_bmin(guard);
-                        if withdrawn_muis_bmin.contains(mui) {
-                            return QueryResult::empty();
-                        }
-                    }
-                }
-
-                // if options.match_type == MatchType::ExactMatch
-                //     && !self.contains(search_pfx, options.mui)
-                // {
-                //     return QueryResult::empty();
-                // }
-
-                if let Some(persist_tree) = &self.persist_tree {
-                    let withdrawn_muis_bmin =
-                        self.in_memory_tree.withdrawn_muis_bmin(guard);
-                    println!("persist store found");
-                    println!("mem record {:#?}", search_pfx);
-                    let tbm_result = self
-                        .in_memory_tree
-                        .match_prefix_by_tree_traversal(search_pfx, options);
-                    println!("found by traversal {:#?}", tbm_result);
-
-                    persist_tree
-                        .match_prefix(
-                            tbm_result,
-                            options,
-                            withdrawn_muis_bmin,
-                        )
-                        .into()
-                } else {
-                    println!("no persist store");
-                    QueryResult::empty()
-                }
-            }
-            // We have the current records in memory, additionally they may be
-            // encriched with historical data from the persisted data.
-            PersistStrategy::WriteAhead => {
-                let mut res: FamilyQueryResult<AF, M> = self
-                    .in_memory_tree
-                    .match_prefix_by_tree_traversal(search_pfx, options)
-                    .into();
-                if let Some(persist_tree) = &self.persist_tree {
-                    match options.include_history {
-                        IncludeHistory::All => {
-                            if self.contains(search_pfx, options.mui) {
-                                res.prefix_meta.extend(
-                                    persist_tree.get_records_for_prefix(
-                                        search_pfx,
-                                        options.mui,
-                                        options.include_withdrawn,
-                                        self.in_memory_tree
-                                            .withdrawn_muis_bmin(guard),
-                                    ),
-                                );
-                            }
-
-                            if let Some(ls) = &mut res.less_specifics {
-                                for rec in ls {
-                                    if self.contains(rec.0, options.mui) {
-                                        // write ahead has the current record,
-                                        // as well as the historical ones, so
-                                        // we just override the recs from the
-                                        // in-memory tree.
-                                        rec.1 = persist_tree
-                                            .get_records_for_prefix(
-                                                rec.0,
-                                                options.mui,
-                                                options.include_withdrawn,
-                                                self.in_memory_tree
-                                                    .withdrawn_muis_bmin(
-                                                        guard,
-                                                    ),
-                                            );
-                                    }
-                                }
-                            };
-
-                            if let Some(rs) = &mut res.more_specifics {
-                                for rec in rs {
-                                    if self.contains(rec.0, options.mui) {
-                                        rec.1 = persist_tree
-                                            .get_records_for_prefix(
-                                                rec.0,
-                                                options.mui,
-                                                options.include_withdrawn,
-                                                self.in_memory_tree
-                                                    .withdrawn_muis_bmin(
-                                                        guard,
-                                                    ),
-                                            );
-                                    }
-                                }
-                            };
-
-                            res.into()
-                        }
-                        IncludeHistory::SearchPrefix => {
-                            if self.contains(search_pfx, options.mui) {
-                                res.prefix_meta.extend(
-                                    persist_tree.get_records_for_prefix(
-                                        search_pfx,
-                                        options.mui,
-                                        options.include_withdrawn,
-                                        self.in_memory_tree
-                                            .withdrawn_muis_bmin(guard),
-                                    ),
-                                );
-                            }
-
-                            res.into()
-                        }
-                        IncludeHistory::None => res.into(),
-                    }
-                } else {
-                    res.into()
-                }
-            }
-            // All current info is in memory so look there. If the user has
-            // requested historical records, we will ask the persist_tree to
-            // add those to the intermedidate result of the in-memory query.
-            PersistStrategy::PersistHistory => {
-                let mut res: FamilyQueryResult<AF, M> = self
-                    .in_memory_tree
-                    .match_prefix_by_tree_traversal(search_pfx, options)
-                    .into();
-
-                if let Some(persist_tree) = &self.persist_tree {
-                    match options.include_history {
-                        IncludeHistory::All => {
-                            res.prefix_meta.extend(
-                                persist_tree.get_records_for_prefix(
-                                    search_pfx,
-                                    options.mui,
-                                    options.include_withdrawn,
-                                    self.in_memory_tree
-                                        .withdrawn_muis_bmin(guard),
-                                ),
-                            );
-
-                            if let Some(ls) = &mut res.less_specifics {
-                                for rec in ls {
-                                    if self.contains(rec.0, options.mui) {
-                                        // the persisted tree only has the
-                                        // historical records, so we extend
-                                        // the vec. It should already hold the
-                                        // current record.
-                                        rec.1.extend(
-                                            persist_tree
-                                                .get_records_for_prefix(
-                                                    rec.0,
-                                                    options.mui,
-                                                    options.include_withdrawn,
-                                                    self.in_memory_tree
-                                                        .withdrawn_muis_bmin(
-                                                            guard,
-                                                        ),
-                                                ),
-                                        );
-                                    }
-                                }
-                            };
-
-                            if let Some(rs) = &mut res.more_specifics {
-                                for rec in rs {
-                                    if self.contains(rec.0, options.mui) {
-                                        rec.1.extend(
-                                            persist_tree
-                                                .get_records_for_prefix(
-                                                    rec.0,
-                                                    options.mui,
-                                                    options.include_withdrawn,
-                                                    self.in_memory_tree
-                                                        .withdrawn_muis_bmin(
-                                                            guard,
-                                                        ),
-                                                ),
-                                        );
-                                    }
-                                }
-                            };
-
-                            res.into()
-                        }
-                        IncludeHistory::SearchPrefix => {
-                            if self.contains(search_pfx, options.mui) {
-                                res.prefix_meta.extend(
-                                    persist_tree.get_records_for_prefix(
-                                        search_pfx,
-                                        options.mui,
-                                        options.include_withdrawn,
-                                        self.in_memory_tree
-                                            .withdrawn_muis_bmin(guard),
-                                    ),
-                                );
-                            }
-
-                            res.into()
-                        }
-                        IncludeHistory::None => res.into(),
-                    }
-                } else {
-                    res.into()
-                }
-            }
-        }
-    }
-
-    pub fn best_path(
+    pub(crate) fn best_path(
         &'a self,
         search_pfx: PrefixId<AF>,
         guard: &Guard,
@@ -581,7 +277,7 @@ where
             })
     }
 
-    pub fn calculate_and_store_best_and_backup_path(
+    pub(crate) fn calculate_and_store_best_and_backup_path(
         &self,
         search_pfx: PrefixId<AF>,
         tbi: &<M as Meta>::TBI,
@@ -595,7 +291,7 @@ where
             })
     }
 
-    pub fn is_ps_outdated(
+    pub(crate) fn is_ps_outdated(
         &self,
         search_pfx: PrefixId<AF>,
         guard: &Guard,
@@ -648,6 +344,7 @@ impl<AF: AddressFamily, M: Meta> From<TreeQueryResult<AF>>
         }
     }
 }
+
 pub(crate) type FamilyRecord<AF, M> =
     Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>;
 
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index c8297f0f..2e180197 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -3,7 +3,7 @@ use std::path::Path;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use inetnum::addr::Prefix;
-use log::{info, trace};
+use log::info;
 
 use crossbeam_epoch::{self as epoch};
 use epoch::{Guard, Owned};
@@ -698,17 +698,6 @@ impl<
                     .unwrap_or_default(),
             )
         })
-
-        // let pt_iter = match self.config.persist_strategy {
-        //     PersistStrategy::PersistOnly => {
-        //         self.persist_tree.as_ref().map(|t| t.prefixes_iter())
-        //     }
-        //     _ => None,
-        // }
-        // .into_iter()
-        // .flatten();
-
-        // in_mem_iter.chain(pt_iter)
     }
 
     //-------- Persistence ---------------------------------------------------
@@ -717,38 +706,38 @@ impl<
         self.config.persist_strategy
     }
 
-    pub fn get_records_for_prefix(
-        &self,
-        prefix: &Prefix,
-        mui: Option<u32>,
-        include_withdrawn: bool,
-    ) -> Vec<PublicRecord<M>> {
-        trace!("get records for prefix in the right store");
-        let guard = epoch::pin();
-        match self.persist_strategy() {
-            PersistStrategy::PersistOnly => self
-                .persist_tree
-                .as_ref()
-                .map(|tree| {
-                    tree.get_records_for_prefix(
-                        PrefixId::from(*prefix),
-                        mui,
-                        include_withdrawn,
-                        self.in_memory_tree.withdrawn_muis_bmin(&guard),
-                    )
-                })
-                .unwrap_or_default(),
-            _ => self
-                .prefix_cht
-                .get_records_for_prefix(
-                    PrefixId::from(*prefix),
-                    mui,
-                    include_withdrawn,
-                    self.in_memory_tree.withdrawn_muis_bmin(&guard),
-                )
-                .unwrap_or_default(),
-        }
-    }
+    // pub fn get_records_for_prefix(
+    //     &self,
+    //     prefix: &Prefix,
+    //     mui: Option<u32>,
+    //     include_withdrawn: bool,
+    // ) -> Vec<PublicRecord<M>> {
+    //     trace!("get records for prefix in the right store");
+    //     let guard = epoch::pin();
+    //     match self.persist_strategy() {
+    //         PersistStrategy::PersistOnly => self
+    //             .persist_tree
+    //             .as_ref()
+    //             .map(|tree| {
+    //                 tree.get_records_for_prefix(
+    //                     PrefixId::from(*prefix),
+    //                     mui,
+    //                     include_withdrawn,
+    //                     self.in_memory_tree.withdrawn_muis_bmin(&guard),
+    //                 )
+    //             })
+    //             .unwrap_or_default(),
+    //         _ => self
+    //             .prefix_cht
+    //             .get_records_for_prefix(
+    //                 PrefixId::from(*prefix),
+    //                 mui,
+    //                 include_withdrawn,
+    //                 self.in_memory_tree.withdrawn_muis_bmin(&guard),
+    //             )
+    //             .unwrap_or_default(),
+    //     }
+    // }
 
     pub fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
         if let Some(p) = &self.persist_tree {
diff --git a/src/local_array/tests.rs b/src/local_array/tests.rs
index e930430e..4f4fd747 100644
--- a/src/local_array/tests.rs
+++ b/src/local_array/tests.rs
@@ -1,11 +1,11 @@
 #[cfg(test)]
 use std::error::Error;
 
-use crate::local_array::bit_span::BitSpan;
-
 //------------ AddressFamily bit flippers -----------------------------------
+
 #[test]
 fn test_af_1() -> Result<(), Box<dyn Error>> {
+    use crate::local_array::bit_span::BitSpan;
     use crate::prelude::multi::StrideNodeId;
     use crate::AddressFamily;
     use crate::IPv4;
@@ -38,6 +38,7 @@ fn test_af_1() -> Result<(), Box<dyn Error>> {
 
 #[test]
 fn test_af_2() -> Result<(), Box<dyn Error>> {
+    use crate::local_array::bit_span::BitSpan;
     use crate::prelude::multi::StrideNodeId;
     use crate::IPv4;
 

From 3d203078d26053f4f548d473813720ad7bd935c6 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 7 Feb 2025 16:37:33 +0100
Subject: [PATCH 069/147] grand cleanup #2

---
 src/local_array/in_memory/node.rs | 90 +++++++++++++++----------------
 1 file changed, 45 insertions(+), 45 deletions(-)

diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index cc7b50fd..1349dd5a 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -455,52 +455,52 @@ where
     //     )
     // }
 
-    pub(crate) fn prefix_exists_in_stride(
-        &'_ self,
-        search_pfx: PrefixId<AF>,
-        nibble: u32,
-        nibble_len: u8,
-        start_bit: u8,
-    ) -> (Option<StrideNodeId<AF>>, bool) {
-        let pfxbitarr = self.pfxbitarr.load();
-        let ptrbitarr = self.ptrbitarr.load();
-        // This is an exact match, so we're only considering the position of
-        // the full nibble.
-        let bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_pfx = false;
-        let mut found_child = None;
-
-        // Is this the last nibble?
-        // Otherwise we're not looking for a prefix (exact matching only
-        // lives at last nibble)
-        match search_pfx.get_len() <= start_bit + nibble_len {
-            // We're at the last nibble.
-            true => {
-                // Check for an actual prefix at the right position, i.e.
-                // consider the complete nibble.
-                if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-                    found_pfx = true;
-                }
-            }
-            // We're not at the last nibble.
-            false => {
-                // Check for a child node at the right position, i.e.
-                // consider the complete nibble.
-                if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-                {
-                    found_child = Some(
-                        StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)
-                    );
-                }
-            }
-        }
+    // pub(crate) fn prefix_exists_in_stride(
+    //     &'_ self,
+    //     search_pfx: PrefixId<AF>,
+    //     nibble: u32,
+    //     nibble_len: u8,
+    //     start_bit: u8,
+    // ) -> (Option<StrideNodeId<AF>>, bool) {
+    //     let pfxbitarr = self.pfxbitarr.load();
+    //     let ptrbitarr = self.ptrbitarr.load();
+    //     // This is an exact match, so we're only considering the position of
+    //     // the full nibble.
+    //     let bit_pos = S::get_bit_pos(nibble, nibble_len);
+    //     let mut found_pfx = false;
+    //     let mut found_child = None;
 
-        (
-            found_child, /* The node that has children in the next stride, if
-                         any */
-            found_pfx, /* The exactly matching prefix, if any */
-        )
-    }
+    //     // Is this the last nibble?
+    //     // Otherwise we're not looking for a prefix (exact matching only
+    //     // lives at last nibble)
+    //     match search_pfx.get_len() <= start_bit + nibble_len {
+    //         // We're at the last nibble.
+    //         true => {
+    //             // Check for an actual prefix at the right position, i.e.
+    //             // consider the complete nibble.
+    //             if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
+    //                 found_pfx = true;
+    //             }
+    //         }
+    //         // We're not at the last nibble.
+    //         false => {
+    //             // Check for a child node at the right position, i.e.
+    //             // consider the complete nibble.
+    //             if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
+    //             {
+    //                 found_child = Some(
+    //                     StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)
+    //                 );
+    //             }
+    //         }
+    //     }
+
+    //     (
+    //         found_child, /* The node that has children in the next stride, if
+    //                      any */
+    //         found_pfx, /* The exactly matching prefix, if any */
+    //     )
+    // }
 
     // This function looks for the exactly matching prefix in the provided
     // nibble, just like the one above, but this *does* iterate over all the

From aee902738b9ea123b99291435ca2b714b6fc105d Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 7 Feb 2025 16:38:03 +0100
Subject: [PATCH 070/147] grand cleanup #3

---
 proc_macros/src/lib.rs  | 127 +++++++++++++++++-----------------------
 src/af.rs               |   2 +-
 src/bin/cli.rs          |  24 ++++----
 src/bin/load_mrt.rs     |  12 ++--
 src/test_types.rs       |   3 +-
 tests/concurrency.rs    |  18 ++++--
 tests/more-specifics.rs |   6 +-
 tests/treebitmap.rs     |  38 ++++++------
 tests/treebitmap_v6.rs  |  25 +++++---
 9 files changed, 128 insertions(+), 127 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index db4d0859..65d5fad8 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -888,25 +888,6 @@ pub fn create_store(
                 }
             }
 
-            // pub fn more_specifics_keys_from(&'a self,
-            //     search_pfx: &Prefix,
-            // ) -> Vec<Prefix> {
-
-            //     match search_pfx.addr() {
-            //         std::net::IpAddr::V4(addr) => self
-            //             .v4
-            //             .more_specifics_keys_from(PrefixId::from(*search_pfx)
-            //             ).map(|p| Prefix::from(p)).collect(),
-            //         std::net::IpAddr::V6(addr) => self
-            //             .v6
-            //             .more_specifics_keys_from(
-            //                 PrefixId::<IPv6>::from(
-            //                     *search_pfx
-            //                 ),
-            //             ).map(|p| Prefix::from(p)).collect()
-            //     }
-            // }
-
             /// Return a `QuerySet` that contains all the less-specific
             /// prefixes of the `search_pfx` in the store, including the
             /// meta-data of these prefixes.
@@ -1007,7 +988,6 @@ pub fn create_store(
                             (
                                 Some(self
                                     .v4
-                                    // .in_memory_tree
                                     .less_specifics_iter_from(
                                         PrefixId::<IPv4>::new(
                                             addr.into(),
@@ -1027,7 +1007,6 @@ pub fn create_store(
                                 None,
                                 Some(self
                                     .v6
-                                    // .in_memory_tree
                                     .less_specifics_iter_from(
                                         PrefixId::<IPv6>::new(
                                             addr.into(),
@@ -1042,7 +1021,11 @@ pub fn create_store(
                             )
                         }
                     };
-                    left.into_iter().flatten().chain(right.into_iter().flatten())
+
+                    left
+                        .into_iter()
+                        .flatten()
+                        .chain(right.into_iter().flatten())
                 }
 
             /// Returns an iterator over all the more-specifics prefixes
@@ -1096,42 +1079,30 @@ pub fn create_store(
                 include_withdrawn: bool,
                 guard: &'a Guard,
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-
-                let (left, right) = match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => {
-
-                        if mui.is_some_and(
-                            |m| { self.v6.mui_is_withdrawn(m, guard) }) {
-                                (None, None)
-                        } else {
-                            (Some(self
-                                .v4
-                                // .in_memory_tree
-                                // .more_specific_prefix_iter_from(
-                                .more_specifics_iter_from(
-                                    PrefixId::<IPv4>::new(
-                                        addr.into(),
-                                        search_pfx.len(),
-                                    ),
-                                    mui,
-                                    include_withdrawn,
-                                    guard
-                                ).map(|p| PrefixRecord::from(p))
-                            ),
-                            None)
+                    let (left, right) = match search_pfx.addr() {
+                        std::net::IpAddr::V4(addr) => {
+                            (
+                                Some(self
+                                    .v4
+                                    .more_specifics_iter_from(
+                                        PrefixId::<IPv4>::new(
+                                            addr.into(),
+                                            search_pfx.len(),
+                                        ),
+                                        mui,
+                                        include_withdrawn,
+                                        guard
+                                    )
+                                    .map(|p| PrefixRecord::from(p))
+                                ),
+                                None
+                            )
                         }
-                    }
-                    std::net::IpAddr::V6(addr) => {
-
-                        if mui.is_some_and(
-                            |m| { self.v6.mui_is_withdrawn(m, guard) }) {
-                                (None, None)
-                        } else {
+                        std::net::IpAddr::V6(addr) => {
                             (
                                 None,
                                 Some(self
                                     .v6
-                                    // .in_memory_tree
                                     .more_specifics_iter_from(
                                         PrefixId::<IPv6>::new(
                                             addr.into(),
@@ -1140,13 +1111,17 @@ pub fn create_store(
                                         mui,
                                         include_withdrawn,
                                         guard
-                                    ).map(|p| PrefixRecord::from(p))
+                                    )
+                                    .map(|p| PrefixRecord::from(p))
                                 )
                             )
                         }
-                    }
-                };
-                left.into_iter().flatten().chain(right.into_iter().flatten())
+                    };
+
+                    left
+                        .into_iter()
+                        .flatten()
+                        .chain(right.into_iter().flatten())
             }
 
             pub fn iter_records_for_mui_v4(
@@ -1161,9 +1136,7 @@ pub fn create_store(
                     None
                 } else {
                     Some(
-                        self
-                            .v4
-                            // .in_memory_tree
+                        self.v4
                             .more_specifics_iter_from(
                                 PrefixId::<IPv4>::new(
                                     0,
@@ -1190,7 +1163,6 @@ pub fn create_store(
                 } else {
                     Some(
                         self.v6
-                            // .in_memory_tree
                             .more_specifics_iter_from(
                                 PrefixId::<IPv6>::new(
                                     0,
@@ -1286,11 +1258,12 @@ pub fn create_store(
             /// ```
             pub fn prefixes_iter(
                 &'a self,
+                guard: &'a Guard
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v4.prefixes_iter()
+                self.v4.prefixes_iter(guard)
                     .map(|p| PrefixRecord::from(p))
                     .chain(
-                        self.v6.prefixes_iter()
+                        self.v6.prefixes_iter(guard)
                         .map(|p| PrefixRecord::from(p))
                     )
             }
@@ -1337,8 +1310,9 @@ pub fn create_store(
             /// ```
             pub fn prefixes_iter_v4(
                 &'a self,
+                guard: &'a Guard
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v4.prefixes_iter()
+                self.v4.prefixes_iter(guard)
                     .map(|p| PrefixRecord::from(p))
             }
 
@@ -1384,8 +1358,9 @@ pub fn create_store(
             /// ```
             pub fn prefixes_iter_v6(
                 &'a self,
+                guard: &'a Guard
             ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v6.prefixes_iter()
+                self.v6.prefixes_iter(guard)
                     .map(|p| PrefixRecord::from(p))
             }
 
@@ -1664,21 +1639,25 @@ pub fn create_store(
             }
 
             pub fn get_records_for_prefix(
-                &self, prefix: &Prefix,
+                &self,
+                prefix: &Prefix,
                 mui: Option<u32>,
                 include_withdrawn: bool
-            ) ->
-                Vec<Record<M>> {
+            ) -> Option<Vec<Record<M>>> {
+                let guard = &epoch::pin();
+
                 match prefix.is_v4() {
-                    true => self.v4.get_records_for_prefix(
-                        prefix,
+                    true => self.v4.get_value(
+                        PrefixId::<IPv4>::from(*prefix),
                         mui,
-                        include_withdrawn
+                        include_withdrawn,
+                        guard
                     ),
-                    false => self.v6.get_records_for_prefix(
-                        prefix,
+                    false => self.v6.get_value(
+                        PrefixId::<IPv6>::from(*prefix),
                         mui,
-                        include_withdrawn
+                        include_withdrawn,
+                        guard
                     )
                 }
             }
diff --git a/src/af.rs b/src/af.rs
index f2e6e494..2e409413 100644
--- a/src/af.rs
+++ b/src/af.rs
@@ -224,7 +224,7 @@ impl AddressFamily for IPv6 {
                 (self >> ((128 - len) as usize)) << (128 - len) as usize
             }
             128 => self,
-            _ => panic!("Can't truncate to more than 128 bits"),
+            len => panic!("Can't truncate to more than 128 bits: {}", len),
         }
     }
 
diff --git a/src/bin/cli.rs b/src/bin/cli.rs
index 05de15b5..b37d5c4c 100644
--- a/src/bin/cli.rs
+++ b/src/bin/cli.rs
@@ -104,28 +104,28 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                         match cmd.to_string().as_ref() {
                             "p" => match line.chars().as_str() {
                                 "p4" => {
-                                    tree_bitmap.prefixes_iter_v4().for_each(
-                                        |pfx| {
+                                    tree_bitmap
+                                        .prefixes_iter_v4(guard)
+                                        .for_each(|pfx| {
                                             println!(
                                                 "{} {}",
                                                 pfx.prefix, pfx.meta[0]
                                             );
-                                        },
-                                    );
+                                        });
                                     println!(
                                         "ipv4 prefixes :\t{}",
                                         tree_bitmap.prefixes_v4_count()
                                     );
                                 }
                                 "p6" => {
-                                    tree_bitmap.prefixes_iter_v6().for_each(
-                                        |pfx| {
+                                    tree_bitmap
+                                        .prefixes_iter_v6(guard)
+                                        .for_each(|pfx| {
                                             println!(
                                                 "{} {}",
                                                 pfx.prefix, pfx.meta[0]
                                             );
-                                        },
-                                    );
+                                        });
                                     println!(
                                         "ipv6 prefixes :\t{}",
                                         tree_bitmap.prefixes_v6_count()
@@ -140,14 +140,14 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                         "ipv6 prefixes :\t{}",
                                         tree_bitmap.prefixes_v6_count()
                                     );
-                                    tree_bitmap.prefixes_iter().for_each(
-                                        |pfx| {
+                                    tree_bitmap
+                                        .prefixes_iter(guard)
+                                        .for_each(|pfx| {
                                             println!(
                                                 "{} {}",
                                                 pfx.prefix, pfx.meta[0]
                                             );
-                                        },
-                                    );
+                                        });
                                     println!(
                                         "total prefixes :\t{}",
                                         tree_bitmap.prefixes_count()
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index c1c2153a..fcb7a20c 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -596,8 +596,10 @@ fn main() {
         println!("\nverifying disk persistence...");
         let mut max_len = 0;
         for pfx in persisted_prefixes {
-            let values =
-                store.unwrap().get_records_for_prefix(&pfx, None, false);
+            let values = store
+                .unwrap()
+                .get_records_for_prefix(&pfx, None, false)
+                .unwrap();
             if values.is_empty() {
                 eprintln!("Found empty prefix on disk");
                 eprintln!("prefix: {}", pfx);
@@ -605,8 +607,10 @@ fn main() {
             }
             if values.len() > max_len {
                 max_len = values.len();
-                let recs =
-                    store.unwrap().get_records_for_prefix(&pfx, None, false);
+                let recs = store
+                    .unwrap()
+                    .get_records_for_prefix(&pfx, None, false)
+                    .unwrap();
                 println!("LEN {} prefix: {}", max_len, pfx);
                 for rec in recs {
                     let pa = OwnedPathAttributes::from((
diff --git a/src/test_types.rs b/src/test_types.rs
index b044db3b..abfc252a 100644
--- a/src/test_types.rs
+++ b/src/test_types.rs
@@ -13,7 +13,6 @@ impl AsRef<[u8]> for BeBytesAsn {
 
 impl From<Vec<u8>> for BeBytesAsn {
     fn from(value: Vec<u8>) -> Self {
-        println!("value {:#?}", value);
         if let Some(value) = value.first_chunk::<4>() {
             Self(*value)
         } else {
@@ -33,7 +32,7 @@ impl Meta for BeBytesAsn {
 
 impl std::fmt::Display for BeBytesAsn {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "AS{:?}", self.0)
+        write!(f, "AS{}", <u32>::from_le_bytes(self.0))
     }
 }
 
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index ec0f5c56..103a5979 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -67,6 +67,7 @@ fn test_concurrent_updates_1(
     // let store = std::sync::Arc::new(
     //     MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
     // );
+    let guard = &rotonda_store::epoch::pin();
 
     let mui_data_1 = MuiData {
         mui: 1,
@@ -127,7 +128,8 @@ fn test_concurrent_updates_1(
 
     println!("COUNT {:?}", tree_bitmap.prefixes_count());
 
-    let all_pfxs_iter = tree_bitmap.prefixes_iter().collect::<Vec<_>>();
+    let all_pfxs_iter = tree_bitmap.prefixes_iter(guard).collect::<Vec<_>>();
+    println!("all_pfxs_iter {:#?}", all_pfxs_iter);
 
     let pfx = Prefix::from_str("185.34.0.0/16").unwrap();
 
@@ -347,7 +349,10 @@ fn test_concurrent_updates_1(
 
     println!(
         "prefixes_iter {:#?}",
-        tree_bitmap.as_ref().prefixes_iter().collect::<Vec<_>>()
+        tree_bitmap
+            .as_ref()
+            .prefixes_iter(guard)
+            .collect::<Vec<_>>()
     );
 
     let match_options = MatchOptions {
@@ -363,7 +368,8 @@ fn test_concurrent_updates_1(
         let guard = rotonda_store::epoch::pin();
         let res = tree_bitmap.match_prefix(&pfx, &match_options, &guard);
         assert_eq!(res.prefix, Some(pfx));
-        println!("PFX {:?}", res);
+        println!("strategy {:?}", tree_bitmap.persist_strategy());
+        println!("PFX {}", res);
         assert_eq!(
             res.prefix_meta
                 .iter()
@@ -419,6 +425,7 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
     let tree_bitmap = std::sync::Arc::new(
         MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
     );
+    let guard = &rotonda_store::epoch::pin();
 
     let _: Vec<_> =
         vec![pfx_vec_1.clone(), pfx_vec_2.clone(), pfx_vec_3.clone()]
@@ -461,10 +468,10 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
 
     println!(
         "prefixes_iter#1 :{:#?}",
-        tree_bitmap.prefixes_iter().collect::<Vec<_>>()
+        tree_bitmap.prefixes_iter(guard).collect::<Vec<_>>()
     );
 
-    let all_pfxs_iter = tree_bitmap.prefixes_iter().collect::<Vec<_>>();
+    let all_pfxs_iter = tree_bitmap.prefixes_iter(guard).collect::<Vec<_>>();
 
     let pfx = Prefix::from_str("185.33.0.0/16").unwrap();
     assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
@@ -666,6 +673,7 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
         include_history: IncludeHistory::None,
     };
 
+    println!("strategy {:?}", tree_bitmap.persist_strategy());
     // should cover all the prefixes
     // let pfx0 = Prefix::from_str("184.0.0.0/6").unwrap();
     let pfx128 = Prefix::from_str("128.0.0.0/1").unwrap();
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 0c911e9e..9aa9a4c5 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -202,13 +202,14 @@ fn test_more_specifics() -> Result<(), Box<dyn Error>> {
         println!("all prefixes");
 
         for (i, p) in tree_bitmap
-            .prefixes_iter_v4()
+            .prefixes_iter_v4(guard)
             .enumerate()
             .map(|(i, p)| (i, p.prefix))
         {
             println!("ms {}: {}", i, p);
         }
 
+        println!("25 {}", pfxs[25].unwrap());
         assert!(tree_bitmap.contains(&pfxs[25].unwrap(), None));
         assert!(tree_bitmap.contains(&pfxs[26].unwrap(), None));
         assert!(tree_bitmap.contains(&pfxs[27].unwrap(), None));
@@ -225,7 +226,7 @@ fn test_more_specifics() -> Result<(), Box<dyn Error>> {
             .collect::<Vec<_>>();
 
         println!(
-            "{:?}",
+            ">> {:?}",
             more_specifics
                 .iter()
                 .find(|ms| ms.prefix == spfx.0.unwrap())
@@ -233,6 +234,7 @@ fn test_more_specifics() -> Result<(), Box<dyn Error>> {
         assert_eq!(found_result.prefix, spfx.1);
 
         println!("round {}", i);
+        println!("{:?}", tree_bitmap.persist_strategy());
         assert_eq!(&more_specifics.len(), &spfx.2.len());
 
         for i in spfx.2.iter() {
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 232eccd3..7bf7cda9 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -354,7 +354,7 @@ mod tests {
         // };
 
         let guard = &epoch::pin();
-        for pfx in tree_bitmap.prefixes_iter() {
+        for pfx in tree_bitmap.prefixes_iter(guard) {
             // let pfx_nm = pfx.strip_meta();
             let res = tree_bitmap.match_prefix(
                 &pfx.prefix,
@@ -368,7 +368,8 @@ mod tests {
                 },
                 guard,
             );
-            println!("{}", pfx);
+            println!("PFX {}", pfx);
+            println!("RES {}", res);
             assert_eq!(res.prefix.unwrap(), pfx.prefix);
         }
 
@@ -389,10 +390,7 @@ mod tests {
 
         assert_eq!(
             res.prefix.unwrap(),
-            Prefix::new_relaxed(
-                std::net::Ipv4Addr::new(192, 0, 0, 0).into(),
-                23
-            )?
+            Prefix::new(std::net::Ipv4Addr::new(192, 0, 0, 0).into(), 23)?
         );
 
         let less_specifics = res.less_specifics.unwrap();
@@ -634,9 +632,9 @@ mod tests {
 
         println!("all records");
 
-        let all_recs = tree_bitmap.prefixes_iter();
+        let all_recs = tree_bitmap.prefixes_iter(guard);
 
-        for rec in tree_bitmap.prefixes_iter().collect::<Vec<_>>() {
+        for rec in tree_bitmap.prefixes_iter(guard).collect::<Vec<_>>() {
             println!("{}", rec);
         }
 
@@ -648,14 +646,14 @@ mod tests {
         assert_eq!(wd_2_rec.len(), 1);
         assert_eq!(wd_2_rec[0].multi_uniq_id, 2);
 
-        let mui_2_recs = tree_bitmap.prefixes_iter().filter_map(|r| {
+        let mui_2_recs = tree_bitmap.prefixes_iter(guard).filter_map(|r| {
             r.get_record_for_mui(2).cloned().map(|rec| (r.prefix, rec))
         });
         println!("mui_2_recs prefixes_iter");
         for rec in mui_2_recs {
             println!("{} {:#?}", rec.0, rec.1);
         }
-        let mui_2_recs = tree_bitmap.prefixes_iter().filter_map(|r| {
+        let mui_2_recs = tree_bitmap.prefixes_iter(guard).filter_map(|r| {
             r.get_record_for_mui(2).cloned().map(|rec| (r.prefix, rec))
         });
 
@@ -704,13 +702,15 @@ mod tests {
         );
 
         println!("more_specifics match {} w/ withdrawn", more_specifics);
+
+        let guard = &rotonda_store::epoch::pin();
+        for p in tree_bitmap.prefixes_iter_v4(guard) {
+            println!("{}", p);
+        }
+
         let more_specifics = more_specifics.more_specifics.unwrap();
-        let ms_v4 = more_specifics
-            .v4
-            .iter()
-            .filter(|p| p.prefix != Prefix::from_str("1.0.0.0/16").unwrap())
-            .collect::<Vec<_>>();
-        assert_eq!(more_specifics.len(), 2);
+        let ms_v4 = more_specifics.v4.iter().collect::<Vec<_>>();
+        assert_eq!(more_specifics.len(), 1);
         assert_eq!(ms_v4.len(), 1);
         let more_specifics = &ms_v4[0];
         assert_eq!(more_specifics.prefix, Prefix::from_str("1.0.0.0/17")?);
@@ -754,7 +754,7 @@ mod tests {
             .iter()
             .filter(|p| p.prefix != Prefix::from_str("1.0.0.0/16").unwrap())
             .collect::<Vec<_>>();
-        assert_eq!(more_specifics.len(), 2);
+        assert_eq!(more_specifics.len(), 1);
         assert_eq!(ms_v4.len(), 1);
         let more_specifics = &ms_v4[0];
         assert_eq!(more_specifics.prefix, Prefix::from_str("1.0.0.0/17")?);
@@ -806,7 +806,7 @@ mod tests {
             .iter()
             .filter(|p| p.prefix != Prefix::from_str("1.0.0.0/16").unwrap())
             .collect::<Vec<_>>();
-        assert_eq!(more_specifics.len(), 2);
+        assert_eq!(more_specifics.len(), 1);
         assert_eq!(ms_v4.len(), 1);
         let more_specifics = &ms_v4[0];
         assert_eq!(more_specifics.prefix, Prefix::from_str("1.0.0.0/17")?);
@@ -864,7 +864,7 @@ mod tests {
             .iter()
             .filter(|p| p.prefix != Prefix::from_str("1.0.0.0/16").unwrap())
             .collect::<Vec<_>>();
-        assert_eq!(more_specifics.len(), 2);
+        assert_eq!(more_specifics.len(), 1);
         assert_eq!(ms_v4.len(), 1);
         let more_specifics = &ms_v4[0];
         assert_eq!(more_specifics.prefix, Prefix::from_str("1.0.0.0/17")?);
diff --git a/tests/treebitmap_v6.rs b/tests/treebitmap_v6.rs
index 3f68a3c2..bc3de8bf 100644
--- a/tests/treebitmap_v6.rs
+++ b/tests/treebitmap_v6.rs
@@ -60,6 +60,8 @@ mod tests {
 
     #[test]
     fn test_insert_extremes_ipv6() -> Result<(), Box<dyn std::error::Error>> {
+        crate::common::init();
+
         let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
         let min_pfx = Prefix::new_relaxed(
             std::net::Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0).into(),
@@ -90,14 +92,17 @@ mod tests {
             },
             guard,
         );
-        println!("prefix: {:?}", &expect_pfx);
-        println!("result: {:#?}", &res);
+        println!("prefix: {}", &expect_pfx.unwrap());
+        println!("result: {}", &res);
         assert!(res.prefix.is_some());
         assert_eq!(res.prefix, Some(expect_pfx?));
 
         let max_pfx = Prefix::new_relaxed(
-            std::net::Ipv6Addr::new(255, 255, 255, 255, 255, 255, 255, 255)
-                .into(),
+            std::net::Ipv6Addr::new(
+                0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+                0xffff,
+            )
+            .into(),
             128,
         );
 
@@ -108,11 +113,15 @@ mod tests {
             None,
         )?;
         let expect_pfx = Prefix::new_relaxed(
-            std::net::Ipv6Addr::new(255, 255, 255, 255, 255, 255, 255, 255)
-                .into(),
+            std::net::Ipv6Addr::new(
+                0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+                0xffff,
+            )
+            .into(),
             128,
         );
 
+        println!("done inserting...");
         let guard = &epoch::pin();
         let res = trie.match_prefix(
             &expect_pfx?,
@@ -289,7 +298,7 @@ mod tests {
         }
 
         let guard = &epoch::pin();
-        for pfx in tree_bitmap.prefixes_iter() {
+        for pfx in tree_bitmap.prefixes_iter(guard) {
             // let pfx_nm = pfx.strip_meta();
             let res = tree_bitmap.match_prefix(
                 &pfx.prefix,
@@ -562,7 +571,7 @@ mod tests {
         // };
 
         let guard = &epoch::pin();
-        for pfx in tree_bitmap.prefixes_iter() {
+        for pfx in tree_bitmap.prefixes_iter(guard) {
             // let pfx_nm = pfx.strip_meta();
             let res = tree_bitmap.match_prefix(
                 &pfx.prefix,

From 476f21efab002cdd73fc38e5c5efdd9fc47f72e2 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 11 Feb 2025 12:48:47 +0100
Subject: [PATCH 071/147] fix prefix counters for PersistOnly

---
 src/local_array/in_memory/macros.rs |   3 -
 src/local_array/in_memory/node.rs   |   1 -
 src/local_array/in_memory/tree.rs   |  59 ++---
 src/local_array/rib/rib.rs          |  25 +-
 tests/concurrency.rs                |   5 +-
 tests/full-table.rs                 |  14 +-
 tests/more-more-specifics.rs        | 380 ++++++++++++++--------------
 tests/more-specifics.rs             |  14 +-
 8 files changed, 247 insertions(+), 254 deletions(-)

diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index 88763ef2..22a629ca 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -126,9 +126,6 @@ macro_rules! insert_match {
                                     ))
                                 },
                                 (NewNodeOrIndex::NewPrefix, retry_count) => {
-                                    $self.counters.inc_prefixes_count(
-                                        $pfx.get_len()
-                                    );
                                     break (
                                         $acc_retry_count +
                                         local_retry_count +
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 1349dd5a..7808ba2b 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -12,7 +12,6 @@ use super::tree::{
     CasResult, Stride, Stride3, Stride4, Stride5,
 };
 
-// pub use crate::in_memory_tree::*;
 use crate::af::AddressFamily;
 use crate::af::Zero;
 use crate::local_array::types::PrefixId;
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index fb16966c..9b17813b 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -191,13 +191,13 @@ use log::{debug, error, log_enabled, trace};
 use roaring::RoaringBitmap;
 
 use std::sync::atomic::{
-    AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering,
+    AtomicBool, AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering,
 };
 use std::{fmt::Debug, marker::PhantomData};
 
 use super::atomic_types::NodeBuckets;
 use crate::af::AddressFamily;
-use crate::rib::Counters;
+use crate::rib::{Counters, UpsertReport};
 use crate::{
     impl_search_level, impl_search_level_for_mui, insert_match,
     retrieve_node_mut_closure, store_node_closure,
@@ -215,35 +215,22 @@ use ansi_term::Colour;
 //--------------------- TreeBitMap ------------------------------------------
 
 #[derive(Debug)]
-pub struct TreeBitMap<
-    AF: AddressFamily,
-    // M: Meta,
-    NB: NodeBuckets<AF>,
-    // PB: PrefixBuckets<AF, M>,
-> {
+pub struct TreeBitMap<AF: AddressFamily, NB: NodeBuckets<AF>> {
     pub(crate) node_buckets: NB,
-    // pub(crate) prefix_buckets: PB,
     pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     counters: Counters,
+    default_route_exists: AtomicBool,
     _af: PhantomData<AF>,
-    // _m: PhantomData<M>,
 }
 
-impl<
-        AF: AddressFamily,
-        // M: Meta,
-        NB: NodeBuckets<AF>,
-        // PB: PrefixBuckets<AF, M>,
-    > TreeBitMap<AF, NB>
-{
+impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
     pub(crate) fn new() -> Result<Self, Box<dyn std::error::Error>> {
         let tree_bitmap = Self {
             node_buckets: NodeBuckets::init(),
-            // prefix_buckets: PB::init(),
             withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
+            default_route_exists: AtomicBool::new(false),
             _af: PhantomData,
-            // _m: PhantomData,
         };
 
         let root_node = match Self::get_first_stride_size() {
@@ -279,15 +266,31 @@ impl<
         Ok(tree_bitmap)
     }
 
+    // Sets the bit for the requested prefix to 1 in the corresponding
+    // pfxbitarr in the tree.
+    //
+    // returns a Result over a tuple of (retry_count, existed), where
+    // `retry_count` is the accumulated number of times all the atomic
+    // operations involved had to be retried.
     pub(crate) fn set_prefix_exists(
         &self,
         pfx: PrefixId<AF>,
         mui: u32,
-        // update_path_selections: Option<M::TBI>,
     ) -> Result<(u32, bool), PrefixStoreError> {
         if pfx.get_len() == 0 {
-            return self.update_default_route_prefix_meta(mui);
-            // return Ok(res);
+            let prefix_new =
+                !self.default_route_exists.swap(true, Ordering::Acquire);
+            return self
+                .update_default_route_prefix_meta(mui)
+                .map(|(rc, _mui_exists)| (rc, !prefix_new));
+            // return self.update_default_route_prefix_meta(mui).map(
+            //     |(rc, mui_exists)| UpsertReport {
+            //         cas_count: rc as usize,
+            //         prefix_new,
+            //         mui_new: !mui_exists,
+            //         mui_count: 0,
+            //     },
+            // );
         }
 
         let mut stride_end: u8 = 0;
@@ -311,7 +314,6 @@ impl<
             );
             let is_last_stride = pfx.get_len() <= stride_end;
             let stride_start = stride_end - stride;
-            // let back_off = crossbeam_utils::Backoff::new();
 
             // insert_match! returns the node_id of the next node to be
             // traversed. It was created if it did not exist.
@@ -324,9 +326,6 @@ impl<
                 is_last_stride;
                 pfx;
                 mui;
-                // record;
-                // perform an update for the paths in this record
-                // update_path_selections;
                 // the length at the start of the stride a.k.a. start_bit
                 stride_start;
                 stride;
@@ -1039,12 +1038,8 @@ impl<
 
 // This implements the funky stats for a tree
 #[cfg(feature = "cli")]
-impl<
-        AF: AddressFamily,
-        // M: Meta,
-        NB: NodeBuckets<AF>,
-        // PB: PrefixBuckets<AF, M>,
-    > std::fmt::Display for TreeBitMap<AF, NB>
+impl<AF: AddressFamily, NB: NodeBuckets<AF>> std::fmt::Display
+    for TreeBitMap<AF, NB>
 {
     fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(_f, "{} prefixes created", self.get_prefixes_count())?;
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 2e180197..bdb67780 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -231,8 +231,6 @@ pub struct Rib<
     #[cfg(feature = "persist")]
     pub(in crate::local_array) persist_tree:
         Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
-    // Global Roaring BitMap INdex that stores MUIs.
-    // pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     pub counters: Counters,
 }
 
@@ -245,7 +243,7 @@ impl<
         const KEY_SIZE: usize,
     > Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
 {
-    pub fn new(
+    pub(crate) fn new(
         config: StoreConfig,
     ) -> Result<
         Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
@@ -269,7 +267,6 @@ impl<
             config,
             in_memory_tree: TreeBitMap::<AF, NB>::new()?,
             persist_tree,
-            // withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
             prefix_cht: PrefixCHT::<AF, M, PB>::new(),
         };
@@ -277,7 +274,7 @@ impl<
         Ok(store)
     }
 
-    pub fn insert(
+    pub(crate) fn insert(
         &self,
         prefix: PrefixId<AF>,
         record: PublicRecord<M>,
@@ -286,22 +283,23 @@ impl<
         let guard = &epoch::pin();
         self.in_memory_tree
             .set_prefix_exists(prefix, record.multi_uniq_id)
-            .and_then(|c1| {
+            .and_then(|(retry_count, exists)| {
                 self.upsert_prefix(
                     prefix,
                     record,
                     update_path_selections,
                     guard,
                 )
-                .map(|mut c| {
-                    if c.prefix_new {
-                        self.counters.inc_prefixes_count(prefix.get_len());
-                    }
-                    if c.mui_new {
+                .map(|mut report| {
+                    if report.mui_new {
                         self.counters.inc_routes_count();
                     }
-                    c.cas_count += c1.0 as usize;
-                    c
+                    report.cas_count += retry_count as usize;
+                    if !exists {
+                        self.counters.inc_prefixes_count(prefix.get_len());
+                        report.prefix_new = true;
+                    }
+                    report
                 })
             })
     }
@@ -499,7 +497,6 @@ impl<
                 if let Some(record) = p_tree
                     .get_most_recent_record_for_prefix_mui::<M>(prefix, mui)
                 {
-                    // for s in stored_prefixes {
                     let new_key: [u8; KEY_SIZE] = PersistTree::<
                         AF,
                         PREFIX_SIZE,
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index 103a5979..825106f5 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -1,7 +1,4 @@
-use std::{
-    str::FromStr,
-    sync::{atomic::Ordering, Arc},
-};
+use std::{str::FromStr, sync::atomic::Ordering};
 
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
diff --git a/tests/full-table.rs b/tests/full-table.rs
index e1d7c297..27c4feeb 100644
--- a/tests/full-table.rs
+++ b/tests/full-table.rs
@@ -52,8 +52,16 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_full_table_from_csv() -> Result<(), Box<dyn Error>> {
+    rotonda_store::all_strategies![
+        full_table_1;
+        test_full_table_from_csv;
+        AsnList
+    ];
+
+    // #[test]
+    fn test_full_table_from_csv(
+        tree_bitmap: MultiThreadedStore<AsnList>,
+    ) -> Result<(), Box<dyn Error>> {
         // These constants are all contingent on the exact csv file,
         // being loaded!
 
@@ -102,7 +110,7 @@ mod tests {
         ];
         for _strides in strides_vec.iter().enumerate() {
             let mut pfxs: Vec<PrefixRecord<AsnList>> = vec![];
-            let tree_bitmap = MultiThreadedStore::<AsnList>::try_default()?;
+            // let tree_bitmap = MultiThreadedStore::<AsnList>::try_default()?;
             // .with_user_data("Testing".to_string());
 
             if let Err(err) = load_prefixes(&mut pfxs) {
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index 817bb6a2..a7d60812 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -1,4 +1,14 @@
 // type Prefix4<'a> = Prefix<u32, PrefixAs>;
+
+use std::error::Error;
+
+use inetnum::addr::Prefix;
+
+use rotonda_store::{
+    meta_examples::PrefixAs, prelude::multi::RouteStatus, rib::StoreConfig,
+    IncludeHistory, MatchOptions, MatchType, MultiThreadedStore,
+    PublicRecord as Record,
+};
 mod common {
     use std::io::Write;
 
@@ -10,213 +20,195 @@ mod common {
     }
 }
 
-mod tests {
-    use inetnum::addr::Prefix;
-    use rotonda_store::{
-        meta_examples::PrefixAs, prelude::multi::*, prelude::*,
-    };
-
-    use std::error::Error;
-
-    #[test]
-    fn test_more_specifics_without_less_specifics(
-    ) -> Result<(), Box<dyn Error>> {
-        crate::common::init();
-
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
-        let pfxs = vec![
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18)?, // 0
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24)?, // 1
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 153, 0).into(), 24)?, // 2
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 21)?, // 3
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 176, 0).into(), 20)?, // 4
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8)?, // 5
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 184, 0).into(), 23)?, // 6
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 71, 0).into(), 24)?, // 7
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9)?, // 8
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 117, 0).into(), 24)?, // 9
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 99, 0).into(), 24)?, // 10
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 224, 0).into(), 24)?, // 11
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 128, 0).into(), 18)?, // 12
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 120, 0).into(), 24)?, // 13
-        ];
-
-        for pfx in pfxs.iter() {
-            tree_bitmap.insert(
-                pfx,
-                Record::new(
-                    0,
-                    0,
-                    RouteStatus::Active,
-                    PrefixAs::new_from_u32(666),
-                ),
-                None,
-            )?;
-        }
-        println!("------ end of inserts\n");
-
-        // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
-        let guard = &epoch::pin();
-        for (r, spfx) in &[
-            (
+rotonda_store::all_strategies![
+    test_ms_w_ls_1;
+    test_more_specifics_without_less_specifics;
+    PrefixAs
+];
+
+// #[test]
+fn test_more_specifics_without_less_specifics(
+    tree_bitmap: MultiThreadedStore<PrefixAs>,
+) -> Result<(), Box<dyn Error>> {
+    crate::common::init();
+
+    // let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    let pfxs = vec![
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18)?, // 0
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24)?, // 1
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 153, 0).into(), 24)?, // 2
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 21)?,  // 3
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 176, 0).into(), 20)?, // 4
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8)?,   // 5
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 184, 0).into(), 23)?, // 6
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 71, 0).into(), 24)?, // 7
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9)?,   // 8
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 117, 0).into(), 24)?, // 9
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 99, 0).into(), 24)?, // 10
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 224, 0).into(), 24)?, // 11
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 128, 0).into(), 18)?, // 12
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 120, 0).into(), 24)?, // 13
+    ];
+
+    for pfx in pfxs.iter() {
+        tree_bitmap.insert(
+            pfx,
+            Record::new(
                 0,
-                (
-                    &Prefix::new(
-                        std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
-                        9,
-                    ),
-                    &Prefix::new(
-                        std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
-                        9,
-                    ), // 0
-                    vec![0, 1, 2, 3, 4, 6, 7, 9, 10, 11, 12, 13],
-                ),
-            ),
-            (
-                1,
-                (
-                    &Prefix::new(
-                        std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
-                        8,
-                    ),
-                    &Prefix::new(
-                        std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
-                        8,
-                    ), // 0
-                    vec![0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13],
-                ),
+                0,
+                RouteStatus::Active,
+                PrefixAs::new_from_u32(666),
             ),
-        ] {
-            println!("start round {}", r);
-            println!("search for: {}", spfx.0.unwrap());
-            let found_result = tree_bitmap.match_prefix(
-                &spfx.0.unwrap(),
-                &MatchOptions {
-                    match_type: MatchType::ExactMatch,
-                    include_withdrawn: false,
-                    include_less_specifics: false,
-                    include_more_specifics: true,
-                    mui: None,
-                    include_history: IncludeHistory::None,
-                },
-                guard,
-            );
-            println!("em/m-s: {:#?}", found_result);
-
-            let more_specifics = found_result
-                .more_specifics
-                .unwrap()
-                .iter()
-                .filter(|p| p.prefix != spfx.0.unwrap())
-                .collect::<Vec<_>>();
-
-            assert_eq!(found_result.prefix.unwrap(), spfx.1.unwrap());
-            assert_eq!(&more_specifics.len(), &spfx.2.len());
-
-            for i in spfx.2.iter() {
-                print!("{} ", i);
-
-                let result_pfx =
-                    more_specifics.iter().find(|pfx| pfx.prefix == pfxs[*i]);
-                assert!(result_pfx.is_some());
-            }
-            println!("end round {}", r);
-            println!("-----------");
-        }
-        Ok(())
+            None,
+        )?;
     }
+    println!("------ end of inserts\n");
 
-    #[test]
-    fn test_more_specifics_with_less_specifics() -> Result<(), Box<dyn Error>>
-    {
-        crate::common::init;
-
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
-        let pfxs = vec![
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18), // 0
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24), // 1
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 153, 0).into(), 24), // 2
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 21), // 3
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 176, 0).into(), 20), // 4
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8), // 5
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 184, 0).into(), 23), // 6
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 71, 0).into(), 24), // 7
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9), // 8
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 117, 0).into(), 24), // 9
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 99, 0).into(), 24), // 10
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 224, 0).into(), 24), // 11
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 128, 0).into(), 18), // 12
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 120, 0).into(), 24), // 13
-        ];
-
-        let ltime = 0;
-        let status = RouteStatus::Active;
-        for pfx in pfxs.iter() {
-            tree_bitmap.insert(
-                &pfx.unwrap(),
-                Record::new(0, ltime, status, PrefixAs::new_from_u32(666)),
-                None,
-            )?;
-        }
-        println!("------ end of inserts\n");
-        let guard = &epoch::pin();
-
-        for spfx in &[
+    // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
+    let guard = &rotonda_store::epoch::pin();
+    for (r, spfx) in &[
+        (
+            0,
             (
                 &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9),
-                Some(&Prefix::new(
-                    std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
-                    9,
-                )), // 0
+                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9), // 0
                 vec![0, 1, 2, 3, 4, 6, 7, 9, 10, 11, 12, 13],
             ),
+        ),
+        (
+            1,
             (
                 &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8),
-                Some(&Prefix::new(
-                    std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
-                    8,
-                )), // 0
+                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8), // 0
                 vec![0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13],
             ),
-        ] {
-            println!("search for: {:#}", (*spfx.0)?);
-            let found_result = tree_bitmap.match_prefix(
-                &spfx.0.unwrap(),
-                &MatchOptions {
-                    match_type: MatchType::LongestMatch,
-                    include_withdrawn: false,
-                    include_less_specifics: false,
-                    include_more_specifics: true,
-                    mui: None,
-                    include_history: IncludeHistory::None,
-                },
-                guard,
-            );
-            println!("em/m-s: {}", found_result);
-
-            let more_specifics = found_result
-                .more_specifics
-                .unwrap()
+        ),
+    ] {
+        println!("start round {}", r);
+        println!("search for: {}", spfx.0.unwrap());
+        let found_result = tree_bitmap.match_prefix(
+            &spfx.0.unwrap(),
+            &MatchOptions {
+                match_type: MatchType::ExactMatch,
+                include_withdrawn: false,
+                include_less_specifics: false,
+                include_more_specifics: true,
+                mui: None,
+                include_history: IncludeHistory::None,
+            },
+            guard,
+        );
+        println!("em/m-s: {:#?}", found_result);
+
+        let more_specifics = found_result
+            .more_specifics
+            .unwrap()
+            .iter()
+            .filter(|p| p.prefix != spfx.0.unwrap())
+            .collect::<Vec<_>>();
+
+        assert_eq!(found_result.prefix.unwrap(), spfx.1.unwrap());
+        assert_eq!(&more_specifics.len(), &spfx.2.len());
+
+        for i in spfx.2.iter() {
+            print!("{} ", i);
+
+            let result_pfx =
+                more_specifics.iter().find(|pfx| pfx.prefix == pfxs[*i]);
+            assert!(result_pfx.is_some());
+        }
+        println!("end round {}", r);
+        println!("-----------");
+    }
+    Ok(())
+}
+
+#[test]
+fn test_more_specifics_with_less_specifics() -> Result<(), Box<dyn Error>> {
+    crate::common::init();
+
+    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    let pfxs = vec![
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18), // 0
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24), // 1
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 153, 0).into(), 24), // 2
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 21),  // 3
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 176, 0).into(), 20), // 4
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8),   // 5
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 184, 0).into(), 23), // 6
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 71, 0).into(), 24), // 7
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9),   // 8
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 117, 0).into(), 24), // 9
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 99, 0).into(), 24), // 10
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 224, 0).into(), 24), // 11
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 128, 0).into(), 18), // 12
+        Prefix::new(std::net::Ipv4Addr::new(17, 0, 120, 0).into(), 24), // 13
+    ];
+
+    let ltime = 0;
+    let status = RouteStatus::Active;
+    for pfx in pfxs.iter() {
+        tree_bitmap.insert(
+            &pfx.unwrap(),
+            Record::new(0, ltime, status, PrefixAs::new_from_u32(666)),
+            None,
+        )?;
+    }
+    println!("------ end of inserts\n");
+    let guard = &rotonda_store::epoch::pin();
+
+    for spfx in &[
+        (
+            &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9),
+            Some(&Prefix::new(
+                std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
+                9,
+            )), // 0
+            vec![0, 1, 2, 3, 4, 6, 7, 9, 10, 11, 12, 13],
+        ),
+        (
+            &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8),
+            Some(&Prefix::new(
+                std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
+                8,
+            )), // 0
+            vec![0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13],
+        ),
+    ] {
+        println!("search for: {:#}", (*spfx.0)?);
+        let found_result = tree_bitmap.match_prefix(
+            &spfx.0.unwrap(),
+            &MatchOptions {
+                match_type: MatchType::LongestMatch,
+                include_withdrawn: false,
+                include_less_specifics: false,
+                include_more_specifics: true,
+                mui: None,
+                include_history: IncludeHistory::None,
+            },
+            guard,
+        );
+        println!("em/m-s: {}", found_result);
+
+        let more_specifics = found_result
+            .more_specifics
+            .unwrap()
+            .iter()
+            .filter(|p| p.prefix != spfx.0.unwrap())
+            .collect::<Vec<_>>();
+
+        assert_eq!(found_result.prefix.unwrap(), spfx.1.unwrap().unwrap());
+        assert_eq!(&more_specifics.len(), &spfx.2.len());
+
+        for i in spfx.2.iter() {
+            print!("{} ", i);
+
+            let result_pfx = more_specifics
                 .iter()
-                .filter(|p| p.prefix != spfx.0.unwrap())
-                .collect::<Vec<_>>();
-
-            assert_eq!(
-                found_result.prefix.unwrap(),
-                spfx.1.unwrap().unwrap()
-            );
-            assert_eq!(&more_specifics.len(), &spfx.2.len());
-
-            for i in spfx.2.iter() {
-                print!("{} ", i);
-
-                let result_pfx = more_specifics
-                    .iter()
-                    .find(|pfx| pfx.prefix == pfxs[*i].unwrap());
-                assert!(result_pfx.is_some());
-            }
-            println!("-----------");
+                .find(|pfx| pfx.prefix == pfxs[*i].unwrap());
+            assert!(result_pfx.is_some());
         }
-        Ok(())
+        println!("-----------");
     }
+    Ok(())
 }
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 9aa9a4c5..1dc6c050 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -16,11 +16,19 @@ mod common {
     }
 }
 
-#[test]
-fn test_more_specifics() -> Result<(), Box<dyn Error>> {
+rotonda_store::all_strategies![
+    test_ms_1;
+    test_more_specifics;
+    PrefixAs
+];
+
+// #[test]
+fn test_more_specifics(
+    tree_bitmap: MultiThreadedStore<PrefixAs>,
+) -> Result<(), Box<dyn Error>> {
     crate::common::init();
 
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    // let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
     let pfxs = vec![
         Prefix::new(std::net::Ipv4Addr::new(130, 55, 240, 0).into(), 24), // 0
         //

From 2e47f415d617ddc8fbf97400978f0a39f1dc0d7a Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 13 Feb 2025 09:52:42 +0100
Subject: [PATCH 072/147] more strategies for some tests

---
 tests/treebitmap.rs    | 164 ++++++++++++++++++++++-------------------
 tests/treebitmap_v6.rs |  72 ++++++++++++++----
 2 files changed, 143 insertions(+), 93 deletions(-)

diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 7bf7cda9..7df45241 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -98,18 +98,19 @@ mod tests {
         Ok(())
     }
 
-    // rotonda_store::all_strategies![
-    //     tree_ipv4;
-    //     test_tree_ipv4;
-    //     PrefixAs
-    // ];
-
-    #[test]
-    fn test_tree_ipv4(// tree_bitmap: MultiThreadedStore<PrefixAs>,
+    rotonda_store::all_strategies![
+        tree_ipv4;
+        test_tree_ipv4;
+        PrefixAs
+    ];
+
+    // #[test]
+    fn test_tree_ipv4(
+        tree_bitmap: MultiThreadedStore<PrefixAs>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+        // let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             // Prefix::new_relaxed(0b0000_0000_0000_0000_0000_0000_0000_000 0_u32.into_ipaddr(), 0),
             Prefix::new_relaxed(
@@ -414,89 +415,98 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn test_ranges_ipv4() -> Result<(), Box<dyn std::error::Error>> {
-        for persist_strategy in [
-            PersistStrategy::MemoryOnly,
-            PersistStrategy::PersistOnly,
-            PersistStrategy::WriteAhead,
-            PersistStrategy::PersistHistory,
-        ] {
-            for i_net in 0..255 {
-                let config = StoreConfig {
-                    persist_strategy,
-                    persist_path: "/tmp/rotonda".into(),
-                };
-                let tree_bitmap =
-                    MultiThreadedStore::<NoMeta>::new_with_config(config)?;
-
-                let pfx_vec: Vec<Prefix> = (1..32)
-                    .collect::<Vec<u8>>()
-                    .into_iter()
-                    .map(|i_len| {
-                        Prefix::new_relaxed(
-                            std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                            i_len,
-                        )
-                        .unwrap()
-                    })
-                    .collect();
-
-                let mut i_len_s = 0;
-                for pfx in pfx_vec {
-                    i_len_s += 1;
-                    tree_bitmap.insert(
-                        &pfx,
-                        Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
-                        None,
-                    )?;
+    rotonda_store::all_strategies![
+        ranges_ipv4;
+        test_ranges_ipv4;
+        NoMeta
+    ];
 
-                    let res_pfx = Prefix::new_relaxed(
+    // #[test]
+    fn test_ranges_ipv4(
+        tree_bitmap: MultiThreadedStore<NoMeta>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        // for persist_strategy in [
+        //     PersistStrategy::MemoryOnly,
+        //     PersistStrategy::PersistOnly,
+        //     PersistStrategy::WriteAhead,
+        //     PersistStrategy::PersistHistory,
+        // ] {
+        for i_net in 0..255 {
+            // let config = StoreConfig {
+            //     persist_strategy,
+            //     persist_path: "/tmp/rotonda".into(),
+            // };
+            // let tree_bitmap =
+            //     MultiThreadedStore::<NoMeta>::new_with_config(config)?;
+
+            let pfx_vec: Vec<Prefix> = (1..32)
+                .collect::<Vec<u8>>()
+                .into_iter()
+                .map(|i_len| {
+                    Prefix::new_relaxed(
                         std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                        i_len_s,
+                        i_len,
+                    )
+                    .unwrap()
+                })
+                .collect();
+
+            let mut i_len_s = 0;
+            for pfx in pfx_vec {
+                i_len_s += 1;
+                tree_bitmap.insert(
+                    &pfx,
+                    Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
+                    None,
+                )?;
+
+                let res_pfx = Prefix::new_relaxed(
+                    std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
+                    i_len_s,
+                );
+
+                let guard = &epoch::pin();
+                for s_len in i_len_s..32 {
+                    let pfx = Prefix::new_relaxed(
+                        std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
+                        s_len,
+                    )?;
+                    let res = tree_bitmap.match_prefix(
+                        &pfx,
+                        &MatchOptions {
+                            match_type: MatchType::LongestMatch,
+                            include_withdrawn: false,
+                            include_less_specifics: false,
+                            include_more_specifics: false,
+                            mui: None,
+                            include_history: IncludeHistory::None,
+                        },
+                        guard,
                     );
+                    println!("{:?}", pfx);
 
-                    let guard = &epoch::pin();
-                    for s_len in i_len_s..32 {
-                        let pfx = Prefix::new_relaxed(
-                            std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                            s_len,
-                        )?;
-                        let res = tree_bitmap.match_prefix(
-                            &pfx,
-                            &MatchOptions {
-                                match_type: MatchType::LongestMatch,
-                                include_withdrawn: false,
-                                include_less_specifics: false,
-                                include_more_specifics: false,
-                                mui: None,
-                                include_history: IncludeHistory::None,
-                            },
-                            guard,
-                        );
-                        println!("{:?}", pfx);
-
-                        assert_eq!(res.prefix.unwrap(), res_pfx?);
-                    }
+                    assert_eq!(res.prefix.unwrap(), res_pfx?);
                 }
             }
+            // }
         }
 
         Ok(())
     }
 
-    // rotonda_store::all_strategies![
-    //     multi_ranges;
-    //     test_multi_ranges_ipv4;
-    //     NoMeta
-    // ];
+    rotonda_store::all_strategies![
+        multi_ranges;
+        test_multi_ranges_ipv4;
+        NoMeta
+    ];
 
-    #[test]
-    fn test_multi_ranges_ipv4(// tree_bitmap: MultiThreadedStore<NoMeta>,
+    // #[test]
+    fn test_multi_ranges_ipv4(
+        tree_bitmap: MultiThreadedStore<NoMeta>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
+        // let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
         for mui in [1_u32, 2, 3, 4, 5] {
             println!("Multi Uniq ID {mui}");
 
diff --git a/tests/treebitmap_v6.rs b/tests/treebitmap_v6.rs
index bc3de8bf..6c90a019 100644
--- a/tests/treebitmap_v6.rs
+++ b/tests/treebitmap_v6.rs
@@ -17,10 +17,18 @@ mod tests {
         prelude::*,
     };
 
-    #[test]
-    fn test_arbitrary_insert_ipv6() -> Result<(), Box<dyn std::error::Error>>
-    {
-        let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
+    rotonda_store::all_strategies![
+        tests_ipv6;
+        test_arbitrary_insert_ipv6;
+        NoMeta
+    ];
+
+    // #[test]
+    fn test_arbitrary_insert_ipv6(
+        trie: MultiThreadedStore<NoMeta>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        crate::common::init();
+        // let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
         let guard = &epoch::pin();
         let a_pfx = Prefix::new_relaxed(
             ("2001:67c:1bfc::").parse::<std::net::Ipv6Addr>()?.into(),
@@ -58,11 +66,19 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn test_insert_extremes_ipv6() -> Result<(), Box<dyn std::error::Error>> {
+    rotonda_store::all_strategies![
+        tests_ipv6_2;
+        test_insert_extremes_ipv6;
+        NoMeta
+    ];
+
+    // #[test]
+    fn test_insert_extremes_ipv6(
+        trie: MultiThreadedStore<NoMeta>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
+        // let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
         let min_pfx = Prefix::new_relaxed(
             std::net::Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0).into(),
             1,
@@ -140,15 +156,23 @@ mod tests {
         Ok(())
     }
 
+    rotonda_store::all_strategies![
+        max_levels;
+        test_max_levels;
+        PrefixAs
+    ];
+
     // This test aims to fill all the levels available in the PrefixBuckets
     // mapping. This tests the prefix-length-to-bucket-sizes-per-storage-
     // level mapping, most notably if the exit condition is met (a zero at
     // the end of a prefix-length array).
-    #[test]
-    fn test_max_levels() -> Result<(), Box<dyn std::error::Error>> {
+    // #[test]
+    fn test_max_levels(
+        tree_bitmap: MultiThreadedStore<PrefixAs>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+        // let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             // 0-7
             Prefix::new_relaxed(
@@ -319,9 +343,17 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn test_tree_ipv6() -> Result<(), Box<dyn std::error::Error>> {
-        let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    rotonda_store::all_strategies![
+        tree_ipv6_2;
+        test_tree_ipv6;
+        PrefixAs
+    ];
+
+    // #[test]
+    fn test_tree_ipv6(
+        tree_bitmap: MultiThreadedStore<PrefixAs>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        // let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
             // Prefix::new_relaxed(0b0000_0000_0000_0000_0000_0000_0000_000 0_u128.into_ipaddr(), 0),
             Prefix::new_relaxed(
@@ -638,10 +670,18 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn test_ranges_ipv4() -> Result<(), Box<dyn std::error::Error>> {
+    rotonda_store::all_strategies![
+        ranges_ipv4;
+        test_ranges_ipv4;
+        NoMeta
+    ];
+
+    // #[test]
+    fn test_ranges_ipv4(
+        tree_bitmap: MultiThreadedStore<NoMeta>,
+    ) -> Result<(), Box<dyn std::error::Error>> {
         for i_net in 0..255 {
-            let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
+            // let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
 
             let pfx_vec: Vec<Prefix> = (1..32)
                 .collect::<Vec<u8>>()

From d01208729475f606bd555c2546124186adf6504b Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 13 Feb 2025 12:13:08 +0100
Subject: [PATCH 073/147] update lsm_tree to 2.6.3

---
 Cargo.toml                          |  2 +-
 src/local_array/persist/lsm_tree.rs | 16 ++++++++--------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index a5aaac3d..348b2c83 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,7 +40,7 @@ clap        = { version = "4.4", optional = true, features = ["derive"] }
 rayon = { version = "1.10", optional = true }
 memmap2 = { version = "0.9", optional = true }
 rand = { version = "^0.8", optional = true }
-lsm-tree = { version = "2.4.0", optional = true }
+lsm-tree = { version = "2.6.3", optional = true }
 uuid = { version = "1.11.0", features = ["v4", "fast-rng"] }
 serde = "1.0.216"
 serde_derive = "1.0.216"
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 5e924c6c..8aaa2c5e 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -71,7 +71,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                 // get the records from the persist store for the (prefix,
                 // mui) tuple only.
                 let prefix_b = Self::prefix_mui_persistence_key(prefix, mui);
-                (*self.tree.prefix(prefix_b))
+                (*self.tree.prefix(prefix_b, None, None))
                     .into_iter()
                     .map(|kv| {
                         let kv = kv.unwrap();
@@ -96,7 +96,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             (None, true) => {
                 // get all records for this prefix
                 let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
-                (*self.tree.prefix(prefix_b))
+                (*self.tree.prefix(prefix_b, None, None))
                     .into_iter()
                     .map(|kv| {
                         let kv = kv.unwrap();
@@ -121,7 +121,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
             (None, false) => {
                 // get all records for this prefix
                 let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
-                (*self.tree.prefix(prefix_b))
+                (*self.tree.prefix(prefix_b, None, None))
                     .into_iter()
                     .filter_map(|kv| {
                         let kv = kv.unwrap();
@@ -149,7 +149,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
                 // get the records from the persist store for the (prefix,
                 // mui) tuple only.
                 let prefix_b = Self::prefix_mui_persistence_key(prefix, mui);
-                (*self.tree.prefix(prefix_b))
+                (*self.tree.prefix(prefix_b, None, None))
                     .into_iter()
                     .filter_map(|kv| {
                         let kv = kv.unwrap();
@@ -233,7 +233,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     ) -> Option<PublicRecord<M>> {
         let key_b = Self::prefix_mui_persistence_key(prefix, mui);
 
-        (*self.tree.prefix(key_b))
+        (*self.tree.prefix(key_b, None, None))
             .into_iter()
             .map(|kv| {
                 let kv = kv.unwrap();
@@ -257,7 +257,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     ) -> Vec<(Vec<u8>, PublicRecord<M>)> {
         let key_b = Self::prefix_mui_persistence_key(prefix, mui);
 
-        (*self.tree.prefix(key_b))
+        (*self.tree.prefix(key_b, None, None))
             .into_iter()
             .map(|kv| {
                 let kv = kv.unwrap();
@@ -290,7 +290,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         let start = PrefixId::new(prefix.get_net(), len);
         let end: [u8; PREFIX_SIZE] = start.inc_len().to_len_first_bytes();
 
-        self.tree.range(start.to_len_first_bytes()..end)
+        self.tree.range(start.to_len_first_bytes()..end, None, None)
     }
 
     // fn enrich_prefix<M: Meta>(
@@ -613,7 +613,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         &'a self,
     ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
         PersistedPrefixIter::<AF, M, PREFIX_SIZE, KEY_SIZE> {
-            tree_iter: self.tree.iter(),
+            tree_iter: self.tree.iter(None, None),
             cur_rec: None,
             _af: PhantomData,
             _m: PhantomData,

From f1846cdcb5feca84f3b605136a08db3de7d5574b Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 13 Feb 2025 12:32:09 +0100
Subject: [PATCH 074/147] harmonise trait bounds for AddressFamily

---
 src/af.rs                         |  5 +++--
 src/local_array/in_memory/tree.rs | 14 ++++++++------
 src/local_array/prefix_cht/cht.rs |  6 +++---
 src/local_array/types.rs          |  4 ++--
 src/prefix_record.rs              | 12 ++++++------
 5 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/src/af.rs b/src/af.rs
index 2e409413..a3c93823 100644
--- a/src/af.rs
+++ b/src/af.rs
@@ -14,12 +14,13 @@ pub trait AddressFamily:
     + std::fmt::Display
     + From<u32>
     + From<u16>
+    + From<u8>
     + Eq
     + std::ops::BitAnd<Output = Self>
     + std::ops::BitOr<Output = Self>
-    + std::ops::Shr<u8, Output = Self>
+    + std::ops::Shr<Self, Output = Self>
     + std::ops::Shl<Output = Self>
-    + std::ops::Shl<u8, Output = Self>
+    + std::ops::Shl<Self, Output = Self>
     + std::ops::Sub<Output = Self>
     + Zero
     + Copy
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 9b17813b..68081e1b 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -921,9 +921,11 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                     // which will panic in debug mode (undefined behaviour
                     // in prod).
                     BitSpan::new(
-                        ((prefix.get_net() << node_len).checked_shr_or_zero(
-                            (AF::BITS - (prefix.get_len() - node_len)).into(),
-                        ))
+                        ((prefix.get_net() << node_len.into())
+                            .checked_shr_or_zero(
+                                (AF::BITS - (prefix.get_len() - node_len))
+                                    .into(),
+                            ))
                         .dangerously_truncate_to_u32(),
                         prefix.get_len() - node_len,
                     ),
@@ -999,9 +1001,9 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
         // );
         // HASHING FUNCTION
-        ((id.get_id().0 << last_level)
-            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
-            .dangerously_truncate_to_u32() as usize
+        ((id.get_id().0 << last_level.into())
+            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS).into())
+        .dangerously_truncate_to_u32() as usize
     }
 }
 
diff --git a/src/local_array/prefix_cht/cht.rs b/src/local_array/prefix_cht/cht.rs
index 2283f8b5..1fe483ea 100644
--- a/src/local_array/prefix_cht/cht.rs
+++ b/src/local_array/prefix_cht/cht.rs
@@ -322,8 +322,8 @@ impl<AF: AddressFamily, M: Meta, PB: PrefixBuckets<AF, M>>
         //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
         // );
         // HASHING FUNCTION
-        ((id.get_net() << last_level)
-            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS))
-            .dangerously_truncate_to_u32() as usize
+        ((id.get_net() << last_level.into())
+            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS).into())
+        .dangerously_truncate_to_u32() as usize
     }
 }
diff --git a/src/local_array/types.rs b/src/local_array/types.rs
index 48e48bd9..7cbc9990 100644
--- a/src/local_array/types.rs
+++ b/src/local_array/types.rs
@@ -80,11 +80,11 @@ impl<AF: AddressFamily> PrefixId<AF> {
         trace!("orig {:032b}", self.net);
         trace!(
             "new  {:032b}",
-            self.net >> (AF::BITS - len) << (AF::BITS - len)
+            self.net >> (AF::BITS - len).into() << (AF::BITS - len).into()
         );
         trace!(
             "truncate to net {} len {}",
-            self.net >> (AF::BITS - len) << (AF::BITS - len),
+            self.net >> (AF::BITS - len).into() << (AF::BITS - len).into(),
             len
         );
         Self {
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index 7a50a8f8..41b19b92 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -72,8 +72,8 @@ where
     AF: AddressFamily,
 {
     fn cmp(&self, other: &Self) -> Ordering {
-        (self.net >> (AF::BITS - self.len))
-            .cmp(&(other.net >> ((AF::BITS - other.len) % 32)))
+        (self.net >> (AF::BITS - self.len).into())
+            .cmp(&(other.net >> ((AF::BITS - other.len) % 32).into()))
     }
 }
 
@@ -83,8 +83,8 @@ where
     AF: AddressFamily,
 {
     fn eq(&self, other: &Self) -> bool {
-        self.net >> (AF::BITS - self.len)
-            == other.net >> ((AF::BITS - other.len) % 32)
+        self.net >> (AF::BITS - self.len).into()
+            == other.net >> ((AF::BITS - other.len) % 32).into()
     }
 }
 
@@ -95,8 +95,8 @@ where
 {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(
-            (self.net >> (AF::BITS - self.len))
-                .cmp(&(other.net >> ((AF::BITS - other.len) % 32))),
+            (self.net >> (AF::BITS - self.len).into())
+                .cmp(&(other.net >> ((AF::BITS - other.len) % 32).into())),
         )
     }
 }

From 5665440ee679b77c80004b209271d9b7e6b2dc70 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 13 Feb 2025 14:16:58 +0100
Subject: [PATCH 075/147] remove uuid, use rand directly for dir names

---
 Cargo.toml                           | 4 +---
 proc_macros/src/lib.rs               | 9 ++++++++-
 src/local_array/rib/default_store.rs | 1 +
 src/prelude/mod.rs                   | 1 -
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 348b2c83..76d3a4df 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -39,9 +39,8 @@ rustyline   = { version = "13", optional = true }
 clap        = { version = "4.4", optional = true, features = ["derive"] }
 rayon = { version = "1.10", optional = true }
 memmap2 = { version = "0.9", optional = true }
-rand = { version = "^0.8", optional = true }
+rand = { version = "0.9" }
 lsm-tree = { version = "2.6.3", optional = true }
-uuid = { version = "1.11.0", features = ["v4", "fast-rng"] }
 serde = "1.0.216"
 serde_derive = "1.0.216"
 serde_json = "1.0.133"
@@ -50,7 +49,6 @@ num-traits = "0.2.19"
 [dev-dependencies]
 csv         = { version = "1" }
 env_logger  = { version = "0.10" }
-rand        = "^0.8"
 
 [features]
 cli = ["ansi_term", "rustyline", "csv"]
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 65d5fad8..0a03bf5b 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -558,7 +558,14 @@ pub fn create_store(
                     mut config: StoreConfig
                 ) -> Result<Self, Box<dyn std::error::Error>> {
 
-                let uuid = Uuid::new_v4();
+                let rng = rand::rng();
+                let uuid: String = rng
+                    .sample_iter(
+                        rand::distr::Alphanumeric
+                    )
+                    .take(12)
+                    .map(|b| char::from(b))
+                    .collect();
                 let mut config_v4 = config.clone();
                 let mut config_v6 = config.clone();
 
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index 997a3334..8f591cfc 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -1,6 +1,7 @@
 use crate::local_array::in_memory::atomic_types::NodeSet;
 use crate::prelude::multi::*;
 use crate::prelude::*;
+use rand::prelude::*;
 
 // The default stride sizes for IPv4, IPv6, resp.
 #[create_store((
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index f7563a8c..3560de81 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -1,5 +1,4 @@
 pub use crate::{AddressFamily, IPv4, IPv6};
-pub use uuid::Uuid;
 
 pub use crate::prefix_record::{Meta, PublicPrefixRecord as PrefixRecord};
 pub use crate::stride::{Stride3, Stride4, Stride5};

From 0dbb63264f6b2a026f3c05cbb27dd26f48aaf426 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 14 Feb 2025 14:02:08 +0100
Subject: [PATCH 076/147] add zerocopy to PrefixId & AddressFamily

---
 src/af.rs                              | 255 +++++++++++++++++--------
 src/local_array/in_memory/iterators.rs |   7 +-
 src/local_array/in_memory/tree.rs      |   8 +-
 src/local_array/prefix_cht/cht.rs      |   6 +-
 src/local_array/tests.rs               |   4 +-
 src/local_array/types.rs               |  60 ++++--
 src/local_vec/store.rs                 |  16 +-
 src/local_vec/tree.rs                  |  24 +--
 src/prefix_record.rs                   |  26 ++-
 src/prelude/mod.rs                     |   2 +
 10 files changed, 280 insertions(+), 128 deletions(-)

diff --git a/src/af.rs b/src/af.rs
index a3c93823..bd3609e9 100644
--- a/src/af.rs
+++ b/src/af.rs
@@ -1,3 +1,5 @@
+use zerocopy::{IntoBytes, NetworkEndian, U128, U32};
+
 use crate::local_array::bit_span::BitSpan;
 
 //------------ AddressFamily (trait) ----------------------------------------
@@ -12,9 +14,9 @@ pub trait AddressFamily:
     + std::fmt::Debug
     + std::hash::Hash
     + std::fmt::Display
-    + From<u32>
-    + From<u16>
-    + From<u8>
+    // + From<u32>
+    // + From<u16>
+    // + From<u8>
     + Eq
     + std::ops::BitAnd<Output = Self>
     + std::ops::BitOr<Output = Self>
@@ -22,14 +24,32 @@ pub trait AddressFamily:
     + std::ops::Shl<Output = Self>
     + std::ops::Shl<Self, Output = Self>
     + std::ops::Sub<Output = Self>
-    + Zero
+    // + Zero
     + Copy
     + Ord
+    + zerocopy::TryFromBytes
+    + zerocopy::IntoBytes
+    + zerocopy::KnownLayout
+    + zerocopy::Immutable
+    + zerocopy::Unaligned
 {
     /// The byte representation of the family filled with 1s.
-    const BITMASK: Self;
+    // const BITMASK: Self;
     /// The number of bits in the byte representation of the family.
     const BITS: u8;
+    type Inner: Into<Self> + From<u32> + From<u8>;
+    type InnerIpAddr;
+
+    fn new(value: Self::Inner) -> Self {
+        value.into()
+    }
+
+    fn from_ipaddr(ip_addr: Self::InnerIpAddr) -> Self;
+
+    fn from_u32(value: u32) -> Self;
+    fn from_u8(value: u8) -> Self;
+
+    fn zero() -> Self;
 
     fn fmt_net(net: Self) -> String;
     // returns the specified nibble from `start_bit` to (and including)
@@ -41,7 +61,7 @@ pub trait AddressFamily:
 
     fn truncate_to_len(self, len: u8) -> Self;
 
-    fn from_ipaddr(net: std::net::IpAddr) -> Self;
+    // fn from_ipaddr(net: std::net::IpAddr) -> Self;
 
     fn into_ipaddr(self) -> std::net::IpAddr;
 
@@ -49,7 +69,7 @@ pub trait AddressFamily:
     fn dangerously_truncate_to_u32(self) -> u32;
 
     // temporary function, this will botch IPv6 completely.
-    fn dangerously_truncate_to_usize(self) -> usize;
+    // fn dangerously_truncate_to_usize(self) -> usize;
 
     // For the sake of searching for 0/0, check the the right shift, since
     // since shifting with MAXLEN (32 in Ipv4, or 128 in IPv6) will panic
@@ -63,32 +83,52 @@ pub trait AddressFamily:
 //-------------- Ipv4 Type --------------------------------------------------
 
 /// Exactly fitting IPv4 bytes (4 octets).
-pub type IPv4 = u32;
+pub type IPv4 = zerocopy::U32<NetworkEndian>;
 
 impl AddressFamily for IPv4 {
-    const BITMASK: u32 = 0x1u32.rotate_right(1);
+    // const BITMASK: u32 = 0x1u32.rotate_right(1);
     const BITS: u8 = 32;
+    type Inner = u32;
+    type InnerIpAddr = std::net::Ipv4Addr;
+
+    fn zero() -> Self {
+        0.into()
+    }
+
+    fn from_u8(value: u8) -> Self {
+        IPv4::from([0, 0, 0, value])
+    }
+
+    fn from_u32(value: u32) -> Self {
+        IPv4::from(value)
+    }
+
+    fn from_ipaddr(ip_addr: Self::InnerIpAddr) -> Self {
+        IPv4::from(ip_addr.octets())
+    }
 
     fn fmt_net(net: Self) -> String {
-        std::net::Ipv4Addr::from(net).to_string()
+        std::net::Ipv4Addr::from(u32::from(net)).to_string()
     }
 
     fn get_nibble(net: Self, start_bit: u8, len: u8) -> u32 {
-        (net << start_bit) >> ((32 - len) % 32)
+        ((net << <U32<NetworkEndian>>::from(start_bit as u32))
+            >> <U32<NetworkEndian>>::from(((32 - len) % 32) as u32))
+        .into()
     }
 
     // You can't shift with the number of bits of self, so we'll just return
     // zero for that case.
     //
     // Panics if len is greater than 32 (the number of bits of self).
-    fn truncate_to_len(self, len: u8) -> Self {
-        match len {
-            0 => 0,
-            1..=31 => (self >> ((32 - len) as usize)) << (32 - len) as usize,
-            32 => self,
-            _ => panic!("Can't truncate to more than 32 bits"),
-        }
-    }
+    // fn truncate_to_len(self, len: u8) -> Self {
+    //     match len {
+    //         0 => U32::new(0),
+    //         1..=31 => (self >> (32 - len as u32).into()) << (32 - len).into(),
+    //         32 => self,
+    //         _ => panic!("Can't truncate to more than 32 bits"),
+    //     }
+    // }
 
     /// Treat self as a prefix and append the given nibble to it.
     ///
@@ -121,63 +161,92 @@ impl AddressFamily for IPv4 {
     /// let nibble = 0b1100110_u32;                             // 7-bit nibble
     /// let (new_prefix, new_len) = prefix.add_nibble(30, nibble, 7);
     /// ```
-    fn add_bit_span(self, len: u8, bs: BitSpan) -> (u32, u8) {
+    fn add_bit_span(self, len: u8, bs: BitSpan) -> (U32<NetworkEndian>, u8) {
         let res = self | (bs.bits << (32 - len - bs.len) as usize);
         (res, len + bs.len)
     }
 
-    fn from_ipaddr(addr: std::net::IpAddr) -> u32 {
-        // Well, this is awkward.
-        if let std::net::IpAddr::V4(addr) = addr {
-            (addr.octets()[0] as u32) << 24
-                | (addr.octets()[1] as u32) << 16
-                | (addr.octets()[2] as u32) << 8
-                | (addr.octets()[3] as u32)
-        } else {
-            panic!("Can't convert IPv6 to IPv4");
-        }
-    }
+    // fn from_ipaddr(addr: std::net::IpAddr) -> U32<NetworkEndian> {
+    //     // Well, this is awkward.
+    //     if let std::net::IpAddr::V4(addr) = addr {
+    //         (addr.octets()[0] as u32) << 24
+    //             | (addr.octets()[1] as u32) << 16
+    //             | (addr.octets()[2] as u32) << 8
+    //             | (addr.octets()[3] as u32)
+    //     } else {
+    //         panic!("Can't convert IPv6 to IPv4");
+    //     }
+    // }
 
     fn into_ipaddr(self) -> std::net::IpAddr {
-        std::net::IpAddr::V4(std::net::Ipv4Addr::from(self))
+        std::net::IpAddr::V4(std::net::Ipv4Addr::from(u32::from(self)))
     }
 
     fn dangerously_truncate_to_u32(self) -> u32 {
         // not dangerous at all.
-        self
+        self.into()
     }
 
-    fn dangerously_truncate_to_usize(self) -> usize {
-        // not dangerous at all.
-        self as usize
+    // fn dangerously_truncate_to_usize(self) -> usize {
+    //     // not dangerous at all.
+    //     <usize>::from(self)
+    // }
+
+    fn truncate_to_len(self, len: u8) -> Self {
+        match len {
+            0 => U32::new(0),
+            1..=31 => {
+                (self >> U32::from(32 - len as u32))
+                    << U32::from(32 - len as u32)
+            }
+            32 => self,
+            len => panic!("Can't truncate to more than 128 bits: {}", len),
+        }
     }
 
     fn checked_shr_or_zero(self, rhs: u32) -> Self {
-        self.checked_shr(rhs).unwrap_or(0)
+        if rhs == 0 {
+            return 0.into();
+        }
+        self >> U32::<NetworkEndian>::from(rhs)
     }
 
     fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
-        *u32::to_be_bytes(*self)
-            .first_chunk::<PREFIX_SIZE>()
-            .unwrap()
+        *self.as_bytes().first_chunk::<PREFIX_SIZE>().unwrap()
+        // *u32::to_be_bytes(*self)
+        //     .first_chunk::<PREFIX_SIZE>()
+        //     .unwrap()
     }
 }
 
 //-------------- Ipv6 Type --------------------------------------------------
 
 /// Exactly fitting IPv6 bytes (16 octets).
-pub type IPv6 = u128;
+pub type IPv6 = U128<NetworkEndian>;
 
 impl AddressFamily for IPv6 {
-    const BITMASK: u128 = 0x1u128.rotate_right(1);
+    // const BITMASK: u128 = 0x1u128.rotate_right(1);
     const BITS: u8 = 128;
+    type Inner = u128;
+    type InnerIpAddr = std::net::Ipv6Addr;
+
+    fn zero() -> Self {
+        0.into()
+    }
+
+    fn from_ipaddr(ip_addr: Self::InnerIpAddr) -> Self {
+        IPv6::from(ip_addr.octets())
+    }
 
     fn fmt_net(net: Self) -> String {
-        std::net::Ipv6Addr::from(net).to_string()
+        std::net::Ipv6Addr::from(u128::from(net)).to_string()
     }
 
     fn get_nibble(net: Self, start_bit: u8, len: u8) -> u32 {
-        ((net << start_bit) >> ((128 - len) % 128)) as u32
+        u128::from(
+            (net << <U128<NetworkEndian>>::from(start_bit as u128))
+                >> (<U128<NetworkEndian>>::from(128 - len as u128) % 128),
+        ) as u32
     }
 
     /// Treat self as a prefix and append the given nibble to it.
@@ -220,9 +289,10 @@ impl AddressFamily for IPv6 {
 
     fn truncate_to_len(self, len: u8) -> Self {
         match len {
-            0 => 0,
+            0 => U128::new(0),
             1..=127 => {
-                (self >> ((128 - len) as usize)) << (128 - len) as usize
+                (self >> U128::from(128 - len as u128))
+                    << U128::from(128 - len as u128)
             }
             128 => self,
             len => panic!("Can't truncate to more than 128 bits: {}", len),
@@ -237,51 +307,80 @@ impl AddressFamily for IPv6 {
     //     }
     // }
 
-    fn from_ipaddr(net: std::net::IpAddr) -> u128 {
-        if let std::net::IpAddr::V6(addr) = net {
-            addr.octets()[15] as u128
-                | (addr.octets()[14] as u128) << 8
-                | (addr.octets()[13] as u128) << 16
-                | (addr.octets()[12] as u128) << 24
-                | (addr.octets()[11] as u128) << 32
-                | (addr.octets()[10] as u128) << 40
-                | (addr.octets()[9] as u128) << 48
-                | (addr.octets()[8] as u128) << 56
-                | (addr.octets()[7] as u128) << 64
-                | (addr.octets()[6] as u128) << 72
-                | (addr.octets()[5] as u128) << 80
-                | (addr.octets()[4] as u128) << 88
-                | (addr.octets()[3] as u128) << 96
-                | (addr.octets()[2] as u128) << 104
-                | (addr.octets()[1] as u128) << 112
-                | (addr.octets()[0] as u128) << 120
-        } else {
-            panic!("Can't convert IPv4 to IPv6");
-        }
-    }
+    // fn from_ipaddr(net: std::net::IpAddr) -> u128 {
+    //     if let std::net::IpAddr::V6(addr) = net {
+    //         addr.octets()[15] as u128
+    //             | (addr.octets()[14] as u128) << 8
+    //             | (addr.octets()[13] as u128) << 16
+    //             | (addr.octets()[12] as u128) << 24
+    //             | (addr.octets()[11] as u128) << 32
+    //             | (addr.octets()[10] as u128) << 40
+    //             | (addr.octets()[9] as u128) << 48
+    //             | (addr.octets()[8] as u128) << 56
+    //             | (addr.octets()[7] as u128) << 64
+    //             | (addr.octets()[6] as u128) << 72
+    //             | (addr.octets()[5] as u128) << 80
+    //             | (addr.octets()[4] as u128) << 88
+    //             | (addr.octets()[3] as u128) << 96
+    //             | (addr.octets()[2] as u128) << 104
+    //             | (addr.octets()[1] as u128) << 112
+    //             | (addr.octets()[0] as u128) << 120
+    //     } else {
+    //         panic!("Can't convert IPv4 to IPv6");
+    //     }
+    // }
 
     fn into_ipaddr(self) -> std::net::IpAddr {
-        std::net::IpAddr::V6(std::net::Ipv6Addr::from(self))
+        std::net::IpAddr::V6(std::net::Ipv6Addr::from(u128::from(self)))
     }
 
     fn dangerously_truncate_to_u32(self) -> u32 {
         // this will chop off the high bits.
-        self as u32
+        u128::from(self) as u32
     }
 
-    fn dangerously_truncate_to_usize(self) -> usize {
-        // this will chop off the high bits.
-        self as usize
-    }
+    // fn dangerously_truncate_to_usize(self) -> usize {
+    //     // this will chop off the high bits.
+    //     self as usize
+    // }
 
     fn checked_shr_or_zero(self, rhs: u32) -> Self {
-        self.checked_shr(rhs).unwrap_or(0)
+        if rhs == 0 {
+            return U128::from(0);
+        };
+
+        self >> U128::from(rhs as u128)
     }
 
     fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
-        *u128::to_be_bytes(*self)
-            .first_chunk::<PREFIX_SIZE>()
-            .unwrap()
+        // *u128::to_be_bytes(*self)
+        //     .first_chunk::<PREFIX_SIZE>()
+        //     .unwrap()
+        *self.as_bytes().first_chunk::<PREFIX_SIZE>().unwrap()
+    }
+
+    fn from_u8(value: u8) -> Self {
+        IPv6::from([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, value])
+    }
+
+    fn from_u32(value: u32) -> Self {
+        (value as u128).into()
+    }
+}
+
+pub trait IntoIpAddr {
+    fn into_ipaddr(self) -> std::net::IpAddr;
+}
+
+impl IntoIpAddr for u32 {
+    fn into_ipaddr(self) -> std::net::IpAddr {
+        std::net::IpAddr::V4(std::net::Ipv4Addr::from(self))
+    }
+}
+
+impl IntoIpAddr for u128 {
+    fn into_ipaddr(self) -> std::net::IpAddr {
+        std::net::IpAddr::V6(std::net::Ipv6Addr::from(self))
     }
 }
 
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 2047c18e..9033cea7 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -426,7 +426,10 @@ impl<
 
     // Iterator over all the prefixes in the in_memory store.
     pub fn prefixes_iter(&'a self) -> impl Iterator<Item = Prefix> + 'a {
-        self.more_specific_prefix_iter_from(PrefixId::new(AF::zero(), 0))
-            .map(Prefix::from)
+        self.more_specific_prefix_iter_from(PrefixId::new(
+            AF::new(0_u32.into()),
+            0,
+        ))
+        .map(Prefix::from)
     }
 }
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 68081e1b..a77c158a 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -921,7 +921,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                     // which will panic in debug mode (undefined behaviour
                     // in prod).
                     BitSpan::new(
-                        ((prefix.get_net() << node_len.into())
+                        ((prefix.get_net() << AF::from_u8(node_len))
                             .checked_shr_or_zero(
                                 (AF::BITS - (prefix.get_len() - node_len))
                                     .into(),
@@ -1001,8 +1001,10 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
         // );
         // HASHING FUNCTION
-        ((id.get_id().0 << last_level.into())
-            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS).into())
+        ((id.get_id().0 << AF::from_u8(last_level))
+            >> AF::from_u8(
+                (<AF>::BITS - (this_level - last_level)) % <AF>::BITS,
+            ))
         .dangerously_truncate_to_u32() as usize
     }
 }
diff --git a/src/local_array/prefix_cht/cht.rs b/src/local_array/prefix_cht/cht.rs
index 1fe483ea..263c9a93 100644
--- a/src/local_array/prefix_cht/cht.rs
+++ b/src/local_array/prefix_cht/cht.rs
@@ -322,8 +322,10 @@ impl<AF: AddressFamily, M: Meta, PB: PrefixBuckets<AF, M>>
         //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
         // );
         // HASHING FUNCTION
-        ((id.get_net() << last_level.into())
-            >> ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS).into())
+        ((id.get_net() << AF::from_u32(last_level as u32))
+            >> AF::from_u8(
+                (<AF>::BITS - (this_level - last_level)) % <AF>::BITS,
+            ))
         .dangerously_truncate_to_u32() as usize
     }
 }
diff --git a/src/local_array/tests.rs b/src/local_array/tests.rs
index 4f4fd747..d494dcf5 100644
--- a/src/local_array/tests.rs
+++ b/src/local_array/tests.rs
@@ -10,7 +10,7 @@ fn test_af_1() -> Result<(), Box<dyn Error>> {
     use crate::AddressFamily;
     use crate::IPv4;
 
-    let bit_addr: IPv4 = 0b1111_1111_1111_1111_1111_1111_1111_1111;
+    let bit_addr: IPv4 = 0b1111_1111_1111_1111_1111_1111_1111_1111.into();
     let base_prefix =
         StrideNodeId::dangerously_new_with_id_as_is(bit_addr, 32);
 
@@ -42,7 +42,7 @@ fn test_af_2() -> Result<(), Box<dyn Error>> {
     use crate::prelude::multi::StrideNodeId;
     use crate::IPv4;
 
-    let bit_addr: IPv4 = 0b1111_1111_1111_1111_1111_1111_1111_1111;
+    let bit_addr: IPv4 = 0b1111_1111_1111_1111_1111_1111_1111_1111.into();
     let nu_prefix = StrideNodeId::dangerously_new_with_id_as_is(bit_addr, 8);
 
     assert_eq!(nu_prefix.get_id().0, bit_addr);
diff --git a/src/local_array/types.rs b/src/local_array/types.rs
index 7cbc9990..793bb3eb 100644
--- a/src/local_array/types.rs
+++ b/src/local_array/types.rs
@@ -1,4 +1,5 @@
 use log::trace;
+use zerocopy::TryFromBytes;
 
 use crate::AddressFamily;
 
@@ -45,16 +46,28 @@ impl TryFrom<u8> for RouteStatus {
 }
 
 //------------ PrefixId ------------------------------------------------------
-
-#[derive(Hash, Eq, PartialEq, Debug, Copy, Clone)]
+#[derive(
+    Hash,
+    Eq,
+    PartialEq,
+    Debug,
+    Copy,
+    Clone,
+    zerocopy::TryFromBytes,
+    zerocopy::IntoBytes,
+    zerocopy::KnownLayout,
+    zerocopy::Immutable,
+    zerocopy::Unaligned,
+)]
+#[repr(C)]
 pub struct PrefixId<AF: AddressFamily> {
-    net: AF,
     len: u8,
+    net: AF,
 }
 
 impl<AF: AddressFamily> PrefixId<AF> {
     pub fn new(net: AF, len: u8) -> Self {
-        PrefixId { net, len }
+        PrefixId { len, net }
     }
 
     pub fn get_net(&self) -> AF {
@@ -77,16 +90,16 @@ impl<AF: AddressFamily> PrefixId<AF> {
     }
 
     pub fn truncate_to_len(self, len: u8) -> Self {
-        trace!("orig {:032b}", self.net);
-        trace!(
-            "new  {:032b}",
-            self.net >> (AF::BITS - len).into() << (AF::BITS - len).into()
-        );
-        trace!(
-            "truncate to net {} len {}",
-            self.net >> (AF::BITS - len).into() << (AF::BITS - len).into(),
-            len
-        );
+        // trace!("orig {:032b}", self.net);
+        // trace!(
+        //     "new  {:032b}",
+        //     self.net >> (AF::BITS - len).into() << (AF::BITS - len).into()
+        // );
+        // trace!(
+        //     "truncate to net {} len {}",
+        //     self.net >> (AF::BITS - len).into() << (AF::BITS - len).into(),
+        //     len
+        // );
         Self {
             // net: (self.net >> (AF::BITS - len)) << (AF::BITS - len),
             net: self.net.truncate_to_len(len),
@@ -110,7 +123,14 @@ impl<AF: AddressFamily> PrefixId<AF> {
 impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
     fn from(value: inetnum::addr::Prefix) -> Self {
         Self {
-            net: AF::from_ipaddr(value.addr()),
+            net: match value.addr() {
+                std::net::IpAddr::V4(addr) => {
+                    *AF::try_ref_from_bytes(&addr.octets()).unwrap()
+                }
+                std::net::IpAddr::V6(addr) => {
+                    *AF::try_ref_from_bytes(&addr.octets()).unwrap()
+                }
+            },
             len: value.len(),
         }
     }
@@ -127,8 +147,16 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<[u8; PREFIX_SIZE]>
 {
     fn from(value: [u8; PREFIX_SIZE]) -> Self {
         Self {
-            net: u32::from_be_bytes(*value.last_chunk::<4>().unwrap()).into(),
+            net: *AF::try_ref_from_bytes(&value.as_slice()[1..]).unwrap(),
             len: value[0],
         }
     }
 }
+
+impl<'a, AF: AddressFamily, const PREFIX_SIZE: usize>
+    From<&'a [u8; PREFIX_SIZE]> for &'a PrefixId<AF>
+{
+    fn from(value: &'a [u8; PREFIX_SIZE]) -> Self {
+        PrefixId::try_ref_from_bytes(value.as_slice()).unwrap()
+    }
+}
diff --git a/src/local_vec/store.rs b/src/local_vec/store.rs
index d9481d92..080ce669 100644
--- a/src/local_vec/store.rs
+++ b/src/local_vec/store.rs
@@ -3,7 +3,7 @@ use crate::local_vec::storage_backend::{InMemStorage, StorageBackend};
 use crate::local_vec::TreeBitMap;
 use crate::node_id::InMemNodeId;
 use crate::prefix_record::InternalPrefixRecord;
-use crate::{MatchOptions, Stats, Strides};
+use crate::{AddressFamily, MatchOptions, Stats, Strides};
 
 use crate::af::{IPv4, IPv6};
 use inetnum::addr::Prefix;
@@ -37,11 +37,17 @@ impl<M: crate::prefix_record::Meta> Store<M> {
     ) -> QuerySingleResult<M> {
         match search_pfx.addr() {
             std::net::IpAddr::V4(addr) => self.v4.match_prefix(
-                PrefixId::<IPv4>::new(addr.into(), search_pfx.len()),
+                PrefixId::<IPv4>::new(
+                    <IPv4 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
                 options,
             ),
             std::net::IpAddr::V6(addr) => self.v6.match_prefix(
-                PrefixId::<IPv6>::new(addr.into(), search_pfx.len()),
+                PrefixId::<IPv6>::new(
+                    <IPv6 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
                 options,
             ),
         }
@@ -56,14 +62,14 @@ impl<M: crate::prefix_record::Meta> Store<M> {
         match prefix.addr() {
             std::net::IpAddr::V4(addr) => {
                 self.v4.insert(InternalPrefixRecord::new_with_meta(
-                    addr.into(),
+                    <IPv4 as AddressFamily>::from_ipaddr(addr),
                     prefix.len(),
                     meta,
                 ))
             }
             std::net::IpAddr::V6(addr) => {
                 self.v6.insert(InternalPrefixRecord::new_with_meta(
-                    addr.into(),
+                    <IPv6 as AddressFamily>::from_ipaddr(addr),
                     prefix.len(),
                     meta,
                 ))
diff --git a/src/local_vec/tree.rs b/src/local_vec/tree.rs
index 4b87f028..9d68e8f3 100644
--- a/src/local_vec/tree.rs
+++ b/src/local_vec/tree.rs
@@ -479,18 +479,18 @@ impl<Store: StorageBackend> std::fmt::Display for TreeBitMap<Store> {
         writeln!(_f, "prefix vec size {}", self.store.get_prefixes_len())?;
         writeln!(_f, "finished building tree...")?;
         writeln!(_f, "{:?} nodes created", total_nodes)?;
-        writeln!(
-            _f,
-            "size of node: {} bytes",
-            std::mem::size_of::<SizedStrideNode<u32, InMemNodeId>>()
-        )?;
-        writeln!(
-            _f,
-            "memory used by nodes: {}kb",
-            self.store.get_nodes_len()
-                * std::mem::size_of::<SizedStrideNode<u32, InMemNodeId>>()
-                / 1024
-        )?;
+        // writeln!(
+        //     _f,
+        //     "size of node: {} bytes",
+        //     std::mem::size_of::<SizedStrideNode<u32, InMemNodeId>>()
+        // )?;
+        // writeln!(
+        //     _f,
+        //     "memory used by nodes: {}kb",
+        //     self.store.get_nodes_len()
+        //         * std::mem::size_of::<SizedStrideNode<u32, InMemNodeId>>()
+        //         / 1024
+        // )?;
 
         writeln!(_f, "stride division {:?}", self.strides)?;
         for s in &self.stats {
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index 41b19b92..70700d79 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -6,6 +6,7 @@ use crate::af::AddressFamily;
 use crate::local_array::types::RouteStatus;
 use crate::prelude::multi::PrefixId;
 use inetnum::addr::Prefix;
+use zerocopy::{NetworkEndian, U128, U32};
 
 //------------ InternalPrefixRecord -----------------------------------------
 
@@ -72,8 +73,8 @@ where
     AF: AddressFamily,
 {
     fn cmp(&self, other: &Self) -> Ordering {
-        (self.net >> (AF::BITS - self.len).into())
-            .cmp(&(other.net >> ((AF::BITS - other.len) % 32).into()))
+        (self.net >> AF::from_u8(AF::BITS - self.len))
+            .cmp(&(other.net >> AF::from_u8((AF::BITS - other.len) % 32)))
     }
 }
 
@@ -83,8 +84,8 @@ where
     AF: AddressFamily,
 {
     fn eq(&self, other: &Self) -> bool {
-        self.net >> (AF::BITS - self.len).into()
-            == other.net >> ((AF::BITS - other.len) % 32).into()
+        self.net >> AF::from_u8(AF::BITS - self.len)
+            == other.net >> AF::from_u8((AF::BITS - other.len) % 32)
     }
 }
 
@@ -95,8 +96,9 @@ where
 {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(
-            (self.net >> (AF::BITS - self.len).into())
-                .cmp(&(other.net >> ((AF::BITS - other.len) % 32).into())),
+            (self.net >> AF::from_u8(AF::BITS - self.len)).cmp(
+                &(other.net >> AF::from_u8((AF::BITS - other.len) % 32)),
+            ),
         )
     }
 }
@@ -159,7 +161,11 @@ impl<M: Meta> From<PublicPrefixSingleRecord<M>>
 {
     fn from(record: PublicPrefixSingleRecord<M>) -> Self {
         Self {
-            net: crate::IPv4::from_ipaddr(record.prefix.addr()),
+            net: if let std::net::IpAddr::V4(ip) = record.prefix.addr() {
+                U32::<NetworkEndian>::from(ip.octets())
+            } else {
+                0.into()
+            },
             len: record.prefix.len(),
             meta: record.meta,
         }
@@ -171,7 +177,11 @@ impl<M: Meta> From<PublicPrefixSingleRecord<M>>
 {
     fn from(record: PublicPrefixSingleRecord<M>) -> Self {
         Self {
-            net: crate::IPv6::from_ipaddr(record.prefix.addr()),
+            net: if let std::net::IpAddr::V6(ip) = record.prefix.addr() {
+                U128::<NetworkEndian>::from(ip.octets())
+            } else {
+                0.into()
+            },
             len: record.prefix.len(),
             meta: record.meta,
         }
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index 3560de81..a9ab37f3 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -1,5 +1,6 @@
 pub use crate::{AddressFamily, IPv4, IPv6};
 
+pub use crate::af::IntoIpAddr;
 pub use crate::prefix_record::{Meta, PublicPrefixRecord as PrefixRecord};
 pub use crate::stride::{Stride3, Stride4, Stride5};
 pub use crate::{IncludeHistory, MatchOptions, MatchType, QueryResult};
@@ -7,6 +8,7 @@ pub use inetnum::addr::Prefix;
 
 pub mod multi {
     pub use crate::MultiThreadedStore;
+
     pub use std::sync::atomic::Ordering;
 
     pub use rotonda_macros::create_store;

From 43368ed3329c3de8f9b8f9c83b3d47e6cf4f0677 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 14 Feb 2025 14:02:22 +0100
Subject: [PATCH 077/147] add zerocopy dep

---
 Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Cargo.toml b/Cargo.toml
index 76d3a4df..5081473f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -45,6 +45,7 @@ serde = "1.0.216"
 serde_derive = "1.0.216"
 serde_json = "1.0.133"
 num-traits = "0.2.19"
+zerocopy = { version = "0.8.17", features = ["derive"] }
 
 [dev-dependencies]
 csv         = { version = "1" }

From c8cff45b80ebd3f13b6070731da19ed7e1684884 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Mon, 24 Feb 2025 17:40:51 +0100
Subject: [PATCH 078/147] add Storage Config type parameter to Store

---
 examples/exact_matches.rs                  |    4 +-
 examples/full_table_multiple_trees_json.rs |    5 +-
 examples/more_specifics.rs                 |    5 +-
 examples/multi_no_thread.rs                |    4 +-
 examples/multi_single_thread.rs            |    4 +-
 examples/multi_thread_1.rs                 |    5 +-
 examples/multi_thread_2.rs                 |    7 +-
 examples/multi_thread_3.rs                 |    7 +-
 examples/multi_thread_4.rs                 |   71 +-
 examples/multi_thread_multi_prefix.rs      |    9 +-
 examples/multi_thread_single_prefix.rs     |   11 +-
 examples/numbers_treebitmap.rs             |    4 +-
 examples/single_thread_24.rs               |    8 +-
 examples/treebitmap.rs                     |    4 +-
 proc_macros/src/lib.rs                     |  199 ++-
 src/bin/cli.rs                             |    4 +-
 src/bin/load_mrt.rs                        |  209 ++-
 src/local_array/persist/lsm_tree.rs        | 1539 +++++++++++---------
 src/local_array/query.rs                   |   76 +-
 src/local_array/rib/default_store.rs       |   61 +-
 src/local_array/rib/mod.rs                 |    1 +
 src/local_array/rib/rib.rs                 |  436 ++++--
 src/local_array/types.rs                   |   23 +-
 src/macros.rs                              |   38 +-
 src/prefix_record.rs                       |   43 +-
 src/prelude/mod.rs                         |    8 +-
 tests/best-path.rs                         |   18 +-
 tests/concurrency.rs                       |   34 +-
 tests/full-table.rs                        |    4 +-
 tests/more-more-specifics.rs               |   11 +-
 tests/more-specifics.rs                    |    4 +-
 tests/treebitmap.rs                        |  157 +-
 tests/treebitmap_v6.rs                     |   35 +-
 33 files changed, 1945 insertions(+), 1103 deletions(-)

diff --git a/examples/exact_matches.rs b/examples/exact_matches.rs
index 16d3a5b3..f69b6183 100644
--- a/examples/exact_matches.rs
+++ b/examples/exact_matches.rs
@@ -1,10 +1,12 @@
 use rotonda_store::meta_examples::NoMeta;
 use rotonda_store::prelude::multi::*;
+use rotonda_store::rib::MemoryOnlyConfig;
 use rotonda_store::{prelude::*, IncludeHistory};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let guard = &epoch::pin();
-    let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
+    let tree_bitmap =
+        MultiThreadedStore::<NoMeta, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new_relaxed(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index 82470e25..b8ba41ce 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -2,6 +2,7 @@
 use rotonda_store::meta_examples::PrefixAs;
 use rotonda_store::prelude::multi::*;
 use rotonda_store::prelude::*;
+use rotonda_store::rib::MemoryOnlyConfig;
 
 use std::error::Error;
 use std::fs::File;
@@ -51,9 +52,9 @@ fn main() -> Result<(), Box<dyn Error>> {
         println!("[");
         for n in 1..6 {
             let mut rec_vec: Vec<PrefixRecord<PrefixAs>> = vec![];
-            let config = StoreConfig::default();
+            let config = MemoryOnlyConfig;
             let tree_bitmap =
-                MultiThreadedStore::<PrefixAs>::new_with_config(config)?;
+                MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::new_with_config(config)?;
 
             if let Err(err) = load_prefixes(&mut rec_vec) {
                 println!("error running example: {}", err);
diff --git a/examples/more_specifics.rs b/examples/more_specifics.rs
index e982cbe9..a3f893d0 100644
--- a/examples/more_specifics.rs
+++ b/examples/more_specifics.rs
@@ -3,11 +3,12 @@ use rotonda_store::prelude::multi::*;
 use rotonda_store::prelude::*;
 
 use inetnum::addr::Prefix;
-use rotonda_store::AddressFamily;
+use rotonda_store::rib::MemoryOnlyConfig;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // type StoreType = InMemStorage<u32, PrefixAs>;
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    let tree_bitmap =
+        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new_relaxed(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
diff --git a/examples/multi_no_thread.rs b/examples/multi_no_thread.rs
index b6a6291b..a5924214 100644
--- a/examples/multi_no_thread.rs
+++ b/examples/multi_no_thread.rs
@@ -3,13 +3,15 @@ use log::trace;
 use rotonda_store::meta_examples::PrefixAs;
 use rotonda_store::prelude::multi::*;
 use rotonda_store::prelude::*;
+use rotonda_store::rib::MemoryOnlyConfig;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    let tree_bitmap =
+        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     // let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_single_thread.rs b/examples/multi_single_thread.rs
index c697ecdc..bf93f82f 100644
--- a/examples/multi_single_thread.rs
+++ b/examples/multi_single_thread.rs
@@ -1,4 +1,5 @@
 use log::trace;
+use rotonda_store::rib::MemoryOnlyConfig;
 use std::thread;
 use std::time::Duration;
 
@@ -14,7 +15,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    let tree_bitmap =
+        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     // let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_thread_1.rs b/examples/multi_thread_1.rs
index 441542c7..7672cff3 100644
--- a/examples/multi_thread_1.rs
+++ b/examples/multi_thread_1.rs
@@ -3,9 +3,12 @@ use std::{sync::Arc, thread};
 use rotonda_store::meta_examples::NoMeta;
 use rotonda_store::prelude::multi::*;
 use rotonda_store::prelude::*;
+use rotonda_store::rib::MemoryOnlyConfig;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let tree_bitmap = Arc::new(MultiThreadedStore::<NoMeta>::try_default()?);
+    let tree_bitmap = Arc::new(
+        MultiThreadedStore::<NoMeta, MemoryOnlyConfig>::try_default()?,
+    );
 
     let _: Vec<_> = (0..16)
         .map(|i: i32| {
diff --git a/examples/multi_thread_2.rs b/examples/multi_thread_2.rs
index 53301668..9175e471 100644
--- a/examples/multi_thread_2.rs
+++ b/examples/multi_thread_2.rs
@@ -1,4 +1,5 @@
 use log::trace;
+use rotonda_store::rib::MemoryOnlyConfig;
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
@@ -12,8 +13,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap =
-        Arc::new(MultiThreadedStore::<PrefixAs>::try_default()?);
+    let tree_bitmap = Arc::new(MultiThreadedStore::<
+        PrefixAs,
+        MemoryOnlyConfig,
+    >::try_default()?);
     let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_thread_3.rs b/examples/multi_thread_3.rs
index 0be99aa2..035d6f18 100644
--- a/examples/multi_thread_3.rs
+++ b/examples/multi_thread_3.rs
@@ -1,4 +1,5 @@
 use log::trace;
+use rotonda_store::rib::MemoryOnlyConfig;
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
@@ -12,8 +13,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap =
-        Arc::new(MultiThreadedStore::<PrefixAs>::try_default()?);
+    let tree_bitmap = Arc::new(MultiThreadedStore::<
+        PrefixAs,
+        MemoryOnlyConfig,
+    >::try_default()?);
     let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_thread_4.rs b/examples/multi_thread_4.rs
index 627fc8ad..ecca4f12 100644
--- a/examples/multi_thread_4.rs
+++ b/examples/multi_thread_4.rs
@@ -1,49 +1,51 @@
 use inetnum::asn::Asn;
 use log::trace;
+use rotonda_store::rib::MemoryOnlyConfig;
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
-
-use rotonda_store::prelude::*;
 use rotonda_store::prelude::multi::*;
+use rotonda_store::prelude::*;
 
+#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
+pub struct BytesPrefixAs(pub [u8; 4]);
 
-    #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
-    pub struct BytesPrefixAs(pub [u8; 4]);
-
-    impl AsRef<[u8]> for BytesPrefixAs {
-        fn as_ref(&self) -> &[u8] {
-            self.0.as_ref()
-        }
+impl AsRef<[u8]> for BytesPrefixAs {
+    fn as_ref(&self) -> &[u8] {
+        self.0.as_ref()
     }
+}
 
-    impl From<Vec<u8>> for BytesPrefixAs {
-        fn from(value: Vec<u8>) -> Self {
-            Self(*value.first_chunk::<4>().unwrap())
-        }
+impl From<Vec<u8>> for BytesPrefixAs {
+    fn from(value: Vec<u8>) -> Self {
+        Self(*value.first_chunk::<4>().unwrap())
     }
+}
 
-    impl Meta for BytesPrefixAs {
-        type Orderable<'a> = Asn;
-        type TBI = ();
+impl Meta for BytesPrefixAs {
+    type Orderable<'a> = Asn;
+    type TBI = ();
 
-        fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
-            u32::from_be_bytes(self.0).into()
-        }
+    fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
+        u32::from_be_bytes(self.0).into()
     }
+}
 
-    impl std::fmt::Display for BytesPrefixAs {
-        fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-            write!(f, "AS{:?}", self.0)
-        }
+impl std::fmt::Display for BytesPrefixAs {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "AS{:?}", self.0)
     }
+}
 
-fn main() -> Result<(), Box<dyn std::error::Error>> {    
+fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = Arc::new(MultiThreadedStore::<BytesPrefixAs>::try_default()?);
+    let tree_bitmap = Arc::new(MultiThreadedStore::<
+        BytesPrefixAs,
+        MemoryOnlyConfig,
+    >::try_default()?);
     let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
@@ -76,13 +78,21 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                 0,
                                 RouteStatus::Active,
                                 BytesPrefixAs((i as u32).to_be_bytes()),
-
                             ),
-                            None
+                            None,
                         ) {
                             Ok(metrics) => {
                                 if metrics.cas_count > 0 {
-                                    eprintln!("{} {} {:?} retry count: {},", std::thread::current().name().unwrap(), metrics.prefix_new, pfx, metrics.cas_count);
+                                    eprintln!(
+                                        "{} {} {:?}
+                                        retry count: {},",
+                                        std::thread::current()
+                                            .name()
+                                            .unwrap(),
+                                        metrics.prefix_new,
+                                        pfx,
+                                        metrics.cas_count
+                                    );
                                 }
                             }
                             Err(e) => {
@@ -92,15 +102,14 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
                         if x % 1_000_000 == 0 {
                             println!(
-                                "{:?} {} (prefixes count: {}, nodes count: {}",
+                                "{:?} {} (prefixes count: {},
+                                nodes count: {}",
                                 std::thread::current().name(),
                                 x,
                                 tree_bitmap.prefixes_count(),
                                 tree_bitmap.nodes_count()
                             );
                         }
-
-                    
                     }
                 },
             )
diff --git a/examples/multi_thread_multi_prefix.rs b/examples/multi_thread_multi_prefix.rs
index 908a8428..446c0a38 100644
--- a/examples/multi_thread_multi_prefix.rs
+++ b/examples/multi_thread_multi_prefix.rs
@@ -1,6 +1,7 @@
 use log::trace;
 
 use rotonda_store::prelude::multi::*;
+use rotonda_store::rib::MemoryOnlyConfig;
 use std::sync::atomic::AtomicU32;
 use std::sync::Arc;
 use std::thread;
@@ -18,7 +19,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     trace!("Starting multi-threaded yolo testing....");
     let tree_bitmap =
-        Arc::new(MultiThreadedStore::<PrefixAs>::try_default()?);
+        Arc::new(MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?);
     // let pfx = Prefix::new_relaxed(
     //     0b1111_1111_1111_1111_1111_1111_1111_1111_u32.into_ipaddr(),
     //     32,
@@ -34,8 +35,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             .name(i.to_string())
             .spawn(
                 move || -> Result<(), Box<dyn std::error::Error + Send>> {
-                    let mut rng = rand::thread_rng();
-
+                    let mut rng = rand::rng(); 
+                    
                     // println!("park thread {}", i);
                     thread::park();
 
@@ -46,7 +47,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                         let pfx = Prefix::new_relaxed(pfx_int.clone().load(std::sync::atomic::Ordering::Relaxed).into_ipaddr(), 32).unwrap();
                         let guard = &crossbeam_epoch::pin();
                         while x < 100 {
-                            let asn = PrefixAs::new_from_u32(rng.gen());
+                            let asn = PrefixAs::new_from_u32(rng.random());
                             match tree_bitmap.insert(
                                 &pfx,
                                 Record::new(0, 0, RouteStatus::Active, asn),
diff --git a/examples/multi_thread_single_prefix.rs b/examples/multi_thread_single_prefix.rs
index db3af6e1..304e6408 100644
--- a/examples/multi_thread_single_prefix.rs
+++ b/examples/multi_thread_single_prefix.rs
@@ -1,6 +1,7 @@
 use log::trace;
 
 use rotonda_store::prelude::multi::*;
+use rotonda_store::rib::MemoryOnlyConfig;
 use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
@@ -16,8 +17,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap =
-        Arc::new(MultiThreadedStore::<PrefixAs>::try_default()?);
+    let tree_bitmap = Arc::new(MultiThreadedStore::<
+        PrefixAs,
+        MemoryOnlyConfig,
+    >::try_default()?);
 
     let pfx = Prefix::new_relaxed(
         0b1111_1111_1111_1111_1111_1111_1111_1111_u32.into_ipaddr(),
@@ -31,7 +34,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             .name(i.to_string())
             .spawn(
                 move || -> Result<(), Box<dyn std::error::Error + Send>> {
-                    let mut rng = rand::thread_rng();
+                    let mut rng = rand::rng();
 
                     println!("park thread {}", i);
                     thread::park();
@@ -42,7 +45,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                     loop {
                         let guard = &crossbeam_epoch::pin();
                         while x < 10_000 {
-                            let asn = PrefixAs::new_from_u32(rng.gen());
+                            let asn = PrefixAs::new_from_u32(rng.random());
                             match tree_bitmap.insert(
                                 &pfx.unwrap(),
                                 Record::new(0, 0, RouteStatus::Active, asn),
diff --git a/examples/numbers_treebitmap.rs b/examples/numbers_treebitmap.rs
index 421b1963..fee74030 100644
--- a/examples/numbers_treebitmap.rs
+++ b/examples/numbers_treebitmap.rs
@@ -1,6 +1,7 @@
 use rotonda_store::meta_examples::PrefixAs;
 use rotonda_store::prelude::multi::*;
 use rotonda_store::prelude::*;
+use rotonda_store::rib::MemoryOnlyConfig;
 
 use std::env;
 use std::error::Error;
@@ -62,9 +63,8 @@ fn main() -> Result<(), Box<dyn Error>> {
 
     for _strides in strides_vec.iter() {
         let mut pfxs: Vec<PrefixRecord<PrefixAs>> = vec![];
-        let config = StoreConfig::default();
         let tree_bitmap =
-            MultiThreadedStore::<PrefixAs>::new_with_config(config)?;
+            MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
 
         if let Err(err) = load_prefixes(&mut pfxs) {
             println!("error running example: {}", err);
diff --git a/examples/single_thread_24.rs b/examples/single_thread_24.rs
index 80a67801..abde1c1f 100644
--- a/examples/single_thread_24.rs
+++ b/examples/single_thread_24.rs
@@ -1,4 +1,5 @@
 use log::trace;
+use rotonda_store::rib::MemoryOnlyConfig;
 use std::thread;
 use std::time::Duration;
 
@@ -14,7 +15,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    let tree_bitmap =
+        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     // let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let mut pfx_int = 0_u32;
@@ -28,7 +30,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .name(1_u8.to_string())
         .spawn(move || -> Result<(), Box<dyn std::error::Error + Send>> {
             // while !start_flag.load(std::sync::atomic::Ordering::Acquire) {
-            let mut rng = rand::thread_rng();
+            let mut rng = rand::rng();
 
             println!("park thread {}", 1);
             thread::park();
@@ -41,7 +43,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let pfx = Prefix::new_relaxed(pfx_int.into_ipaddr(), 32);
                 // x += 1;
                 // print!("{}-", i);
-                let asn: u32 = rng.gen();
+                let asn: u32 = rng.random();
                 match tree_bitmap.insert(
                     &pfx.unwrap(),
                     Record::new(
diff --git a/examples/treebitmap.rs b/examples/treebitmap.rs
index b208e01b..741e7067 100644
--- a/examples/treebitmap.rs
+++ b/examples/treebitmap.rs
@@ -2,11 +2,13 @@ use inetnum::addr::Prefix;
 use rotonda_store::meta_examples::NoMeta;
 use rotonda_store::prelude::multi::*;
 use rotonda_store::prelude::*;
+use rotonda_store::rib::MemoryOnlyConfig;
 
 type Prefix4<'a> = Prefix;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let tree_bitmap = MultiThreadedStore::try_default()?;
+    let tree_bitmap =
+        MultiThreadedStore::<_, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 0a03bf5b..f860a504 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -28,7 +28,7 @@ pub fn stride_sizes(
         _ => panic!("Expected Adress Family Type"),
     };
 
-    let prefix_size = match attrs
+    let _prefix_size = match attrs
         .get(2)
         .unwrap_or_else(|| panic!("Missing Prefix Size for Address Family"))
     {
@@ -44,6 +44,22 @@ pub fn stride_sizes(
         l => panic!("Expected Key Size for Address Family, got {:?}", l),
     };
 
+    // let _config = match attrs
+    //     .get(4)
+    //     .unwrap_or_else(|| panic!("Missing config type for store"))
+    // {
+    //     syn::Expr::Path(p) => p,
+    //     p => panic!("Expected Config type, got {:?}", p),
+    // };
+
+    let key_type = match attrs
+        .get(4)
+        .unwrap_or_else(|| panic!("Missing Key Type for Persist Strategy"))
+    {
+        syn::Expr::Path(l) => l,
+        l => panic!("Expected Key type for Persist Strategy, got {:?}", l),
+    };
+
     let prefixes_all_len;
     let all_len;
     let prefixes_buckets_name: syn::Ident;
@@ -327,7 +343,7 @@ pub fn stride_sizes(
     };
 
     let type_alias = quote! {
-        type #type_name<M> = Rib<#ip_af, M, #buckets_name<#ip_af>, #prefixes_buckets_name<#ip_af, M>, #prefix_size, #key_size>;
+        type #type_name<M, C> = Rib<#ip_af, M, #key_type<#ip_af>, #buckets_name<#ip_af>, #prefixes_buckets_name<#ip_af, M>, C, #key_size>;
     };
 
     let result = quote! {
@@ -403,7 +419,7 @@ pub fn create_store(
     struct_def: TokenStream,
 ) -> TokenStream {
     let struct_def = parse_macro_input!(struct_def as syn::ItemStruct);
-    let store_name = &struct_def.ident;
+    let rib_name = &struct_def.ident;
 
     let attr = parse_macro_input!(attr as syn::ExprTuple);
     let attrs = attr.elems.iter().collect::<Vec<_>>();
@@ -451,18 +467,34 @@ pub fn create_store(
         panic!("Expected Prefix Size for IPv6, got {:?}", tuple_6.attrs)
     });
 
-    let strides4_name = format_ident!("{}IPv4", store_name);
-    let strides6_name = format_ident!("{}IPv6", store_name);
+    // let config_type4 = tuple_4.elems.get(3).unwrap_or_else(|| {
+    //     panic!("Expected config type for IPv4, got {:?}", tuple_4.attrs)
+    // });
+
+    // let config_type6 = tuple_6.elems.get(3).unwrap_or_else(|| {
+    //     panic!("Expected config type for IPv6, got {:?}", tuple_6.attrs)
+    // });
+
+    let key_type4 = tuple_4.elems.get(3).unwrap_or_else(|| {
+        panic!("Expected Key Type, got {:?}", tuple_4.elems)
+    });
+
+    let key_type6 = tuple_6.elems.get(3).unwrap_or_else(|| {
+        panic!("Expected Key Type, got {:?}", tuple_6.elems)
+    });
+
+    let ipv4_rib_name = format_ident!("{}IPv4", rib_name);
+    let ipv6_rib_name = format_ident!("{}IPv6", rib_name);
 
     let create_strides = quote! {
         use ::std::marker::PhantomData;
         use ::inetnum::addr::Prefix;
 
-        #[stride_sizes((IPv4, #strides4, #key_size4, #prefix_size4))]
-        struct #strides4_name;
+        #[stride_sizes((IPv4, #strides4, #key_size4, #prefix_size4, #key_type4))]
+        struct #ipv4_rib_name;
 
-        #[stride_sizes((IPv6, #strides6, #key_size6, #prefix_size6))]
-        struct #strides6_name;
+        #[stride_sizes((IPv6, #strides6, #key_size6, #prefix_size6, #key_type6))]
+        struct #ipv6_rib_name;
     };
 
     let store = quote! {
@@ -492,17 +524,19 @@ pub fn create_store(
         ///
         /// This way the store can hold RIBs for multiple peers in one
         /// data-structure.
-        pub struct #store_name<
-            M: Meta
+        pub struct #rib_name<
+            M: Meta,
+            C: Config
         > {
-            v4: #strides4_name<M>,
-            v6: #strides6_name<M>,
-            config: StoreConfig
+            v4: #ipv4_rib_name<M, C>,
+            v6: #ipv6_rib_name<M, C>,
+            config: C
         }
 
         impl<
-                M: Meta
-            > #store_name<M>
+                M: Meta,
+                C: Config
+            > #rib_name<M, C>
         {
             /// Creates a new empty store with a tree for IPv4 and on for IPv6.
             ///
@@ -555,7 +589,7 @@ pub fn create_store(
             ///      }).map(|t| t.join()).collect();
             /// ```
             pub fn new_with_config(
-                    mut config: StoreConfig
+                    mut config: C
                 ) -> Result<Self, Box<dyn std::error::Error>> {
 
                 let rng = rand::rng();
@@ -569,22 +603,83 @@ pub fn create_store(
                 let mut config_v4 = config.clone();
                 let mut config_v6 = config.clone();
 
-                config_v4.persist_path = format!(
-                    "{}/{}/ipv4/", config_v4.persist_path, uuid);
+                if let Some(path) = config_v4.persist_path() {
+                    let pp = format!("{}/{}/ipv4/", path, uuid);
+                    config_v4.set_persist_path(pp);
+                };
 
-                config_v6.persist_path = format!(
-                    "{}/{}/ipv6/", config.persist_path, uuid);
+                if let Some(path) = config_v6.persist_path() {
+                    config_v6.set_persist_path(
+                        format!("{}/{}/ipv6/", path, uuid)
+                    );
+                }
 
                 Ok(Self {
-                    v4: #strides4_name::new(config_v4)?,
-                    v6: #strides6_name::new(config_v6)?,
+                    v4: #ipv4_rib_name::new(config_v4)?,
+                    v6: #ipv6_rib_name::new(config_v6)?,
                     config
                 })
             }
+
+            // pub fn new_with_short_key_persist(
+            //         mut config: StoreConfig
+            //     ) -> Result<Self, Box<dyn std::error::Error>> {
+
+            //     let rng = rand::rng();
+            //     let uuid: String = rng
+            //         .sample_iter(
+            //             rand::distr::Alphanumeric
+            //         )
+            //         .take(12)
+            //         .map(|b| char::from(b))
+            //         .collect();
+            //     let mut config_v4 = config.clone();
+            //     let mut config_v6 = config.clone();
+
+            //     config_v4.persist_path = format!(
+            //         "{}/{}/ipv4/", config_v4.persist_path, uuid);
+
+            //     config_v6.persist_path = format!(
+            //         "{}/{}/ipv6/", config.persist_path, uuid);
+
+            //     Ok(Self {
+            //         v4: #ipv4_rib_name::new_short_key(config_v4)?,
+            //         v6: #ipv6_rib_name::new_short_key(config_v6)?,
+            //         config
+            //     })
+            // }
+
+            // pub fn new_with_long_key_persist(
+            //         mut config: StoreConfig
+            //     ) -> Result<Self, Box<dyn std::error::Error>> {
+
+            //     let rng = rand::rng();
+            //     let uuid: String = rng
+            //         .sample_iter(
+            //             rand::distr::Alphanumeric
+            //         )
+            //         .take(12)
+            //         .map(|b| char::from(b))
+            //         .collect();
+            //     let mut config_v4 = config.clone();
+            //     let mut config_v6 = config.clone();
+
+            //     config_v4.persist_path = format!(
+            //         "{}/{}/ipv4/", config_v4.persist_path, uuid);
+
+            //     config_v6.persist_path = format!(
+            //         "{}/{}/ipv6/", config.persist_path, uuid);
+
+            //     Ok(Self {
+            //         v4: #ipv4_rib_name::new_long_key(config_v4)?,
+            //         v6: #ipv6_rib_name::new_long_key(config_v6)?,
+            //         config
+            //     })
+            // }
         }
 
-        impl<'a, M: Meta,
-            > #store_name<M>
+        impl<'a, M: Meta, C: Config
+            > #rib_name<M, C>
         {
             /// Search for and return one or more prefixes that match the
             ///given  `search_pfx` argument.   The search will return a
@@ -696,7 +791,8 @@ pub fn create_store(
                     std::net::IpAddr::V4(addr) => {
                         self.v4.match_prefix(
                             PrefixId::<IPv4>::new(
-                                addr.into(),
+                                <IPv4 as AddressFamily>::
+                                    from_ipaddr(addr),
                                 search_pfx.len(),
                             ),
                             options,
@@ -706,7 +802,8 @@ pub fn create_store(
                     std::net::IpAddr::V6(addr) => {
                         self.v6.match_prefix(
                             PrefixId::<IPv6>::new(
-                                addr.into(),
+                                <IPv6 as AddressFamily>::
+                                    from_ipaddr(addr),
                                 search_pfx.len(),
                             ),
                             options,
@@ -761,14 +858,16 @@ pub fn create_store(
                 match search_pfx.addr() {
                     std::net::IpAddr::V4(addr) => self.v4.best_path(
                         PrefixId::<IPv4>::new(
-                            addr.into(),
-                            search_pfx.len()
+                            <IPv4 as AddressFamily>::
+                                from_ipaddr(addr),
+                            search_pfx.len(),
                         ),
                         guard
                     ),
                     std::net::IpAddr::V6(addr) => self.v6.best_path(
                         PrefixId::<IPv6>::new(
-                            addr.into(),
+                            <IPv6 as AddressFamily>::
+                                from_ipaddr(addr),
                             search_pfx.len(),
                         ),
                         guard
@@ -800,7 +899,8 @@ pub fn create_store(
                     std::net::IpAddr::V4(addr) => self.v4
                         .calculate_and_store_best_and_backup_path(
                             PrefixId::<IPv4>::new(
-                                addr.into(),
+                                <IPv4 as AddressFamily>::
+                                    from_ipaddr(addr),
                                 search_pfx.len(),
                             ),
                             tbi,
@@ -809,7 +909,8 @@ pub fn create_store(
                     std::net::IpAddr::V6(addr) => self.v6
                         .calculate_and_store_best_and_backup_path(
                             PrefixId::<IPv6>::new(
-                                addr.into(),
+                                <IPv6 as AddressFamily>::
+                                    from_ipaddr(addr),
                                 search_pfx.len(),
                             ),
                             tbi,
@@ -836,7 +937,8 @@ pub fn create_store(
                     std::net::IpAddr::V4(addr) => self.v4
                         .is_ps_outdated(
                             PrefixId::<IPv4>::new(
-                                addr.into(),
+                                <IPv4 as AddressFamily>::
+                                    from_ipaddr(addr),
                                 search_pfx.len(),
                             ),
                             guard
@@ -844,7 +946,8 @@ pub fn create_store(
                     std::net::IpAddr::V6(addr) => self.v6
                         .is_ps_outdated(
                             PrefixId::<IPv6>::new(
-                                addr.into(),
+                                <IPv6 as AddressFamily>::
+                                    from_ipaddr(addr),
                                 search_pfx.len(),
                             ),
                             guard
@@ -876,7 +979,8 @@ pub fn create_store(
                 match search_pfx.addr() {
                     std::net::IpAddr::V4(addr) => self.v4.more_specifics_from(
                         PrefixId::<IPv4>::new(
-                            addr.into(),
+                            <IPv4 as AddressFamily>::
+                                from_ipaddr(addr),
                             search_pfx.len(),
                         ),
                         mui,
@@ -885,7 +989,8 @@ pub fn create_store(
                     ),
                     std::net::IpAddr::V6(addr) => self.v6.more_specifics_from(
                         PrefixId::<IPv6>::new(
-                            addr.into(),
+                            <IPv6 as AddressFamily>::
+                                from_ipaddr(addr),
                             search_pfx.len(),
                         ),
                         mui,
@@ -919,7 +1024,8 @@ pub fn create_store(
                 match search_pfx.addr() {
                     std::net::IpAddr::V4(addr) => self.v4.less_specifics_from(
                         PrefixId::<IPv4>::new(
-                            addr.into(),
+                            <IPv4 as AddressFamily>::
+                                from_ipaddr(addr),
                             search_pfx.len(),
                         ),
                         mui,
@@ -928,7 +1034,8 @@ pub fn create_store(
                     ),
                     std::net::IpAddr::V6(addr) => self.v6.less_specifics_from(
                         PrefixId::<IPv6>::new(
-                            addr.into(),
+                            <IPv6 as AddressFamily>::
+                                from_ipaddr(addr),
                             search_pfx.len(),
                         ),
                         mui,
@@ -997,7 +1104,8 @@ pub fn create_store(
                                     .v4
                                     .less_specifics_iter_from(
                                         PrefixId::<IPv4>::new(
-                                            addr.into(),
+                                            <IPv4 as AddressFamily>::
+                                                from_ipaddr(addr),
                                             search_pfx.len(),
                                         ),
                                         mui,
@@ -1016,7 +1124,8 @@ pub fn create_store(
                                     .v6
                                     .less_specifics_iter_from(
                                         PrefixId::<IPv6>::new(
-                                            addr.into(),
+                                            <IPv6 as AddressFamily>::
+                                                from_ipaddr(addr),
                                             search_pfx.len(),
                                         ),
                                         mui,
@@ -1093,7 +1202,8 @@ pub fn create_store(
                                     .v4
                                     .more_specifics_iter_from(
                                         PrefixId::<IPv4>::new(
-                                            addr.into(),
+                                            <IPv4 as AddressFamily>::
+                                                from_ipaddr(addr),
                                             search_pfx.len(),
                                         ),
                                         mui,
@@ -1112,7 +1222,8 @@ pub fn create_store(
                                     .v6
                                     .more_specifics_iter_from(
                                         PrefixId::<IPv6>::new(
-                                            addr.into(),
+                                            <IPv6 as AddressFamily>::
+                                                from_ipaddr(addr),
                                             search_pfx.len(),
                                         ),
                                         mui,
@@ -1146,7 +1257,7 @@ pub fn create_store(
                         self.v4
                             .more_specifics_iter_from(
                                 PrefixId::<IPv4>::new(
-                                    0,
+                                    <IPv4 as AddressFamily>::zero(),
                                     0,
                                 ),
                                 Some(mui),
@@ -1172,7 +1283,7 @@ pub fn create_store(
                         self.v6
                             .more_specifics_iter_from(
                                 PrefixId::<IPv6>::new(
-                                    0,
+                                    <IPv6 as AddressFamily>::zero(),
                                     0,
                                 ),
                                 Some(mui),
diff --git a/src/bin/cli.rs b/src/bin/cli.rs
index b37d5c4c..b78b3041 100644
--- a/src/bin/cli.rs
+++ b/src/bin/cli.rs
@@ -1,5 +1,6 @@
 #![cfg(feature = "cli")]
 use ansi_term::Colour;
+use rotonda_store::rib::MemoryOnlyConfig;
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 
@@ -66,7 +67,8 @@ fn load_prefixes(
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut pfxs: Vec<PrefixRecord<PrefixAs>> = vec![];
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    let tree_bitmap =
+        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
 
     if let Err(err) = load_prefixes(&mut pfxs) {
         println!("error running example: {}", err);
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index fcb7a20c..8c544317 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -1,3 +1,7 @@
+use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::rib::PersistHistoryConfig;
+use rotonda_store::rib::PersistOnlyConfig;
+use rotonda_store::rib::WriteAheadConfig;
 use routecore::bgp::aspath::HopPath;
 use routecore::bgp::message::update_builder::StandardCommunitiesList;
 use std::collections::BTreeSet;
@@ -14,8 +18,8 @@ use rayon::iter::ParallelIterator;
 use rayon::prelude::*;
 use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::{MultiThreadedStore, RouteStatus};
+use rotonda_store::rib::Config;
 use rotonda_store::rib::PersistStrategy;
-use rotonda_store::rib::StoreConfig;
 use rotonda_store::rib::UpsertReport;
 use rotonda_store::PublicRecord;
 use routecore::bgp::message::PduParseInfo;
@@ -170,8 +174,8 @@ struct Cli {
     persist_strategy: Option<String>,
 }
 
-fn insert<T: rotonda_store::Meta>(
-    store: &MultiThreadedStore<T>,
+fn insert<C: Config, T: rotonda_store::Meta>(
+    store: &MultiThreadedStore<T, C>,
     prefix: &Prefix,
     mui: u32,
     ltime: u64,
@@ -209,16 +213,16 @@ fn par_load_prefixes(
     if shuffle {
         let t_s = Instant::now();
         eprint!("shuffling prefixes... ");
-        prefixes.shuffle(&mut rand::thread_rng());
+        prefixes.shuffle(&mut rand::rng());
         eprintln!("done! took {}ms", t_s.elapsed().as_millis());
     }
 
     prefixes
 }
 
-fn mt_parse_and_insert_table(
+fn mt_parse_and_insert_table<C: Config + Sync>(
     tables: TableDumpIterator<&[u8]>,
-    store: Option<&MultiThreadedStore<PaBytes>>,
+    store: Option<&MultiThreadedStore<PaBytes, C>>,
     ltime: u64,
 ) -> (UpsertCounters, Vec<Prefix>) {
     let persist_strategy =
@@ -328,9 +332,9 @@ fn mt_parse_and_insert_table(
     counters
 }
 
-fn st_parse_and_insert_table(
+fn st_parse_and_insert_table<C: Config>(
     entries: RibEntryIterator<&[u8]>,
-    store: Option<&MultiThreadedStore<PaBytes>>,
+    store: Option<&MultiThreadedStore<PaBytes, C>>,
     ltime: u64,
 ) -> UpsertCounters {
     let mut counters = UpsertCounters::default();
@@ -359,9 +363,9 @@ fn st_parse_and_insert_table(
     counters
 }
 
-fn mt_prime_store(
+fn mt_prime_store<C: Config + Sync>(
     prefixes: &Vec<(Prefix, u16)>,
-    store: &MultiThreadedStore<PaBytes>,
+    store: &MultiThreadedStore<PaBytes, C>,
 ) -> UpsertCounters {
     let t0 = std::time::Instant::now();
 
@@ -392,9 +396,9 @@ fn mt_prime_store(
     counters
 }
 
-fn st_prime_store(
+fn st_prime_store<C: Config>(
     prefixes: &Vec<(Prefix, u16)>,
-    store: &MultiThreadedStore<PaBytes>,
+    store: &MultiThreadedStore<PaBytes, C>,
 ) -> UpsertCounters {
     let mut counters = UpsertCounters::default();
 
@@ -414,54 +418,26 @@ fn st_prime_store(
     counters
 }
 
-fn main() {
-    let mut store_config = StoreConfig {
-        persist_strategy: PersistStrategy::PersistHistory,
-        persist_path: "/tmp/rotonda/".into(),
-    };
-
-    let args = Cli::parse();
-
-    let t_total = Instant::now();
-
-    let mut global_counters = UpsertCounters::default();
-    let mut mib_total: usize = 0;
-    let mut inner_stores = vec![];
-    let mut persisted_prefixes = BTreeSet::new();
-
-    store_config.persist_strategy = match &args.persist_strategy {
-        Some(a) if a == &"memory_only".to_string() => {
-            PersistStrategy::MemoryOnly
-        }
-        Some(a) if a == &"write_ahead".to_string() => {
-            PersistStrategy::WriteAhead
-        }
-        Some(a) if a == &"persist_only".to_string() => {
-            PersistStrategy::PersistOnly
-        }
-        Some(a) if a == &"persist_history".to_string() => {
-            PersistStrategy::PersistHistory
-        }
-        None => PersistStrategy::PersistHistory,
-        Some(a) => {
-            eprintln!("Unknown persist strategy: {}", a);
-            return;
-        }
-    };
+type Stores<C> = Vec<MultiThreadedStore<PaBytes, C>>;
 
-    // Create all the stores necessary, and if at least one is created, create
-    // a reference to the first one.
-    let mut store = match &args {
+// Create all the stores necessary, and if at least one is created, create
+// a reference to the first one.
+fn create_stores<'a, C: Config + Sync>(
+    stores: &'a mut Stores<C>,
+    args: &'a Cli,
+    store_config: C,
+) -> Option<&'a MultiThreadedStore<PaBytes, C>> {
+    match &args {
         a if a.single_store && a.parse_only => {
             eprintln!(
                 "Can't combine --parse-only and --single-store. 
                 Make up your mind."
             );
-            return;
+            None
         }
         a if a.single_store => {
-            inner_stores.push(
-                MultiThreadedStore::<PaBytes>::new_with_config(
+            stores.push(
+                MultiThreadedStore::<PaBytes, C>::new_with_config(
                     store_config.clone(),
                 )
                 .unwrap(),
@@ -470,7 +446,9 @@ fn main() {
                 "created a single-store with strategy: {:?}\n",
                 store_config
             );
-            Some(&inner_stores[0])
+
+            exec_for_store(Some(&stores[0]), stores, args);
+            Some(&stores[0])
         }
         a if a.parse_only => {
             println!("No store created (parse only)");
@@ -478,15 +456,27 @@ fn main() {
         }
         _ => {
             for _ in &args.mrt_files {
-                inner_stores.push(
-                    MultiThreadedStore::<PaBytes>::try_default().unwrap(),
+                stores.push(
+                    MultiThreadedStore::<PaBytes, C>::try_default().unwrap(),
                 );
             }
-            println!("Number of created stores: {}", inner_stores.len());
+            println!("Number of created stores: {}", stores.len());
             println!("store config: {:?}", store_config);
-            Some(&inner_stores[0])
+            exec_for_store(Some(&stores[0]), stores, args);
+            Some(&stores[0])
         }
-    };
+    }
+}
+
+fn exec_for_store<'a, C: Config + Sync>(
+    mut store: Option<&'a MultiThreadedStore<PaBytes, C>>,
+    inner_stores: &'a Stores<C>,
+    args: &'a Cli,
+) {
+    let mut global_counters = UpsertCounters::default();
+    let mut mib_total: usize = 0;
+    let mut persisted_prefixes = BTreeSet::new();
+    let t_total = Instant::now();
 
     // Loop over all the mrt-files specified as arguments
     for (f_index, mrtfile) in args.mrt_files.iter().enumerate() {
@@ -641,3 +631,106 @@ fn main() {
         }
     }
 }
+
+fn main() {
+    let args = Cli::parse();
+
+    // let t_total = Instant::now();
+
+    // let mut global_counters = UpsertCounters::default();
+    // let mut mib_total: usize = 0;
+    // let mut persisted_prefixes = BTreeSet::new();
+
+    match &args.persist_strategy {
+        Some(a) if a == &"memory_only".to_string() => {
+            let mut store_config = MemoryOnlyConfig;
+            store_config.set_persist_path("/tmp/rotonda/".into());
+            let mut inner_stores: Stores<MemoryOnlyConfig> = vec![];
+            create_stores::<MemoryOnlyConfig>(
+                &mut inner_stores,
+                &args,
+                store_config,
+            );
+        }
+        Some(a) if a == &"write_ahead".to_string() => {
+            let mut store_config = WriteAheadConfig::default();
+            store_config.set_persist_path("/tmp/rotonda/".into());
+            let mut inner_stores: Stores<WriteAheadConfig> = vec![];
+            create_stores::<WriteAheadConfig>(
+                &mut inner_stores,
+                &args,
+                store_config,
+            );
+        }
+        Some(a) if a == &"persist_only".to_string() => {
+            let mut store_config = PersistOnlyConfig::default();
+            store_config.set_persist_path("/tmp/rotonda/".into());
+            let mut inner_stores: Stores<PersistOnlyConfig> = vec![];
+            create_stores::<PersistOnlyConfig>(
+                &mut inner_stores,
+                &args,
+                store_config,
+            );
+        }
+        Some(a) if a == &"persist_history".to_string() => {
+            let mut store_config = PersistHistoryConfig::default();
+            store_config.set_persist_path("/tmp/rotonda/".into());
+            let mut inner_stores: Stores<PersistHistoryConfig> = vec![];
+            create_stores::<PersistHistoryConfig>(
+                &mut inner_stores,
+                &args,
+                store_config,
+            );
+        }
+        None => {
+            let mut store_config = PersistHistoryConfig::default();
+            store_config.set_persist_path("/tmp/rotonda/".into());
+            let mut inner_stores: Stores<PersistHistoryConfig> = vec![];
+            create_stores::<PersistHistoryConfig>(
+                &mut inner_stores,
+                &args,
+                store_config,
+            );
+        }
+        Some(a) => {
+            eprintln!("Unknown persist strategy: {}", a);
+        }
+    }
+
+    // let mut store = match &args {
+    //     a if a.single_store && a.parse_only => {
+    //         eprintln!(
+    //             "Can't combine --parse-only and --single-store.
+    //             Make up your mind."
+    //         );
+    //         return;
+    //     }
+    //     a if a.single_store => {
+    //         inner_stores.push(
+    //             MultiThreadedStore::<PaBytes, C>::new_with_config(
+    //                 store_config.clone(),
+    //             )
+    //             .unwrap(),
+    //         );
+    //         println!(
+    //             "created a single-store with strategy: {:?}\n",
+    //             store_config
+    //         );
+    //         Some(&inner_stores[0])
+    //     }
+    //     a if a.parse_only => {
+    //         println!("No store created (parse only)");
+    //         None
+    //     }
+    //     _ => {
+    //         for _ in &args.mrt_files {
+    //             inner_stores.push(
+    //                 MultiThreadedStore::<PaBytes>::try_default().unwrap(),
+    //             );
+    //         }
+    //         println!("Number of created stores: {}", inner_stores.len());
+    //         println!("store config: {:?}", store_config);
+    //         Some(&inner_stores[0])
+    //     }
+    // };
+}
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 8aaa2c5e..8922e1dd 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -3,25 +3,230 @@
 use std::marker::PhantomData;
 use std::path::Path;
 
-use inetnum::addr::Prefix;
 use log::trace;
 use lsm_tree::{AbstractTree, KvPair};
 use roaring::RoaringBitmap;
+use zerocopy::{
+    FromBytes, Immutable, IntoBytes, KnownLayout, NativeEndian, TryFromBytes,
+    Unaligned, U32, U64,
+};
 
 use crate::local_array::types::{PrefixId, RouteStatus};
-use crate::rib::query::{FamilyQueryResult, FamilyRecord, TreeQueryResult};
+use crate::prefix_record::{ValueHeader, ZeroCopyRecord};
 use crate::rib::Counters;
-use crate::{
-    AddressFamily, IncludeHistory, MatchOptions, MatchType, Meta,
-    PublicRecord,
-};
+use crate::{AddressFamily, Meta, PublicRecord};
+
+type ZeroCopyError<'a, T> = zerocopy::ConvertError<
+    zerocopy::AlignmentError<&'a [u8], T>,
+    zerocopy::SizeError<&'a [u8], T>,
+    zerocopy::ValidityError<&'a [u8], T>,
+>;
+type ZeroCopyMutError<'a, T> = zerocopy::ConvertError<
+    zerocopy::AlignmentError<&'a mut [u8], T>,
+    zerocopy::SizeError<&'a mut [u8], T>,
+    zerocopy::ValidityError<&'a mut [u8], T>,
+>;
+
+pub trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
+    TryFromBytes + KnownLayout + IntoBytes + Unaligned + Immutable
+{
+    fn mut_from_bytes(
+        bytes: &mut [u8],
+    ) -> std::result::Result<&mut Self, ZeroCopyMutError<'_, Self>> {
+        Self::try_mut_from_bytes(bytes.as_mut_bytes())
+    }
+
+    fn from_bytes(
+        bytes: &[u8],
+    ) -> std::result::Result<&Self, ZeroCopyError<'_, Self>> {
+        Self::try_ref_from_bytes(bytes.as_bytes())
+    }
+
+    // fn as_key_size_bytes(&self) -> [u8; KEY_SIZE] {
+    //     *self.as_bytes().first_chunk::<KEY_SIZE>().unwrap()
+    // }
+
+    fn new_write_key(
+        prefix_id: PrefixId<AF>,
+        mui: u32,
+        ltime: u64,
+        status: RouteStatus,
+    ) -> [u8; KEY_SIZE];
+
+    fn header(bytes: &[u8]) -> &LongKey<AF> {
+        LongKey::try_ref_from_bytes(bytes.as_bytes()).unwrap()
+    }
+
+    fn header_mut(bytes: &mut [u8]) -> &mut LongKey<AF> {
+        trace!("key size {}", KEY_SIZE);
+        trace!("bytes len {}", bytes.len());
+        LongKey::try_mut_from_bytes(bytes.as_mut_bytes()).unwrap()
+    }
+
+    fn short_key(bytes: &[u8]) -> &ShortKey<AF> {
+        trace!("short key from bytes {:?}", bytes);
+        let s_b = &bytes[..(AF::BITS as usize / 8) + 6];
+        trace!("short key {:?}", s_b);
+        ShortKey::try_ref_from_prefix(bytes).unwrap().0
+    }
+    // fn persistence_key(
+    //     // PREFIX_SIZE bytes
+    //     prefix_id: PrefixId<AF>,
+    //     // 4 bytes
+    //     mui: u32,
+    //     // 8 bytes
+    //     ltime: u64,
+    //     // 1 byte
+    //     status: RouteStatus,
+    // ) -> [u8; KEY_SIZE] {
+    //     assert!(KEY_SIZE > PREFIX_SIZE);
+    //     let key = &mut [0_u8; KEY_SIZE];
+
+    //     // prefix 5 or 17 bytes
+    //     *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() =
+    //         prefix_id.to_len_first_bytes();
+
+    //     // mui 4 bytes
+    //     *key[PREFIX_SIZE..PREFIX_SIZE + 4]
+    //         .first_chunk_mut::<4>()
+    //         .unwrap() = mui.to_le_bytes();
+
+    //     // ltime 8 bytes
+    //     *key[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
+    //         .first_chunk_mut::<8>()
+    //         .unwrap() = ltime.to_le_bytes();
+
+    //     // status 1 byte
+    //     key[PREFIX_SIZE + 12] = status.into();
+
+    //     *key
+    // }
+
+    // fn prefix_mui_persistence_key(
+    //     prefix_id: PrefixId<AF>,
+    //     mui: u32,
+    // ) -> Vec<u8> {
+    //     let mut key = vec![0; PREFIX_SIZE + 4];
+    //     // prefix 5 or 17 bytes
+    //     *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() =
+    //         prefix_id.to_len_first_bytes();
+
+    //     // mui 4 bytes
+    //     *key[PREFIX_SIZE..PREFIX_SIZE + 4]
+    //         .first_chunk_mut::<4>()
+    //         .unwrap() = mui.to_le_bytes();
+
+    //     key
+    // }
+
+    // fn parse_key(bytes: &[u8]) -> (PrefixId<AF>, u32, u64, RouteStatus) {
+    //     (
+    //         // prefix 5 or 17 bytes
+    //         PrefixId::from(*bytes.first_chunk::<PREFIX_SIZE>().unwrap()),
+    //         // mui 4 bytes
+    //         u32::from_le_bytes(
+    //             *bytes[PREFIX_SIZE..PREFIX_SIZE + 4]
+    //                 .first_chunk::<4>()
+    //                 .unwrap(),
+    //         ),
+    //         // ltime 8 bytes
+    //         u64::from_le_bytes(
+    //             *bytes[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
+    //                 .first_chunk::<8>()
+    //                 .unwrap(),
+    //         ),
+    //         // status 1 byte
+    //         RouteStatus::try_from(bytes[PREFIX_SIZE + 12]).unwrap(),
+    //     )
+    // }
+}
+
+#[derive(Debug, KnownLayout, Immutable, FromBytes, Unaligned, IntoBytes)]
+#[repr(C)]
+pub struct ShortKey<AF: AddressFamily> {
+    prefix: PrefixId<AF>,
+    mui: U32<NativeEndian>,
+}
+
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    KnownLayout,
+    Immutable,
+    TryFromBytes,
+    Unaligned,
+    IntoBytes,
+)]
+#[repr(C)]
+pub struct LongKey<AF: AddressFamily> {
+    prefix: PrefixId<AF>,
+    mui: U32<NativeEndian>,
+    ltime: U64<NativeEndian>,
+    status: RouteStatus,
+}
+
+impl<AF: AddressFamily, const KEY_SIZE: usize> KeySize<AF, KEY_SIZE>
+    for ShortKey<AF>
+{
+    fn new_write_key(
+        prefix: PrefixId<AF>,
+        mui: u32,
+        _ltime: u64,
+        _status: RouteStatus,
+    ) -> [u8; KEY_SIZE] {
+        *Self::from((prefix, mui))
+            .as_bytes()
+            .first_chunk::<KEY_SIZE>()
+            .unwrap()
+    }
+}
+
+impl<AF: AddressFamily> From<(PrefixId<AF>, u32)> for ShortKey<AF> {
+    fn from(value: (PrefixId<AF>, u32)) -> Self {
+        Self {
+            prefix: value.0,
+            mui: value.1.into(),
+        }
+    }
+}
+
+impl<AF: AddressFamily, const KEY_SIZE: usize> KeySize<AF, KEY_SIZE>
+    for LongKey<AF>
+{
+    fn new_write_key(
+        prefix: PrefixId<AF>,
+        mui: u32,
+        ltime: u64,
+        status: RouteStatus,
+    ) -> [u8; KEY_SIZE] {
+        *Self::from((prefix, mui, ltime, status))
+            .as_bytes()
+            .first_chunk::<KEY_SIZE>()
+            .unwrap()
+    }
+}
+
+impl<AF: AddressFamily> From<(PrefixId<AF>, u32, u64, RouteStatus)>
+    for LongKey<AF>
+{
+    fn from(value: (PrefixId<AF>, u32, u64, RouteStatus)) -> Self {
+        Self {
+            prefix: value.0,
+            mui: value.1.into(),
+            ltime: value.2.into(),
+            status: value.3,
+        }
+    }
+}
 
 pub struct PersistTree<
     AF: AddressFamily,
+    K: KeySize<AF, KEY_SIZE>,
     // The size in bytes of the prefix in the persisted storage (disk), this
     // amounnts to the bytes for the addres (4 for IPv4, 16 for IPv6) and 1
     // bytefor the prefix length.
-    const PREFIX_SIZE: usize,
+    // const PREFIX_SIZE: usize,
     // The size in bytes of the complete key in the persisted storage, this
     // is PREFIX_SIZE bytes (4; 16) + mui size (4) + ltime (8)
     const KEY_SIZE: usize,
@@ -29,148 +234,191 @@ pub struct PersistTree<
     tree: lsm_tree::Tree,
     counters: Counters,
     _af: PhantomData<AF>,
+    _k: PhantomData<K>,
 }
 
-impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
-    PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
+impl<
+        AF: AddressFamily,
+        K: KeySize<AF, KEY_SIZE>,
+        // const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > PersistTree<AF, K, KEY_SIZE>
 {
-    pub fn new(
-        persist_path: &Path,
-    ) -> PersistTree<AF, PREFIX_SIZE, KEY_SIZE> {
-        PersistTree::<AF, PREFIX_SIZE, KEY_SIZE> {
+    pub fn new(persist_path: &Path) -> PersistTree<AF, K, KEY_SIZE> {
+        PersistTree::<AF, K, KEY_SIZE> {
             tree: lsm_tree::Config::new(persist_path).open().unwrap(),
             counters: Counters::default(),
             _af: PhantomData,
+            _k: PhantomData,
         }
     }
 
-    pub(crate) fn insert(
-        &self,
-        key: [u8; KEY_SIZE],
-        value: &[u8],
-    ) -> (u32, u32) {
-        self.tree.insert::<[u8; KEY_SIZE], &[u8]>(key, value, 0)
+    fn insert(&self, key: &[u8], value: &[u8]) -> (u32, u32) {
+        self.tree.insert::<&[u8], &[u8]>(key, value, 0)
     }
 
-    pub fn remove(&self, key: [u8; KEY_SIZE]) {
+    pub fn remove(&self, key: &[u8]) {
         self.tree.remove_weak(key, 0);
-        // the last byte of the prefix holds the length of the prefix.
-        self.counters.dec_prefixes_count(key[PREFIX_SIZE]);
+        // the first byte of the prefix holds the length of the prefix.
+        self.counters.dec_prefixes_count(key[0]);
     }
 
-    pub fn get_records_for_prefix<M: Meta>(
+    pub fn get_records_for_prefix(
         &self,
         prefix: PrefixId<AF>,
         mui: Option<u32>,
         include_withdrawn: bool,
         withdrawn_muis_bmin: &RoaringBitmap,
-    ) -> Vec<PublicRecord<M>> {
+    ) -> lsm_tree::Result<Vec<Vec<u8>>> {
         match (mui, include_withdrawn) {
             // Specific mui, include withdrawn routes
             (Some(mui), true) => {
                 // get the records from the persist store for the (prefix,
                 // mui) tuple only.
-                let prefix_b = Self::prefix_mui_persistence_key(prefix, mui);
-                (*self.tree.prefix(prefix_b, None, None))
-                    .into_iter()
+                let prefix_b = ShortKey::from((prefix, mui));
+                self.tree
+                    .prefix(prefix_b.as_bytes(), None, None)
                     .map(|kv| {
-                        let kv = kv.unwrap();
-                        let (_, r_mui, ltime, mut status) =
-                            Self::parse_key(kv.0.as_ref());
-
-                        // If mui is in the global withdrawn muis table, then
-                        // rewrite the routestatus of the record to withdrawn.
-                        if withdrawn_muis_bmin.contains(r_mui) {
-                            status = RouteStatus::Withdrawn;
-                        }
-                        PublicRecord::new(
-                            mui,
-                            ltime,
-                            status,
-                            kv.1.as_ref().to_vec().into(),
-                        )
+                        kv.map(|kv| {
+                            trace!("mui i persist kv pair found: {:?}", kv);
+                            let mut bytes = [kv.0, kv.1].concat();
+                            // let key: &mut LongKey<AF> =
+                            //     LongKey::try_mut_from_bytes(
+                            //         bytes.as_mut_bytes(),
+                            //     )
+                            //     .unwrap();
+                            let key = K::header_mut(&mut bytes[..KEY_SIZE]);
+                            // If mui is in the global withdrawn muis table,
+                            // then rewrite the routestatus of the record
+                            // to withdrawn.
+                            if withdrawn_muis_bmin.contains(key.mui.into()) {
+                                key.status = RouteStatus::Withdrawn;
+                            }
+                            bytes
+                        })
                     })
                     .collect::<Vec<_>>()
+                    .into_iter()
+                    .collect()
             }
             // Al muis, include withdrawn routes
             (None, true) => {
                 // get all records for this prefix
-                let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
-                (*self.tree.prefix(prefix_b, None, None))
-                    .into_iter()
+                // let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
+                self.tree
+                    .prefix(prefix.as_bytes(), None, None)
+                    // .into_iter()
                     .map(|kv| {
-                        let kv = kv.unwrap();
-                        let (_, r_mui, ltime, mut status) =
-                            Self::parse_key(kv.0.as_ref());
-
-                        // If mui is in the global withdrawn muis table, then
-                        // rewrite the routestatus of the record to withdrawn.
-                        if withdrawn_muis_bmin.contains(r_mui) {
-                            status = RouteStatus::Withdrawn;
-                        }
-                        PublicRecord::new(
-                            r_mui,
-                            ltime,
-                            status,
-                            kv.1.as_ref().to_vec().into(),
-                        )
+                        kv.map(|kv| {
+                            trace!("n i persist kv pair found: {:?}", kv);
+                            // let kv = kv.unwrap();
+                            // let (_, r_mui, ltime, mut status) =
+                            //     Self::parse_key(kv.0.as_ref());
+
+                            // If mui is in the global withdrawn muis table, then
+                            // rewrite the routestatus of the record to withdrawn.
+                            let mut bytes = [kv.0, kv.1].concat();
+                            trace!("bytes {:?}", bytes);
+                            // let key: &mut LongKey<AF> =
+                            //     LongKey::try_mut_from_bytes(
+                            //         bytes.as_mut_bytes(),
+                            //     )
+                            //     .unwrap();
+                            let key = K::header_mut(&mut bytes[..KEY_SIZE]);
+                            trace!("key {:?}", key);
+                            trace!("wm_bmin {:?}", withdrawn_muis_bmin);
+                            if withdrawn_muis_bmin.contains(key.mui.into()) {
+                                trace!("rewrite status");
+                                key.status = RouteStatus::Withdrawn;
+                            }
+                            // PublicRecord::new(
+                            //     r_mui,
+                            //     ltime,
+                            //     status,
+                            //     kv.1.as_ref().to_vec().into(),
+                            // )
+                            bytes
+                        })
                     })
                     .collect::<Vec<_>>()
+                    .into_iter()
+                    .collect()
             }
             // All muis, exclude withdrawn routes
             (None, false) => {
                 // get all records for this prefix
-                let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
-                (*self.tree.prefix(prefix_b, None, None))
-                    .into_iter()
+                // let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
+                self.tree
+                    .prefix(prefix.as_bytes(), None, None)
                     .filter_map(|kv| {
-                        let kv = kv.unwrap();
-                        let (_, r_mui, ltime, status) =
-                            Self::parse_key(kv.0.as_ref());
-
-                        // If mui is in the global withdrawn muis table, then
-                        // skip this record
-                        if status == RouteStatus::Withdrawn
-                            || withdrawn_muis_bmin.contains(r_mui)
-                        {
-                            return None;
-                        }
-                        Some(PublicRecord::new(
-                            r_mui,
-                            ltime,
-                            status,
-                            kv.1.as_ref().to_vec().into(),
-                        ))
+                        kv.map(|kv| {
+                            trace!("n f persist kv pair found: {:?}", kv);
+                            let mut bytes = [kv.0, kv.1].concat();
+                            // let key: &mut LongKey<AF> =
+                            //     LongKey::try_mut_from_bytes(
+                            //         bytes.as_mut_bytes(),
+                            //     )
+                            //     .unwrap();
+                            let key = K::header_mut(&mut bytes[..KEY_SIZE]);
+                            // If mui is in the global withdrawn muis table,
+                            // then skip this record
+                            if key.status == RouteStatus::Withdrawn
+                                || withdrawn_muis_bmin
+                                    .contains(key.mui.into())
+                            {
+                                return None;
+                            }
+                            Some(bytes)
+                        })
+                        .transpose()
                     })
                     .collect::<Vec<_>>()
+                    .into_iter()
+                    .collect()
             }
             // Specific mui, exclude withdrawn routes
             (Some(mui), false) => {
                 // get the records from the persist store for the (prefix,
                 // mui) tuple only.
-                let prefix_b = Self::prefix_mui_persistence_key(prefix, mui);
-                (*self.tree.prefix(prefix_b, None, None))
-                    .into_iter()
+                let prefix_b = ShortKey::<AF>::from((prefix, mui));
+                self.tree
+                    .prefix(prefix_b.as_bytes(), None, None)
+                    // .into_iter()
                     .filter_map(|kv| {
-                        let kv = kv.unwrap();
-                        let (_, r_mui, ltime, status) =
-                            Self::parse_key(kv.0.as_ref());
-
-                        // If mui is in the global withdrawn muis table, then
-                        // skip this record
-                        if status == RouteStatus::Withdrawn
-                            || withdrawn_muis_bmin.contains(r_mui)
-                        {
-                            return None;
-                        }
-                        Some(PublicRecord::new(
-                            mui,
-                            ltime,
-                            status,
-                            kv.1.as_ref().to_vec().into(),
-                        ))
+                        // let kv = kv.unwrap();
+                        kv.map(|kv| {
+                            trace!("mui f persist kv pair found: {:?}", kv);
+                            let bytes = [kv.0, kv.1].concat();
+                            // let (_, r_mui, ltime, status) =
+                            //     Self::parse_key(kv.0.as_ref());
+
+                            // let key: &mut LongKey<AF> =
+                            //     LongKey::try_mut_from_bytes(
+                            //         bytes.as_mut_bytes(),
+                            //     )
+                            //     .unwrap();
+                            let key = K::header(&bytes[..KEY_SIZE]);
+                            // If mui is in the global withdrawn muis table, then
+                            // skip this record
+                            if key.status == RouteStatus::Withdrawn
+                                || withdrawn_muis_bmin
+                                    .contains(key.mui.into())
+                            {
+                                return None;
+                            }
+                            // Some(PublicRecord::new(
+                            //     mui,
+                            //     ltime,
+                            //     status,
+                            //     kv.1.as_ref().to_vec().into(),
+                            // ))
+                            Some(bytes)
+                        })
+                        .transpose()
                     })
                     .collect::<Vec<_>>()
+                    .into_iter()
+                    .collect()
             }
         }
 
@@ -226,72 +474,69 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         // }
     }
 
-    pub fn get_most_recent_record_for_prefix_mui<M: Meta>(
+    pub fn get_most_recent_record_for_prefix_mui(
         &self,
         prefix: PrefixId<AF>,
         mui: u32,
-    ) -> Option<PublicRecord<M>> {
-        let key_b = Self::prefix_mui_persistence_key(prefix, mui);
+    ) -> Option<Vec<u8>> {
+        trace!("get most recent record for prefix mui combo");
+        let key_b = ShortKey::from((prefix, mui));
 
-        (*self.tree.prefix(key_b, None, None))
+        (*self.tree.prefix(key_b.as_bytes(), None, None))
             .into_iter()
-            .map(|kv| {
+            .map(move |kv| {
                 let kv = kv.unwrap();
-                (Self::parse_key(kv.0.as_ref()), kv.1)
-            })
-            .max_by(|(a, _), (b, _)| a.2.cmp(&b.2))
-            .map(|((_pfx, mui, ltime, status), m)| {
-                PublicRecord::new(
-                    mui,
-                    ltime,
-                    status,
-                    m.as_ref().to_vec().into(),
-                )
+                [kv.0, kv.1].concat()
             })
+            .max_by(|b0, b1| K::header(b0).ltime.cmp(&K::header(b1).ltime))
     }
 
-    pub(crate) fn get_records_with_keys_for_prefix_mui<M: Meta>(
+    pub(crate) fn get_records_with_keys_for_prefix_mui(
         &self,
         prefix: PrefixId<AF>,
         mui: u32,
-    ) -> Vec<(Vec<u8>, PublicRecord<M>)> {
-        let key_b = Self::prefix_mui_persistence_key(prefix, mui);
+    ) -> Vec<Vec<u8>> {
+        // let key_b: [u8; KEY_SIZE] =
+        //     ShortKey::from((prefix, mui)).as_key_size_bytes();
+
+        let key_b = ShortKey::from((prefix, mui));
 
-        (*self.tree.prefix(key_b, None, None))
+        (*self.tree.prefix(key_b.as_bytes(), None, None))
             .into_iter()
             .map(|kv| {
                 let kv = kv.unwrap();
-                let (_, mui, ltime, status) = Self::parse_key(kv.0.as_ref());
-                (
-                    kv.0.to_vec(),
-                    PublicRecord::new(
-                        mui,
-                        ltime,
-                        status,
-                        kv.1.as_ref().to_vec().into(),
-                    ),
-                )
+                [kv.0, kv.1].concat()
+                // let (_, mui, ltime, status) = Self::parse_key(kv.0.as_ref());
+                // (
+                //     kv.0.to_vec(),
+                //     PublicRecord::new(
+                //         mui,
+                //         ltime,
+                //         status,
+                //         kv.1.as_ref().to_vec().into(),
+                //     ),
+                // )
             })
             .collect::<Vec<_>>()
     }
 
-    pub fn get_records_for_more_specific_prefix_in_len(
-        &self,
-        prefix: PrefixId<AF>,
-        len: u8,
-    ) -> Box<
-        dyn DoubleEndedIterator<
-            Item = Result<
-                (lsm_tree::Slice, lsm_tree::Slice),
-                lsm_tree::Error,
-            >,
-        >,
-    > {
-        let start = PrefixId::new(prefix.get_net(), len);
-        let end: [u8; PREFIX_SIZE] = start.inc_len().to_len_first_bytes();
-
-        self.tree.range(start.to_len_first_bytes()..end, None, None)
-    }
+    // fn get_records_for_more_specific_prefix_in_len(
+    //     &self,
+    //     prefix: PrefixId<AF>,
+    //     len: u8,
+    // ) -> Box<
+    //     dyn DoubleEndedIterator<
+    //         Item = Result<
+    //             (lsm_tree::Slice, lsm_tree::Slice),
+    //             lsm_tree::Error,
+    //         >,
+    //     >,
+    // > {
+    //     let start = PrefixId::new(prefix.get_net(), len);
+    //     let end: [u8; PREFIX_SIZE] = start.inc_len().to_len_first_bytes();
+
+    //     self.tree.range(start.to_len_first_bytes()..end, None, None)
+    // }
 
     // fn enrich_prefix<M: Meta>(
     //     &self,
@@ -317,175 +562,175 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
     // .collect()
     // }
 
-    fn enrich_prefixes<M: Meta>(
-        &self,
-        prefixes: Option<Vec<PrefixId<AF>>>,
-        mui: Option<u32>,
-        include_withdrawn: bool,
-        bmin: &RoaringBitmap,
-    ) -> Option<FamilyRecord<AF, M>> {
-        prefixes.map(|recs| {
-            recs.iter()
-                .flat_map(move |pfx| {
-                    Some((
-                        *pfx,
-                        self.get_records_for_prefix(
-                            *pfx,
-                            mui,
-                            include_withdrawn,
-                            bmin,
-                        )
-                        .into_iter()
-                        .filter_map(|mut r| {
-                            if bmin.contains(r.multi_uniq_id) {
-                                if !include_withdrawn {
-                                    return None;
-                                }
-                                r.status = RouteStatus::Withdrawn;
-                            }
-                            Some(r)
-                        })
-                        .collect(),
-                    ))
-                })
-                .collect()
-        })
-    }
-
-    fn sparse_record_set<M: Meta>(
-        &self,
-        prefixes: Option<Vec<PrefixId<AF>>>,
-    ) -> Option<FamilyRecord<AF, M>> {
-        prefixes.map(|recs| {
-            recs.iter().flat_map(|pfx| Some((*pfx, vec![]))).collect()
-        })
-    }
-
-    pub(crate) fn match_prefix<M: Meta>(
-        &self,
-        search_pfxs: TreeQueryResult<AF>,
-        options: &MatchOptions,
-        bmin: &RoaringBitmap,
-    ) -> FamilyQueryResult<AF, M> {
-        let (prefix, prefix_meta) = if let Some(prefix) = search_pfxs.prefix {
-            (
-                prefix,
-                self.get_records_for_prefix(
-                    prefix,
-                    options.mui,
-                    options.include_withdrawn,
-                    bmin,
-                ),
-            )
-        } else {
-            return FamilyQueryResult {
-                match_type: MatchType::EmptyMatch,
-                prefix: None,
-                prefix_meta: vec![],
-                less_specifics: if options.include_less_specifics {
-                    search_pfxs.less_specifics.map(|v| {
-                        v.into_iter().map(|p| (p, vec![])).collect::<Vec<_>>()
-                    })
-                } else {
-                    None
-                },
-                more_specifics: if options.include_more_specifics {
-                    search_pfxs.more_specifics.map(|v| {
-                        v.into_iter().map(|p| (p, vec![])).collect::<Vec<_>>()
-                    })
-                } else {
-                    None
-                },
-            };
-        };
+    // fn enrich_prefixes<M: Meta>(
+    //     &self,
+    //     prefixes: Option<Vec<PrefixId<AF>>>,
+    //     mui: Option<u32>,
+    //     include_withdrawn: bool,
+    //     bmin: &RoaringBitmap,
+    // ) -> Option<FamilyRecord<AF, M>> {
+    //     prefixes.map(|recs| {
+    //         recs.iter()
+    //             .flat_map(move |pfx| {
+    //                 Some((
+    //                     *pfx,
+    //                     self.get_records_for_prefix(
+    //                         *pfx,
+    //                         mui,
+    //                         include_withdrawn,
+    //                         bmin,
+    //                     )
+    //                     .into_iter()
+    //                     .filter_map(|mut r| {
+    //                         if bmin.contains(r.multi_uniq_id) {
+    //                             if !include_withdrawn {
+    //                                 return None;
+    //                             }
+    //                             r.status = RouteStatus::Withdrawn;
+    //                         }
+    //                         Some(r)
+    //                     })
+    //                     .collect(),
+    //                 ))
+    //             })
+    //             .collect()
+    //     })
+    // }
 
-        let mut res = match options.include_history {
-            // All the records for all the prefixes
-            IncludeHistory::All => FamilyQueryResult {
-                prefix: Some(prefix),
-                prefix_meta,
-                match_type: search_pfxs.match_type,
-                less_specifics: self.enrich_prefixes(
-                    search_pfxs.less_specifics,
-                    options.mui,
-                    options.include_withdrawn,
-                    bmin,
-                ),
-                more_specifics: search_pfxs.more_specifics.map(|ms| {
-                    self.more_specific_prefix_iter_from(
-                        prefix,
-                        ms.iter().map(|p| p.get_len()).collect::<Vec<_>>(),
-                        options.mui,
-                        bmin,
-                        options.include_withdrawn,
-                    )
-                    .collect::<Vec<_>>()
-                }),
-            },
-            // Only the search prefix itself has historical records attached
-            // to it, other prefixes (less|more specifics), have no records
-            // attached. Not useful with the MemoryOnly strategy (historical
-            // records are neve kept in memory).
-            IncludeHistory::SearchPrefix => FamilyQueryResult {
-                prefix: Some(prefix),
-                prefix_meta,
-                match_type: search_pfxs.match_type,
-                less_specifics: self
-                    .sparse_record_set(search_pfxs.less_specifics),
-                more_specifics: self
-                    .sparse_record_set(search_pfxs.more_specifics),
-            },
-            // Only the most recent record of the search prefix is returned
-            // with the prefixes. This is used for the PersistOnly strategy.
-            IncludeHistory::None => {
-                println!("Include history: None");
-                FamilyQueryResult {
-                    prefix: Some(prefix),
-                    prefix_meta,
-                    match_type: search_pfxs.match_type,
-                    less_specifics: search_pfxs.less_specifics.map(|ls| {
-                        self.less_specific_prefix_iter_from(
-                            ls,
-                            options.mui,
-                            bmin,
-                            options.include_withdrawn,
-                        )
-                        .collect::<Vec<_>>()
-                    }),
-                    more_specifics: search_pfxs.more_specifics.map(|ms| {
-                        self.more_specific_prefix_iter_from(
-                            prefix,
-                            ms.iter()
-                                .map(|p| p.get_len())
-                                .collect::<Vec<_>>(),
-                            options.mui,
-                            bmin,
-                            options.include_withdrawn,
-                        )
-                        .collect::<Vec<_>>()
-                    }),
-                }
-            }
-        };
+    // fn sparse_record_set<M: Meta>(
+    //     &self,
+    //     prefixes: Option<Vec<PrefixId<AF>>>,
+    // ) -> Option<FamilyRecord<AF, M>> {
+    //     prefixes.map(|recs| {
+    //         recs.iter().flat_map(|pfx| Some((*pfx, vec![]))).collect()
+    //     })
+    // }
 
-        res.match_type = match (options.match_type, &res) {
-            (_, res) if !res.prefix_meta.is_empty() => MatchType::ExactMatch,
-            (MatchType::LongestMatch | MatchType::EmptyMatch, _) => {
-                if res
-                    .less_specifics
-                    .as_ref()
-                    .is_some_and(|lp| !lp.is_empty())
-                {
-                    MatchType::LongestMatch
-                } else {
-                    MatchType::EmptyMatch
-                }
-            }
-            (MatchType::ExactMatch, _) => MatchType::EmptyMatch,
-        };
+    // pub(crate) fn match_prefix<M: Meta>(
+    //     &self,
+    //     search_pfxs: TreeQueryResult<AF>,
+    //     options: &MatchOptions,
+    //     bmin: &RoaringBitmap,
+    // ) -> FamilyQueryResult<AF, M> {
+    //     let (prefix, prefix_meta) = if let Some(prefix) = search_pfxs.prefix {
+    //         (
+    //             prefix,
+    //             self.get_records_for_prefix(
+    //                 prefix,
+    //                 options.mui,
+    //                 options.include_withdrawn,
+    //                 bmin,
+    //             ),
+    //         )
+    //     } else {
+    //         return FamilyQueryResult {
+    //             match_type: MatchType::EmptyMatch,
+    //             prefix: None,
+    //             prefix_meta: vec![],
+    //             less_specifics: if options.include_less_specifics {
+    //                 search_pfxs.less_specifics.map(|v| {
+    //                     v.into_iter().map(|p| (p, vec![])).collect::<Vec<_>>()
+    //                 })
+    //             } else {
+    //                 None
+    //             },
+    //             more_specifics: if options.include_more_specifics {
+    //                 search_pfxs.more_specifics.map(|v| {
+    //                     v.into_iter().map(|p| (p, vec![])).collect::<Vec<_>>()
+    //                 })
+    //             } else {
+    //                 None
+    //             },
+    //         };
+    //     };
+
+    //     let mut res = match options.include_history {
+    //         // All the records for all the prefixes
+    //         IncludeHistory::All => FamilyQueryResult {
+    //             prefix: Some(prefix),
+    //             prefix_meta,
+    //             match_type: search_pfxs.match_type,
+    //             less_specifics: self.enrich_prefixes(
+    //                 search_pfxs.less_specifics,
+    //                 options.mui,
+    //                 options.include_withdrawn,
+    //                 bmin,
+    //             ),
+    //             more_specifics: search_pfxs.more_specifics.map(|ms| {
+    //                 self.more_specific_prefix_iter_from(
+    //                     prefix,
+    //                     ms.iter().map(|p| p.get_len()).collect::<Vec<_>>(),
+    //                     options.mui,
+    //                     bmin,
+    //                     options.include_withdrawn,
+    //                 )
+    //                 .collect::<Vec<_>>()
+    //             }),
+    //         },
+    //         // Only the search prefix itself has historical records attached
+    //         // to it, other prefixes (less|more specifics), have no records
+    //         // attached. Not useful with the MemoryOnly strategy (historical
+    //         // records are neve kept in memory).
+    //         IncludeHistory::SearchPrefix => FamilyQueryResult {
+    //             prefix: Some(prefix),
+    //             prefix_meta,
+    //             match_type: search_pfxs.match_type,
+    //             less_specifics: self
+    //                 .sparse_record_set(search_pfxs.less_specifics),
+    //             more_specifics: self
+    //                 .sparse_record_set(search_pfxs.more_specifics),
+    //         },
+    //         // Only the most recent record of the search prefix is returned
+    //         // with the prefixes. This is used for the PersistOnly strategy.
+    //         IncludeHistory::None => {
+    //             println!("Include history: None");
+    //             FamilyQueryResult {
+    //                 prefix: Some(prefix),
+    //                 prefix_meta,
+    //                 match_type: search_pfxs.match_type,
+    //                 less_specifics: search_pfxs.less_specifics.map(|ls| {
+    //                     self.less_specific_prefix_iter_from(
+    //                         ls,
+    //                         options.mui,
+    //                         bmin,
+    //                         options.include_withdrawn,
+    //                     )
+    //                     .collect::<Vec<_>>()
+    //                 }),
+    //                 more_specifics: search_pfxs.more_specifics.map(|ms| {
+    //                     self.more_specific_prefix_iter_from(
+    //                         prefix,
+    //                         ms.iter()
+    //                             .map(|p| p.get_len())
+    //                             .collect::<Vec<_>>(),
+    //                         options.mui,
+    //                         bmin,
+    //                         options.include_withdrawn,
+    //                     )
+    //                     .collect::<Vec<_>>()
+    //                 }),
+    //             }
+    //         }
+    //     };
+
+    //     res.match_type = match (options.match_type, &res) {
+    //         (_, res) if !res.prefix_meta.is_empty() => MatchType::ExactMatch,
+    //         (MatchType::LongestMatch | MatchType::EmptyMatch, _) => {
+    //             if res
+    //                 .less_specifics
+    //                 .as_ref()
+    //                 .is_some_and(|lp| !lp.is_empty())
+    //             {
+    //                 MatchType::LongestMatch
+    //             } else {
+    //                 MatchType::EmptyMatch
+    //             }
+    //         }
+    //         (MatchType::ExactMatch, _) => MatchType::EmptyMatch,
+    //     };
 
-        res
-    }
+    //     res
+    // }
 
     pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
         let segment = self.tree.flush_active_memtable(0);
@@ -517,161 +762,147 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
         self.counters.get_prefixes_count()[len as usize]
     }
 
-    #[cfg(feature = "persist")]
-    pub fn persistence_key(
-        // PREFIX_SIZE bytes
-        prefix_id: PrefixId<AF>,
-        // 4 bytes
-        mui: u32,
-        // 8 bytes
-        ltime: u64,
-        // 1 byte
-        status: RouteStatus,
-    ) -> [u8; KEY_SIZE] {
-        assert!(KEY_SIZE > PREFIX_SIZE);
-        let key = &mut [0_u8; KEY_SIZE];
-
-        // prefix 5 or 17 bytes
-        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() =
-            prefix_id.to_len_first_bytes();
+    pub(crate) fn persist_record_w_long_key<M: Meta>(
+        &self,
+        prefix: PrefixId<AF>,
+        record: &PublicRecord<M>,
+    ) {
+        self.insert(
+            LongKey::from((
+                prefix,
+                record.multi_uniq_id,
+                record.ltime,
+                record.status,
+            ))
+            .as_bytes(),
+            record.meta.as_ref(),
+        );
+    }
 
-        // mui 4 bytes
-        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
-            .first_chunk_mut::<4>()
-            .unwrap() = mui.to_le_bytes();
+    pub(crate) fn persist_record_w_short_key<M: Meta>(
+        &self,
+        prefix: PrefixId<AF>,
+        record: &PublicRecord<M>,
+    ) {
+        trace!("Record to persist {}", record);
+        let mut value = ValueHeader {
+            ltime: record.ltime,
+            status: record.status,
+        }
+        .as_bytes()
+        .to_vec();
 
-        // ltime 8 bytes
-        *key[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
-            .first_chunk_mut::<8>()
-            .unwrap() = ltime.to_le_bytes();
+        trace!("header in bytes {:?}", value);
 
-        // status 1 byte
-        key[PREFIX_SIZE + 12] = status.into();
+        value.extend_from_slice(record.meta.as_ref());
 
-        *key
-    }
+        trace!("value complete {:?}", value);
 
-    #[cfg(feature = "persist")]
-    pub fn prefix_mui_persistence_key(
-        prefix_id: PrefixId<AF>,
-        mui: u32,
-    ) -> Vec<u8> {
-        let mut key = vec![0; PREFIX_SIZE + 4];
-        // prefix 5 or 17 bytes
-        *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() =
-            prefix_id.to_len_first_bytes();
-
-        // mui 4 bytes
-        *key[PREFIX_SIZE..PREFIX_SIZE + 4]
-            .first_chunk_mut::<4>()
-            .unwrap() = mui.to_le_bytes();
-
-        key
+        self.insert(
+            ShortKey::from((prefix, record.multi_uniq_id)).as_bytes(),
+            &value,
+        );
     }
 
-    #[cfg(feature = "persist")]
-    pub fn parse_key(bytes: &[u8]) -> (PrefixId<AF>, u32, u64, RouteStatus) {
-        (
-            // prefix 5 or 17 bytes
-            PrefixId::from(*bytes.first_chunk::<PREFIX_SIZE>().unwrap()),
-            // mui 4 bytes
-            u32::from_le_bytes(
-                *bytes[PREFIX_SIZE..PREFIX_SIZE + 4]
-                    .first_chunk::<4>()
-                    .unwrap(),
-            ),
-            // ltime 8 bytes
-            u64::from_le_bytes(
-                *bytes[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
-                    .first_chunk::<8>()
-                    .unwrap(),
-            ),
-            // status 1 byte
-            RouteStatus::try_from(bytes[PREFIX_SIZE + 12]).unwrap(),
-        )
+    pub(crate) fn rewrite_header_for_record(
+        &self,
+        header: ValueHeader,
+        record_b: &[u8],
+    ) {
+        let record = ZeroCopyRecord::<AF>::try_ref_from_prefix(&record_b)
+            .unwrap()
+            .0;
+        let key = ShortKey::from((record.prefix, record.multi_uniq_id));
+        trace!("insert key {:?}", key);
+
+        header
+            .as_bytes()
+            .to_vec()
+            .extend_from_slice(record.meta.as_ref());
+
+        self.insert(key.as_bytes(), header.as_bytes());
     }
 
-    #[cfg(feature = "persist")]
-    pub(crate) fn persist_record<M: Meta>(
+    pub(crate) fn insert_empty_record(
         &self,
         prefix: PrefixId<AF>,
-        // mui: u32,
-        record: &PublicRecord<M>,
+        mui: u32,
+        ltime: u64,
     ) {
         self.insert(
-            PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::persistence_key(
-                prefix,
-                record.multi_uniq_id,
-                record.ltime,
-                record.status,
-            ),
-            record.meta.as_ref(),
+            LongKey::from((prefix, mui, ltime, RouteStatus::Withdrawn))
+                .as_bytes(),
+            &[],
         );
     }
 
-    pub(crate) fn prefixes_iter<'a, M: Meta + 'a>(
-        &'a self,
-    ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
-        PersistedPrefixIter::<AF, M, PREFIX_SIZE, KEY_SIZE> {
+    pub(crate) fn prefixes_iter(
+        &self,
+    ) -> impl Iterator<Item = Vec<Vec<u8>>> + '_ {
+        PersistedPrefixIter::<AF, K, KEY_SIZE> {
             tree_iter: self.tree.iter(None, None),
             cur_rec: None,
             _af: PhantomData,
-            _m: PhantomData,
+            _k: PhantomData,
         }
     }
 
-    pub(crate) fn more_specific_prefix_iter_from<'a, M: Meta + 'a>(
-        &'a self,
-        search_prefix: PrefixId<AF>,
-        mut search_lengths: Vec<u8>,
-        mui: Option<u32>,
-        global_withdrawn_bmin: &'a RoaringBitmap,
-        include_withdrawn: bool,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-        trace!("search more specifics in the persist store.");
-        if search_lengths.is_empty() {
-            for l in search_prefix.get_len() + 1..=AF::BITS {
-                search_lengths.push(l);
-            }
-        }
-        println!("more specific prefix lengths {:?}", search_lengths);
-
-        let len = search_lengths.pop().unwrap();
-        let cur_range = self
-            .get_records_for_more_specific_prefix_in_len(search_prefix, len);
-
-        MoreSpecificPrefixIter {
-            store: self,
-            search_prefix,
-            search_lengths,
-            mui,
-            global_withdrawn_bmin,
-            include_withdrawn,
-            cur_range,
-            next_rec: None,
-        }
-    }
+    // pub(crate) fn more_specific_prefix_iter_from<'a, M: Meta + 'a>(
+    //     &'a self,
+    //     search_prefix: PrefixId<AF>,
+    //     mut search_lengths: Vec<u8>,
+    //     mui: Option<u32>,
+    //     global_withdrawn_bmin: &'a RoaringBitmap,
+    //     include_withdrawn: bool,
+    // ) -> impl Iterator<Item = (PrefixId<AF>, Vec<Vec<u8>>)> + 'a {
+    //     trace!("search more specifics in the persist store.");
+    //     if search_lengths.is_empty() {
+    //         for l in search_prefix.get_len() + 1..=AF::BITS {
+    //             search_lengths.push(l);
+    //         }
+    //     }
+    //     println!("more specific prefix lengths {:?}", search_lengths);
+
+    //     let len = search_lengths.pop().unwrap();
+    //     let cur_range = self
+    //         .get_records_for_more_specific_prefix_in_len(search_prefix, len);
+
+    //     MoreSpecificPrefixIter {
+    //         store: self,
+    //         search_prefix,
+    //         search_lengths,
+    //         mui,
+    //         global_withdrawn_bmin,
+    //         include_withdrawn,
+    //         cur_range,
+    //         next_rec: None,
+    //     }
+    // }
 
-    pub(crate) fn less_specific_prefix_iter_from<'a, M: Meta + 'a>(
-        &'a self,
-        search_lengths: Vec<PrefixId<AF>>,
-        mui: Option<u32>,
-        global_withdrawn_bmin: &'a RoaringBitmap,
-        include_withdrawn: bool,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-        LessSpecificPrefixIter {
-            store: self,
-            search_lengths,
-            mui,
-            global_withdrawn_bmin,
-            include_withdrawn,
-            _m: PhantomData,
-        }
-    }
+    // pub(crate) fn less_specific_prefix_iter_from<'a, M: Meta + 'a>(
+    //     &'a self,
+    //     search_lengths: Vec<PrefixId<AF>>,
+    //     mui: Option<u32>,
+    //     global_withdrawn_bmin: &'a RoaringBitmap,
+    //     include_withdrawn: bool,
+    // ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+    //     LessSpecificPrefixIter {
+    //         store: self,
+    //         search_lengths,
+    //         mui,
+    //         global_withdrawn_bmin,
+    //         include_withdrawn,
+    //         _m: PhantomData,
+    //     }
+    // }
 }
 
-impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
-    std::fmt::Debug for PersistTree<AF, PREFIX_SIZE, KEY_SIZE>
+impl<
+        AF: AddressFamily,
+        K: KeySize<AF, KEY_SIZE>,
+        // const PREFIX_SIZE: usize,
+        const KEY_SIZE: usize,
+    > std::fmt::Debug for PersistTree<AF, K, KEY_SIZE>
 {
     fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         todo!()
@@ -683,25 +914,28 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
 // specified offset.
 pub(crate) struct PersistedPrefixIter<
     AF: AddressFamily,
-    M: Meta,
-    const PREFIX_SIZE: usize,
+    K: KeySize<AF, KEY_SIZE>,
+    // M: Meta,
+    // const PREFIX_SIZE: usize,
     const KEY_SIZE: usize,
 > {
-    cur_rec: Option<(PrefixId<AF>, Vec<PublicRecord<M>>)>,
+    cur_rec: Option<Vec<Vec<u8>>>,
     tree_iter:
         Box<dyn DoubleEndedIterator<Item = Result<KvPair, lsm_tree::Error>>>,
     _af: PhantomData<AF>,
-    _m: PhantomData<M>,
+    // _m: PhantomData<M>,
+    _k: PhantomData<K>,
 }
 
 impl<
         AF: AddressFamily,
-        M: Meta,
-        const PREFIX_SIZE: usize,
+        K: KeySize<AF, KEY_SIZE>,
+        // M: Meta,
+        // const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
-    > Iterator for PersistedPrefixIter<AF, M, PREFIX_SIZE, KEY_SIZE>
+    > Iterator for PersistedPrefixIter<AF, K, KEY_SIZE>
 {
-    type Item = (Prefix, Vec<PublicRecord<M>>);
+    type Item = Vec<Vec<u8>>;
     fn next(&mut self) -> Option<Self::Item> {
         let rec;
 
@@ -719,19 +953,20 @@ impl<
                     return None;
                 }
                 Some(Ok((k, v))) => {
-                    let p_k =
-                        PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::parse_key(
-                            k.as_ref(),
-                        );
-                    rec = Some((
-                        p_k.0,
-                        vec![PublicRecord::<M> {
-                            multi_uniq_id: p_k.1,
-                            ltime: p_k.2,
-                            status: p_k.3,
-                            meta: v.to_vec().into(),
-                        }],
-                    ));
+                    // let p_k =
+                    //     PersistTree::<AF, K, PREFIX_SIZE, KEY_SIZE>::parse_key(
+                    //         k.as_ref(),
+                    //     );
+                    // rec = Some((
+                    //     p_k.0,
+                    //     vec![PublicRecord::<M> {
+                    //         multi_uniq_id: p_k.1,
+                    //         ltime: p_k.2,
+                    //         status: p_k.3,
+                    //         meta: v.to_vec().into(),
+                    //     }],
+                    // ));
+                    rec = Some(vec![[k, v].concat()]);
                 }
                 Some(Err(_)) => {
                     // This is NOT GOOD. Both that it happens, and that we are
@@ -743,200 +978,208 @@ impl<
         };
 
         if let Some(mut r_rec) = rec {
+            let outer_pfx = K::header(&r_rec[0]).prefix;
+
             for (k, v) in self.tree_iter.by_ref().flatten() {
-                let (pfx, mui, ltime, status) =
-                    PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::parse_key(
-                        k.as_ref(),
-                    );
-
-                if pfx == r_rec.0 {
-                    r_rec.1.push(PublicRecord {
-                        meta: v.to_vec().into(),
-                        multi_uniq_id: mui,
-                        ltime,
-                        status,
-                    });
+                // let (pfx, mui, ltime, status) =
+                //     PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::parse_key(
+                //         k.as_ref(),
+                //     );
+                let header = K::header(&k);
+
+                if header.prefix == outer_pfx {
+                    r_rec.push([k, v].concat());
+                    // r_rec.1.push(PublicRecord {
+                    //     meta: v.to_vec().into(),
+                    //     multi_uniq_id: header.mui.into(),
+                    //     ltime: header.ltime.into(),
+                    //     status: header.status,
+                    // });
                 } else {
-                    self.cur_rec = Some((
-                        pfx,
-                        vec![PublicRecord {
-                            meta: v.to_vec().into(),
-                            multi_uniq_id: mui,
-                            ltime,
-                            status,
-                        }],
-                    ));
+                    self.cur_rec = Some(vec![[k, v].concat()]);
+                    // self.cur_rec = Some((
+                    //     header.prefix,
+                    //     vec![PublicRecord {
+                    //         meta: v.to_vec().into(),
+                    //         multi_uniq_id: header.mui.into(),
+                    //         ltime: header.ltime.into(),
+                    //         status: header.status.into(),
+                    //     }],
+                    // ));
                     break;
                 }
             }
 
-            Some((r_rec.0.into(), r_rec.1))
+            Some(r_rec)
         } else {
             None
         }
     }
 }
 
-pub(crate) struct MoreSpecificPrefixIter<
-    'a,
-    AF: AddressFamily + 'a,
-    M: Meta + 'a,
-    const PREFIX_SIZE: usize,
-    const KEY_SIZE: usize,
-> {
-    next_rec: Option<(PrefixId<AF>, Vec<PublicRecord<M>>)>,
-    store: &'a PersistTree<AF, PREFIX_SIZE, KEY_SIZE>,
-    search_prefix: PrefixId<AF>,
-    search_lengths: Vec<u8>,
-    cur_range: Box<
-        dyn DoubleEndedIterator<
-            Item = lsm_tree::Result<(lsm_tree::Slice, lsm_tree::Slice)>,
-        >,
-    >,
-    mui: Option<u32>,
-    global_withdrawn_bmin: &'a RoaringBitmap,
-    include_withdrawn: bool,
-}
-
-impl<
-        'a,
-        AF: AddressFamily + 'a,
-        M: Meta + 'a,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > Iterator for MoreSpecificPrefixIter<'a, AF, M, PREFIX_SIZE, KEY_SIZE>
-{
-    type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
-    fn next(&mut self) -> Option<Self::Item> {
-        let mut cur_pfx = None;
-        let mut recs =
-            if let Some(next_rec) = std::mem::take(&mut self.next_rec) {
-                cur_pfx = Some(next_rec.0);
-                next_rec.1
-            } else {
-                vec![]
-            };
-        loop {
-            if let Some(Ok((k, v))) = self.cur_range.next() {
-                let (pfx, mui, ltime, mut status) =
-                    PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::parse_key(
-                        k.as_ref(),
-                    );
-
-                if !self.include_withdrawn
-                    && (status == RouteStatus::Withdrawn)
-                {
-                    continue;
-                }
-
-                if self.global_withdrawn_bmin.contains(mui) {
-                    if !self.include_withdrawn {
-                        continue;
-                    } else {
-                        status = RouteStatus::Withdrawn;
-                    }
-                }
-
-                if let Some(m) = self.mui {
-                    if m != mui {
-                        continue;
-                    }
-                }
-
-                cur_pfx = if cur_pfx.is_some() {
-                    cur_pfx
-                } else {
-                    Some(pfx)
-                };
-
-                if cur_pfx.is_some_and(|c| c == pfx) {
-                    recs.push(PublicRecord::new(
-                        mui,
-                        ltime,
-                        status,
-                        v.as_ref().to_vec().into(),
-                    ));
-                } else {
-                    self.next_rec = cur_pfx.map(|_p| {
-                        (
-                            pfx,
-                            vec![PublicRecord::new(
-                                mui,
-                                ltime,
-                                status,
-                                v.as_ref().to_vec().into(),
-                            )],
-                        )
-                    });
-                    return Some((pfx, recs));
-                }
-            } else {
-                // See if there's a next prefix length to iterate over
-                if let Some(len) = self.search_lengths.pop() {
-                    self.cur_range = self
-                        .store
-                        .get_records_for_more_specific_prefix_in_len(
-                            self.search_prefix,
-                            len,
-                        );
-                } else {
-                    return cur_pfx.map(|p| (p, recs));
-                }
-            }
-        }
-    }
-}
-
-pub(crate) struct LessSpecificPrefixIter<
-    'a,
-    AF: AddressFamily + 'a,
-    M: Meta + 'a,
-    const PREFIX_SIZE: usize,
-    const KEY_SIZE: usize,
-> {
-    store: &'a PersistTree<AF, PREFIX_SIZE, KEY_SIZE>,
-    search_lengths: Vec<PrefixId<AF>>,
-    mui: Option<u32>,
-    global_withdrawn_bmin: &'a RoaringBitmap,
-    include_withdrawn: bool,
-    _m: PhantomData<M>,
-}
-
-impl<
-        'a,
-        AF: AddressFamily + 'a,
-        M: Meta + 'a,
-        const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > Iterator for LessSpecificPrefixIter<'a, AF, M, PREFIX_SIZE, KEY_SIZE>
-{
-    type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
-    fn next(&mut self) -> Option<Self::Item> {
-        loop {
-            if let Some(lp) = self.search_lengths.pop() {
-                let recs = self.store.get_records_for_prefix(
-                    lp,
-                    self.mui,
-                    self.include_withdrawn,
-                    self.global_withdrawn_bmin,
-                );
-                // .into_iter()
-                // .filter(|r| self.mui.is_none_or(|m| m == r.multi_uniq_id))
-                // .filter(|r| {
-                //     self.include_withdrawn
-                //         || (!self
-                //             .global_withdrawn_bmin
-                //             .contains(r.multi_uniq_id)
-                //             && r.status != RouteStatus::Withdrawn)
-                // })
-                // .collect::<Vec<_>>();
-
-                if !recs.is_empty() {
-                    return Some((lp, recs));
-                }
-            } else {
-                return None;
-            }
-        }
-    }
-}
+// pub(crate) struct MoreSpecificPrefixIter<
+//     'a,
+//     AF: AddressFamily + 'a,
+//     K: KeySize<AF, KEY_SIZE> + 'a,
+//     // M: Meta + 'a,
+//     const PREFIX_SIZE: usize,
+//     const KEY_SIZE: usize,
+// > {
+//     next_rec: Option<(PrefixId<AF>, Vec<Vec<u8>>)>,
+//     store: &'a PersistTree<AF, K, PREFIX_SIZE, KEY_SIZE>,
+//     search_prefix: PrefixId<AF>,
+//     search_lengths: Vec<u8>,
+//     cur_range: Box<
+//         dyn DoubleEndedIterator<
+//             Item = lsm_tree::Result<(lsm_tree::Slice, lsm_tree::Slice)>,
+//         >,
+//     >,
+//     mui: Option<u32>,
+//     global_withdrawn_bmin: &'a RoaringBitmap,
+//     include_withdrawn: bool,
+// }
+
+// impl<
+//         'a,
+//         AF: AddressFamily + 'a,
+//         K: KeySize<AF, KEY_SIZE> + 'a,
+//         // M: Meta + 'a,
+//         const PREFIX_SIZE: usize,
+//         const KEY_SIZE: usize,
+//     > Iterator for MoreSpecificPrefixIter<'a, AF, K, PREFIX_SIZE, KEY_SIZE>
+// {
+//     type Item = (PrefixId<AF>, Vec<Vec<u8>>);
+//     fn next(&mut self) -> Option<Self::Item> {
+//         let mut cur_pfx = None;
+//         let mut recs =
+//             if let Some(next_rec) = std::mem::take(&mut self.next_rec) {
+//                 cur_pfx = Some(next_rec.0);
+//                 next_rec.1
+//             } else {
+//                 vec![]
+//             };
+//         loop {
+//             if let Some(Ok((k, v))) = self.cur_range.next() {
+//                 // let (pfx, mui, ltime, mut status) =
+//                 let mut v = [k, v].concat();
+//                 let key = K::header_mut(&mut v);
+
+//                 if !self.include_withdrawn
+//                     && (key.status == RouteStatus::Withdrawn)
+//                 {
+//                     continue;
+//                 }
+
+//                 if self.global_withdrawn_bmin.contains(key.mui.into()) {
+//                     if !self.include_withdrawn {
+//                         continue;
+//                     } else {
+//                         key.status = RouteStatus::Withdrawn;
+//                     }
+//                 }
+
+//                 if let Some(m) = self.mui {
+//                     if m != key.mui.into() {
+//                         continue;
+//                     }
+//                 }
+
+//                 cur_pfx = if cur_pfx.is_some() {
+//                     cur_pfx
+//                 } else {
+//                     Some(key.prefix)
+//                 };
+
+//                 if cur_pfx.is_some_and(|c| c == key.prefix) {
+//                     // recs.push(PublicRecord::new(
+//                     //     mui,
+//                     //     ltime,
+//                     //     status,
+//                     //     v.as_ref().to_vec().into(),
+//                     // ));
+//                     recs.push(v);
+//                 } else {
+//                     self.next_rec = cur_pfx.map(|_| {
+//                         (key.prefix, vec![v])
+//                         // vec![PublicRecord::new(
+//                         //     mui,
+//                         //     ltime,
+//                         //     status,
+//                         //     v.as_ref().to_vec().into(),
+//                         // )],
+//                     });
+//                     return Some((key.prefix, recs));
+//                 }
+//             } else {
+//                 // See if there's a next prefix length to iterate over
+//                 if let Some(len) = self.search_lengths.pop() {
+//                     self.cur_range = self
+//                         .store
+//                         .get_records_for_more_specific_prefix_in_len(
+//                             self.search_prefix,
+//                             len,
+//                         );
+//                 } else {
+//                     return cur_pfx.map(|p| (p, recs));
+//                 }
+//             }
+//         }
+//     }
+// }
+
+// pub(crate) struct LessSpecificPrefixIter<
+//     'a,
+//     AF: AddressFamily + 'a,
+//     K: KeySize<AF, KEY_SIZE> + 'a,
+//     M: Meta + 'a,
+//     const PREFIX_SIZE: usize,
+//     const KEY_SIZE: usize,
+// > {
+//     store: &'a PersistTree<AF, K, PREFIX_SIZE, KEY_SIZE>,
+//     search_lengths: Vec<PrefixId<AF>>,
+//     mui: Option<u32>,
+//     global_withdrawn_bmin: &'a RoaringBitmap,
+//     include_withdrawn: bool,
+//     _m: PhantomData<M>,
+// }
+
+// impl<
+//         'a,
+//         AF: AddressFamily + 'a,
+//         K: KeySize<AF, KEY_SIZE> + 'a,
+//         M: Meta + 'a,
+//         const PREFIX_SIZE: usize,
+//         const KEY_SIZE: usize,
+//     > Iterator
+//     for LessSpecificPrefixIter<'a, AF, K, M, PREFIX_SIZE, KEY_SIZE>
+// {
+//     type Item = (PrefixId<AF>, Vec<PublicRecord<M>>);
+//     fn next(&mut self) -> Option<Self::Item> {
+//         loop {
+//             if let Some(lp) = self.search_lengths.pop() {
+//                 let recs = self.store.get_records_for_prefix(
+//                     lp,
+//                     self.mui,
+//                     self.include_withdrawn,
+//                     self.global_withdrawn_bmin,
+//                 );
+//                 // .into_iter()
+//                 // .filter(|r| self.mui.is_none_or(|m| m == r.multi_uniq_id))
+//                 // .filter(|r| {
+//                 //     self.include_withdrawn
+//                 //         || (!self
+//                 //             .global_withdrawn_bmin
+//                 //             .contains(r.multi_uniq_id)
+//                 //             && r.status != RouteStatus::Withdrawn)
+//                 // })
+//                 // .collect::<Vec<_>>();
+
+//                 if !recs.is_empty() {
+//                     return Some((lp, recs));
+//                 }
+//             } else {
+//                 return None;
+//             }
+//         }
+//     }
+// }
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index cf20aa05..93012a20 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -1,12 +1,14 @@
 use crossbeam_epoch::{self as epoch};
 use epoch::Guard;
 use log::trace;
+use zerocopy::TryFromBytes;
 
 use crate::af::AddressFamily;
 use crate::local_array::in_memory::atomic_types::{
     NodeBuckets, PrefixBuckets,
 };
-use crate::rib::{PersistStrategy, Rib};
+use crate::prefix_record::ZeroCopyRecord;
+use crate::rib::{Config, PersistStrategy, Rib};
 use crate::PublicRecord;
 use inetnum::addr::Prefix;
 
@@ -15,15 +17,17 @@ use crate::{Meta, QueryResult};
 use crate::{MatchOptions, MatchType};
 
 use super::errors::PrefixStoreError;
+use super::persist::lsm_tree::KeySize;
 use super::types::PrefixId;
 
 //------------ Prefix Matching ----------------------------------------------
 
-impl<'a, AF, M, NB, PB, const PREFIX_SIZE: usize, const KEY_SIZE: usize>
-    Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+impl<'a, AF, M, K, NB, PB, C: Config, const KEY_SIZE: usize>
+    Rib<AF, M, K, NB, PB, C, KEY_SIZE>
 where
     AF: AddressFamily,
     M: Meta,
+    K: KeySize<AF, KEY_SIZE>,
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
 {
@@ -36,13 +40,33 @@ where
     ) -> Option<Vec<PublicRecord<M>>> {
         match self.persist_strategy() {
             PersistStrategy::PersistOnly => {
-                self.persist_tree.as_ref().map(|tree| {
+                trace!("get value from persist_store for {:?}", prefix_id);
+                self.persist_tree.as_ref().and_then(|tree| {
                     tree.get_records_for_prefix(
                         prefix_id,
                         mui,
                         include_withdrawn,
                         self.in_memory_tree.withdrawn_muis_bmin(guard),
                     )
+                    .map(|v| {
+                        v.iter()
+                            .map(|bytes| {
+                                let record: &ZeroCopyRecord<AF> =
+                                    ZeroCopyRecord::try_ref_from_bytes(bytes)
+                                        .unwrap();
+                                PublicRecord::<M> {
+                                    multi_uniq_id: record.multi_uniq_id,
+                                    ltime: record.ltime,
+                                    status: record.status,
+                                    meta: <Vec<u8>>::from(
+                                        record.meta.as_ref(),
+                                    )
+                                    .into(),
+                                }
+                            })
+                            .collect::<Vec<_>>()
+                    })
+                    .ok()
                 })
             }
             _ => self.prefix_cht.get_records_for_prefix(
@@ -148,28 +172,28 @@ where
         })
         .into_iter()
         .flatten()
-        .chain(
-            (if mui.is_some_and(|m| {
-                self.config.persist_strategy == PersistStrategy::WriteAhead
-                    || (!include_withdrawn && self.mui_is_withdrawn(m, guard))
-            }) {
-                None
-            } else {
-                let global_withdrawn_bmin =
-                    self.in_memory_tree.withdrawn_muis_bmin(guard);
-                self.persist_tree.as_ref().map(|persist_tree| {
-                    persist_tree.more_specific_prefix_iter_from(
-                        prefix_id,
-                        vec![],
-                        mui,
-                        global_withdrawn_bmin,
-                        include_withdrawn,
-                    )
-                })
-            })
-            .into_iter()
-            .flatten(),
-        )
+        // .chain(
+        //     (if mui.is_some_and(|m| {
+        //         self.config.persist_strategy == PersistStrategy::WriteAhead
+        //             || (!include_withdrawn && self.mui_is_withdrawn(m, guard))
+        //     }) {
+        //         None
+        //     } else {
+        //         let global_withdrawn_bmin =
+        //             self.in_memory_tree.withdrawn_muis_bmin(guard);
+        //         self.persist_tree.as_ref().map(|persist_tree| {
+        //             persist_tree.more_specific_prefix_iter_from(
+        //                 prefix_id,
+        //                 vec![],
+        //                 mui,
+        //                 global_withdrawn_bmin,
+        //                 include_withdrawn,
+        //             )
+        //         })
+        //     })
+        //     .into_iter()
+        //     .flatten(),
+        // )
     }
 
     pub(crate) fn less_specifics_iter_from(
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index 8f591cfc..e52ab0e8 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -1,31 +1,64 @@
 use crate::local_array::in_memory::atomic_types::NodeSet;
+use crate::local_array::persist::lsm_tree::LongKey;
 use crate::prelude::multi::*;
 use crate::prelude::*;
 use rand::prelude::*;
 
 // The default stride sizes for IPv4, IPv6, resp.
 #[create_store((
-    // ([5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 5, 18),
-    ([4, 4, 4, 4, 4, 4, 4, 4], 5, 18),
-    ([4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-    4, 4, 4, 4, 4, 4, 4, 4, 4], 17, 30)
+    ([4, 4, 4, 4, 4, 4, 4, 4], 5, 18, LongKey),
+    ([4, 4, 4, 4, 4, 4, 4, 4,
+      4, 4, 4, 4, 4, 4, 4, 4,
+      4, 4, 4, 4, 4, 4, 4, 4,
+      4, 4, 4, 4, 4, 4, 4, 4], 17, 30, LongKey)
 ))]
 struct DefaultStore;
 
 /// Try some
-impl<M: Meta> DefaultStore<M> {
+impl<M: Meta, C: Config> DefaultStore<M, C> {
     pub fn try_default() -> Result<Self, PrefixStoreError> {
-        let config = StoreConfig::default();
+        let config = C::default();
         Self::new_with_config(config)
             .map_err(|_| PrefixStoreError::StoreNotReadyError)
     }
 }
 
-impl Default for StoreConfig {
-    fn default() -> Self {
-        Self {
-            persist_strategy: PersistStrategy::MemoryOnly,
-            persist_path: "/tmp/rotonda/".to_string(),
-        }
-    }
-}
+// impl Default for StoreConfig {
+//     fn default() -> Self {
+//         Self {
+//             persist_strategy: PersistStrategy::MemoryOnly,
+//             persist_path: "/tmp/rotonda/".to_string(),
+//         }
+//     }
+// }
+
+// impl StoreConfig {
+//     fn persist_default() -> Self {
+//         Self {
+//             persist_strategy: PersistStrategy::PersistOnly,
+//             persist_path: "/tmp/rotonda/".to_string(),
+//         }
+//     }
+// }
+
+// pub mod persist_only_store {
+//     use crate::local_array::in_memory::atomic_types::NodeSet;
+//     use crate::local_array::persist::lsm_tree::ShortKey;
+//     use crate::prelude::multi::*;
+//     use crate::prelude::*;
+//     use rand::prelude::*;
+//     #[create_store((
+//         ([4], 5, 18, PersistOnlyConfig, ShortKey),
+//         ([4], 17, 30, PersistOnlyconfig, ShortKey)
+//     ))]
+//     struct PersistOnlyStore;
+
+//     /// Try some
+//     impl<M: Meta, C: Config> PersistOnlyStore<M, C> {
+//         pub fn try_default() -> Result<Self, PrefixStoreError> {
+//             let config = C::default();
+//             Self::new_with_config(config)
+//                 .map_err(|_| PrefixStoreError::StoreNotReadyError)
+//         }
+//     }
+// }
diff --git a/src/local_array/rib/mod.rs b/src/local_array/rib/mod.rs
index 3c39e96a..96373e90 100644
--- a/src/local_array/rib/mod.rs
+++ b/src/local_array/rib/mod.rs
@@ -4,5 +4,6 @@ pub mod rib;
 pub(crate) mod default_store;
 
 pub use default_store::DefaultStore;
+
 #[macro_use]
 mod macros;
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index bdb67780..7dae0d64 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -3,14 +3,17 @@ use std::path::Path;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use inetnum::addr::Prefix;
-use log::info;
+use log::{info, trace};
 
 use crossbeam_epoch::{self as epoch};
 use epoch::{Guard, Owned};
+use zerocopy::TryFromBytes;
 
 use crate::local_array::in_memory::tree::TreeBitMap;
+use crate::local_array::persist::lsm_tree::{KeySize, LongKey, ShortKey};
 use crate::local_array::prefix_cht::cht::PrefixCHT;
 use crate::local_array::types::PrefixId;
+use crate::prefix_record::{ValueHeader, ZeroCopyRecord};
 use crate::prelude::multi::RouteStatus;
 use crate::stats::CreatedNodes;
 use crate::{
@@ -28,8 +31,9 @@ pub use crate::local_array::query;
 use crate::{IPv4, IPv6, Meta};
 
 use crate::AddressFamily;
+use zerocopy::IntoBytes;
 
-//------------ StoreConfig ---------------------------------------------------
+//------------ Config --------------------------------------------------------
 
 /// Defines where records are stored, in-memory and/or persisted (to disk),
 /// and, whether new records for a unique (prefix, mui) pair are overwritten
@@ -48,15 +52,121 @@ pub enum PersistStrategy {
     PersistOnly,
 }
 
-#[derive(Debug, Clone)]
-pub struct StoreConfig {
-    pub persist_strategy: PersistStrategy,
-    pub persist_path: String,
+pub trait Config: Clone + Default + std::fmt::Debug {
+    fn persist_strategy(&self) -> PersistStrategy;
+    fn persist_path(&self) -> Option<String>;
+    fn set_persist_path(&mut self, path: String);
 }
 
-impl StoreConfig {
-    pub fn persist_strategy(&self) -> PersistStrategy {
-        self.persist_strategy
+//------------ MemoryOnlyConfig ----------------------------------------------
+
+#[derive(Copy, Clone, Debug)]
+pub struct MemoryOnlyConfig;
+
+impl Config for MemoryOnlyConfig {
+    fn persist_strategy(&self) -> PersistStrategy {
+        PersistStrategy::MemoryOnly
+    }
+
+    fn persist_path(&self) -> Option<String> {
+        None
+    }
+
+    fn set_persist_path(&mut self, _: String) {
+        unimplemented!()
+    }
+}
+
+impl Default for MemoryOnlyConfig {
+    fn default() -> Self {
+        Self
+    }
+}
+
+//------------ PeristOnlyConfig ----------------------------------------------
+
+impl Default for PersistOnlyConfig {
+    fn default() -> Self {
+        Self {
+            persist_path: "/tmp/rotonda/".to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct PersistOnlyConfig {
+    persist_path: String,
+}
+
+impl Config for PersistOnlyConfig {
+    fn persist_strategy(&self) -> PersistStrategy {
+        PersistStrategy::PersistOnly
+    }
+
+    fn persist_path(&self) -> Option<String> {
+        Some(self.persist_path.clone())
+    }
+
+    fn set_persist_path(&mut self, path: String) {
+        self.persist_path = path;
+    }
+}
+
+//------------ WriteAheadConfig ----------------------------------------------
+
+impl Default for WriteAheadConfig {
+    fn default() -> Self {
+        Self {
+            persist_path: "/tmp/rotonda/".to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct WriteAheadConfig {
+    persist_path: String,
+}
+
+impl Config for WriteAheadConfig {
+    fn persist_strategy(&self) -> PersistStrategy {
+        PersistStrategy::WriteAhead
+    }
+
+    fn persist_path(&self) -> Option<String> {
+        Some(self.persist_path.clone())
+    }
+
+    fn set_persist_path(&mut self, path: String) {
+        self.persist_path = path;
+    }
+}
+
+//------------ PersistHistoryConfig ------------------------------------------
+
+#[derive(Clone, Debug)]
+pub struct PersistHistoryConfig {
+    persist_path: String,
+}
+
+impl Config for PersistHistoryConfig {
+    fn persist_strategy(&self) -> PersistStrategy {
+        PersistStrategy::PersistHistory
+    }
+
+    fn persist_path(&self) -> Option<String> {
+        Some(self.persist_path.clone())
+    }
+
+    fn set_persist_path(&mut self, path: String) {
+        self.persist_path = path;
+    }
+}
+
+impl Default for PersistHistoryConfig {
+    fn default() -> Self {
+        Self {
+            persist_path: "/tmp/rotonda/".to_string(),
+        }
     }
 }
 
@@ -220,46 +330,65 @@ pub struct UpsertReport {
 pub struct Rib<
     AF: AddressFamily,
     M: Meta,
+    K: KeySize<AF, KEY_SIZE>,
     NB: NodeBuckets<AF>,
     PB: PrefixBuckets<AF, M>,
-    const PREFIX_SIZE: usize,
+    C: Config,
     const KEY_SIZE: usize,
 > {
-    pub config: StoreConfig,
+    pub config: C,
     pub(crate) in_memory_tree: TreeBitMap<AF, NB>,
     pub(crate) prefix_cht: PrefixCHT<AF, M, PB>,
-    #[cfg(feature = "persist")]
     pub(in crate::local_array) persist_tree:
-        Option<PersistTree<AF, PREFIX_SIZE, KEY_SIZE>>,
+        Option<PersistTree<AF, K, KEY_SIZE>>,
     pub counters: Counters,
 }
 
 impl<
         AF: AddressFamily,
         M: crate::prefix_record::Meta,
+        K: KeySize<AF, KEY_SIZE>,
         NB: NodeBuckets<AF>,
         PB: PrefixBuckets<AF, M>,
-        const PREFIX_SIZE: usize,
+        C: Config,
         const KEY_SIZE: usize,
-    > Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+    > Rib<AF, M, K, NB, PB, C, KEY_SIZE>
 {
+    #[allow(clippy::type_complexity)]
     pub(crate) fn new(
-        config: StoreConfig,
-    ) -> Result<
-        Rib<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>,
-        Box<dyn std::error::Error>,
-    > {
-        Rib::<AF, M, NB, PB, PREFIX_SIZE, KEY_SIZE>::init(config)
+        config: C,
+    ) -> Result<Rib<AF, M, K, NB, PB, C, KEY_SIZE>, Box<dyn std::error::Error>>
+    {
+        Rib::<AF, M, K, NB, PB, C, KEY_SIZE>::init(config)
     }
 
-    fn init(config: StoreConfig) -> Result<Self, Box<dyn std::error::Error>> {
+    // pub(crate) fn new_short_key(
+    //     config: StoreConfig,
+    // ) -> Result<
+    //     Rib<AF, M, ShortKey<AF>, NB, PB, KEY_SIZE>,
+    //     Box<dyn std::error::Error>,
+    // > {
+    //     Rib::<AF, M, ShortKey<AF>, NB, PB, KEY_SIZE>::init(config)
+    // }
+
+    // pub(crate) fn new_long_key(
+    //     config: StoreConfig,
+    // ) -> Result<
+    //     Rib<AF, M, LongKey<AF>, NB, PB, KEY_SIZE>,
+    //     Box<dyn std::error::Error>,
+    // > {
+    //     Rib::<AF, M, LongKey<AF>, NB, PB, KEY_SIZE>::init(config)
+    // }
+
+    fn init(config: C) -> Result<Self, Box<dyn std::error::Error>> {
         info!("store: initialize store {}", AF::BITS);
 
-        let persist_tree = match config.persist_strategy {
+        let persist_tree = match config.persist_strategy() {
             PersistStrategy::MemoryOnly => None,
             _ => {
-                let persist_path = &Path::new(&config.persist_path);
-                Some(PersistTree::new(persist_path))
+                let persist_path = &config.persist_path().unwrap();
+                let pp_ref = &Path::new(persist_path);
+                Some(PersistTree::new(pp_ref))
             }
         };
 
@@ -280,10 +409,12 @@ impl<
         record: PublicRecord<M>,
         update_path_selections: Option<M::TBI>,
     ) -> Result<UpsertReport, PrefixStoreError> {
+        trace!("try insertingf {:?}", prefix);
         let guard = &epoch::pin();
         self.in_memory_tree
             .set_prefix_exists(prefix, record.multi_uniq_id)
             .and_then(|(retry_count, exists)| {
+                trace!("exists, upsert it");
                 self.upsert_prefix(
                     prefix,
                     record,
@@ -312,10 +443,10 @@ impl<
         guard: &Guard,
     ) -> Result<UpsertReport, PrefixStoreError> {
         let mui = record.multi_uniq_id;
-        match self.config.persist_strategy {
+        match self.config.persist_strategy() {
             PersistStrategy::WriteAhead => {
                 if let Some(persist_tree) = &self.persist_tree {
-                    persist_tree.persist_record(prefix, &record);
+                    persist_tree.persist_record_w_short_key(prefix, &record);
 
                     self.prefix_cht
                         .upsert_prefix(
@@ -335,7 +466,7 @@ impl<
                 .map(|(report, old_rec)| {
                     if let Some(rec) = old_rec {
                         if let Some(persist_tree) = &self.persist_tree {
-                            persist_tree.persist_record(
+                            persist_tree.persist_record_w_long_key(
                                 prefix,
                                 &PublicRecord::from((mui, &rec)),
                             );
@@ -352,7 +483,7 @@ impl<
                     let (retry_count, exists) = self
                         .in_memory_tree
                         .set_prefix_exists(prefix, record.multi_uniq_id)?;
-                    persist_tree.persist_record(prefix, &record);
+                    persist_tree.persist_record_w_short_key(prefix, &record);
                     Ok(UpsertReport {
                         cas_count: retry_count as usize,
                         prefix_new: exists,
@@ -404,26 +535,55 @@ impl<
                     prefix, mui
                 );
                 let p_tree = self.persist_tree.as_ref().unwrap();
-                let stored_prefixes = p_tree
-                    .get_records_with_keys_for_prefix_mui::<M>(prefix, mui);
+                let stored_prefixes =
+                    p_tree.get_records_with_keys_for_prefix_mui(prefix, mui);
 
                 for s in stored_prefixes {
-                    let key: [u8; KEY_SIZE] = PersistTree::<
-                        AF,
-                        PREFIX_SIZE,
-                        KEY_SIZE,
-                    >::persistence_key(
-                        prefix,
-                        mui,
+                    // let key: [u8; KEY_SIZE] = PersistTree::<
+                    //     AF,
+                    //     // PREFIX_SIZE,
+                    //     K,
+                    //     KEY_SIZE,
+                    // >::persistence_key(
+                    //     prefix,
+                    //     mui,
+                    //     ltime,
+                    //     RouteStatus::Withdrawn,
+                    // );
+
+                    // let key = K::new_write_key(
+                    //     prefix,
+                    //     mui,
+                    //     ltime,
+                    //     RouteStatus::Withdrawn,
+                    // );
+                    // let record =
+                    //     ZeroCopyRecord::<AF>::try_ref_from_prefix(&s)
+                    //         .unwrap()
+                    //         .0;
+                    // let key = ShortKey::from((record.prefix, mui));
+                    // trace!("insert key {:?}", key);
+
+                    // // new header for this value
+                    // let mut value = ValueHeader {
+                    //     ltime,
+                    //     status: RouteStatus::Withdrawn,
+                    // }
+                    // .as_bytes()
+                    // .to_vec();
+                    // value.extend_from_slice(record.meta.as_ref());
+
+                    // p_tree.insert(key.as_bytes(), &value);
+                    let header = ValueHeader {
                         ltime,
-                        RouteStatus::Withdrawn,
-                    );
-
-                    p_tree.insert(key, s.1.meta.as_ref());
+                        status: RouteStatus::Withdrawn,
+                    };
+                    p_tree.rewrite_header_for_record(header, &s);
 
                     // remove the entry for the same (prefix, mui), but with
                     // an older logical time
-                    p_tree.remove(*s.0.first_chunk::<KEY_SIZE>().unwrap());
+                    // let key = K::short_key(&s);
+                    // p_tree.remove(key.as_bytes());
                 }
             }
             PersistStrategy::PersistHistory => {
@@ -449,21 +609,40 @@ impl<
                             return Err(PrefixStoreError::StoreNotReadyError);
                         };
 
-                    let key: [u8; KEY_SIZE] = PersistTree::<
-                        AF,
-                        PREFIX_SIZE,
-                        KEY_SIZE,
-                    >::persistence_key(
-                        prefix,
-                        mui,
-                        ltime,
-                        RouteStatus::Withdrawn,
-                    );
+                    // let key: [u8; KEY_SIZE] = PersistTree::<
+                    //     AF,
+                    //     K,
+                    //     // PREFIX_SIZE,
+                    //     KEY_SIZE,
+                    // >::persistence_key(
+                    //     prefix,
+                    //     mui,
+                    //     ltime,
+                    //     RouteStatus::Withdrawn,
+                    // );
+                    // let key = K::new_write_key(
+                    //     prefix,
+                    //     mui,
+                    //     ltime,
+                    //     RouteStatus::Withdrawn,
+                    // );
+                    //
+
                     // Here we are keeping persisted history, so no removal of
                     // old (prefix, mui) records.
                     // We are inserting an empty record, since this is a
                     // withdrawal.
-                    p_tree.insert(key, &[]);
+                    // p_tree.insert(
+                    //     LongKey::from((
+                    //         prefix,
+                    //         mui,
+                    //         ltime,
+                    //         RouteStatus::Withdrawn,
+                    //     ))
+                    //     .as_bytes(),
+                    //     &[],
+                    // );
+                    p_tree.insert_empty_record(prefix, mui, ltime);
                 }
             }
         }
@@ -494,33 +673,54 @@ impl<
                 // let stored_prefixes = p_tree
                 //     .get_records_with_keys_for_prefix_mui::<M>(prefix, mui)
 
-                if let Some(record) = p_tree
-                    .get_most_recent_record_for_prefix_mui::<M>(prefix, mui)
+                if let Some(mut record_b) =
+                    p_tree.get_most_recent_record_for_prefix_mui(prefix, mui)
                 {
-                    let new_key: [u8; KEY_SIZE] = PersistTree::<
-                        AF,
-                        PREFIX_SIZE,
-                        KEY_SIZE,
-                    >::persistence_key(
-                        prefix,
-                        mui,
+                    // let new_key: [u8; KEY_SIZE] = PersistTree::<
+                    //     AF,
+                    //     // PREFIX_SIZE,
+                    //     KEY_SIZE,
+                    // >::persistence_key(
+                    //     prefix,
+                    //     mui,
+                    //     ltime,
+                    //     RouteStatus::Active,
+                    // );
+
+                    // let record: &mut ZeroCopyRecord<AF> =
+                    //     ZeroCopyRecord::try_mut_from_bytes(&mut record_b)
+                    //         .unwrap();
+                    // // record.prefix = prefix;
+                    // // record.multi_uniq_id = mui;
+                    // record.ltime = ltime;
+                    // record.status = RouteStatus::Active;
+
+                    let header = ValueHeader {
                         ltime,
-                        RouteStatus::Active,
-                    );
-                    p_tree.insert(new_key, record.meta.as_ref());
-
-                    // remove the entry for the same (prefix, mui), but with
-                    // an older logical time
-                    let old_key = PersistTree::<
-                        AF,
-                        PREFIX_SIZE,
-                        KEY_SIZE>::persistence_key(
-                            prefix,
-                            mui,
-                            record.ltime,
-                            record.status
-                    );
-                    p_tree.remove(old_key);
+                        status: RouteStatus::Active,
+                    };
+                    // .as_bytes()
+                    // .to_vec();
+                    // value.extend_from_slice(record.meta.as_ref());
+
+                    // p_tree.insert(
+                    //     ShortKey::from((prefix, mui)).as_bytes(),
+                    //     value.as_bytes(),
+                    // );
+                    p_tree.rewrite_header_for_record(header, &record_b);
+                    // // remove the entry for the same (prefix, mui), but with
+                    // // an older logical time
+                    // let old_key = PersistTree::<
+                    //     AF,
+                    //     // PREFIX_SIZE,
+                    //     KEY_SIZE,
+                    // >::persistence_key(
+                    //     prefix,
+                    //     mui,
+                    //     record.ltime,
+                    //     record.status,
+                    // );
+                    // p_tree.remove(old_key);
                 }
             }
             PersistStrategy::PersistHistory => {
@@ -544,21 +744,27 @@ impl<
                             return Err(PrefixStoreError::StoreNotReadyError);
                         };
 
-                    let key: [u8; KEY_SIZE] = PersistTree::<
-                        AF,
-                        PREFIX_SIZE,
-                        KEY_SIZE,
-                    >::persistence_key(
-                        prefix,
-                        mui,
-                        ltime,
-                        RouteStatus::Active,
-                    );
+                    // let key: [u8; KEY_SIZE] = PersistTree::<
+                    //     AF,
+                    //     // PREFIX_SIZE,
+                    //     KEY_SIZE,
+                    // >::persistence_key(
+                    //     prefix,
+                    //     mui,
+                    //     ltime,
+                    //     RouteStatus::Active,
+                    // );
+                    // let key = K::new_write_key(
+                    //     prefix,
+                    //     mui,
+                    //     ltime,
+                    //     RouteStatus::Active,
+                    // );
                     // Here we are keeping persisted history, so no removal of
                     // old (prefix, mui) records.
                     // We are inserting an empty record, since this is a
                     // withdrawal.
-                    p_tree.insert(key, &[]);
+                    p_tree.insert_empty_record(prefix, mui, ltime);
                 }
             }
         }
@@ -700,9 +906,45 @@ impl<
     //-------- Persistence ---------------------------------------------------
 
     pub fn persist_strategy(&self) -> PersistStrategy {
-        self.config.persist_strategy
+        self.config.persist_strategy()
     }
 
+    pub fn persist_prefixes_iter(
+        &self,
+    ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + '_ {
+        self.persist_tree
+            .as_ref()
+            .map(|tree| {
+                tree.prefixes_iter().map(|recs| {
+                    (
+                        Prefix::from(
+                            ZeroCopyRecord::<AF>::try_ref_from_bytes(
+                                &recs[0],
+                            )
+                            .unwrap()
+                            .prefix,
+                        ),
+                        recs.iter()
+                            .map(|rec| {
+                                let rec =
+                                    ZeroCopyRecord::<AF>::try_ref_from_bytes(
+                                        rec,
+                                    )
+                                    .unwrap();
+                                PublicRecord {
+                                    multi_uniq_id: rec.multi_uniq_id,
+                                    ltime: rec.ltime,
+                                    status: rec.status,
+                                    meta: rec.meta.to_vec().into(),
+                                }
+                            })
+                            .collect::<Vec<PublicRecord<M>>>(),
+                    )
+                })
+            })
+            .into_iter()
+            .flatten()
+    }
     // pub fn get_records_for_prefix(
     //     &self,
     //     prefix: &Prefix,
@@ -764,11 +1006,13 @@ impl<
 
 impl<
         M: Meta,
+        K: KeySize<IPv4, KEY_SIZE>,
         NB: NodeBuckets<IPv4>,
         PB: PrefixBuckets<IPv4, M>,
-        const PREFIX_SIZE: usize,
+        // const PREFIX_SIZE: usize,
+        C: Config,
         const KEY_SIZE: usize,
-    > std::fmt::Display for Rib<IPv4, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+    > std::fmt::Display for Rib<IPv4, M, K, NB, PB, C, KEY_SIZE>
 {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "Rib<IPv4, {}>", std::any::type_name::<M>())
@@ -777,11 +1021,13 @@ impl<
 
 impl<
         M: Meta,
+        K: KeySize<IPv6, KEY_SIZE>,
         NB: NodeBuckets<IPv6>,
         PB: PrefixBuckets<IPv6, M>,
-        const PREFIX_SIZE: usize,
+        C: Config,
+        // const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
-    > std::fmt::Display for Rib<IPv6, M, NB, PB, PREFIX_SIZE, KEY_SIZE>
+    > std::fmt::Display for Rib<IPv6, M, K, NB, PB, C, KEY_SIZE>
 {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "Rib<IPv6, {}>", std::any::type_name::<M>())
diff --git a/src/local_array/types.rs b/src/local_array/types.rs
index 793bb3eb..958c195b 100644
--- a/src/local_array/types.rs
+++ b/src/local_array/types.rs
@@ -1,15 +1,28 @@
 use log::trace;
-use zerocopy::TryFromBytes;
+use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes, Unaligned};
 
 use crate::AddressFamily;
 
 use super::errors::PrefixStoreError;
 
-#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    Hash,
+    PartialEq,
+    Eq,
+    TryFromBytes,
+    KnownLayout,
+    Immutable,
+    Unaligned,
+    IntoBytes,
+)]
+#[repr(u8)]
 pub enum RouteStatus {
-    Active,
-    InActive,
-    Withdrawn,
+    Active = 1,
+    InActive = 2,
+    Withdrawn = 3,
 }
 
 impl std::fmt::Display for RouteStatus {
diff --git a/src/macros.rs b/src/macros.rs
index f0c94061..57e90774 100644
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -51,25 +51,24 @@ macro_rules! all_strategies {
         $(
             #[test]
             fn $fn_name() -> Result<(), Box<dyn std::error::Error>> {
+                use rotonda_store::prelude::multi::*;
                 //------- Default (MemoryOnly)
-
-                println!("default strategy starting...");
+                println!("MemoryOnly strategy starting...");
                 let tree_bitmap =
-                    MultiThreadedStore::<$ty>::try_default()?;
+                    MultiThreadedStore::<$ty, MemoryOnlyConfig>::try_default()?;
 
                 $test_name(tree_bitmap)?;
 
                 //------- PersistOnly
 
                 println!("PersistOnly strategy starting...");
-                let store_config = StoreConfig {
-                    persist_strategy:
-                        rotonda_store::rib::PersistStrategy::PersistOnly,
-                    persist_path: "/tmp/rotonda/".into(),
-                };
+                let mut store_config = PersistOnlyConfig::default();
+                store_config.set_persist_path(
+                    "/tmp/rotonda/".into()
+                );
 
                 let tree_bitmap = MultiThreadedStore::<
-                    $ty,
+                    $ty, PersistOnlyConfig
                 >::new_with_config(
                     store_config
                 )?;
@@ -79,14 +78,14 @@ macro_rules! all_strategies {
                 //------- PersistHistory
 
                 println!("PersistHistory strategy starting...");
-                let store_config = StoreConfig {
-                    persist_strategy:
-                        rotonda_store::rib::PersistStrategy::PersistHistory,
-                    persist_path: "/tmp/rotonda/".into(),
-                };
+                let mut store_config = PersistHistoryConfig::default();
+                store_config.set_persist_path(
+                    "/tmp/rotonda/".into()
+                );
 
                 let tree_bitmap = MultiThreadedStore::<
                     $ty,
+                    PersistHistoryConfig
                 >::new_with_config(
                     store_config
                 )?;
@@ -96,14 +95,15 @@ macro_rules! all_strategies {
                 //------- WriteAhead
 
                 println!("WriteAhead strategy starting...");
-                let store_config = StoreConfig {
-                    persist_strategy:
-                        rotonda_store::rib::PersistStrategy::WriteAhead,
-                    persist_path: "/tmp/rotonda/".into(),
-                };
+
+                let mut store_config = WriteAheadConfig::default();
+                store_config.set_persist_path(
+                    "/tmp/rotonda/".into()
+                );
 
                 let tree_bitmap = MultiThreadedStore::<
                     $ty,
+                    WriteAheadConfig
                 >::new_with_config(
                     store_config
                 )?;
diff --git a/src/prefix_record.rs b/src/prefix_record.rs
index 70700d79..33169ebc 100644
--- a/src/prefix_record.rs
+++ b/src/prefix_record.rs
@@ -6,7 +6,10 @@ use crate::af::AddressFamily;
 use crate::local_array::types::RouteStatus;
 use crate::prelude::multi::PrefixId;
 use inetnum::addr::Prefix;
-use zerocopy::{NetworkEndian, U128, U32};
+use zerocopy::{
+    Immutable, IntoBytes, KnownLayout, NetworkEndian, TryFromBytes,
+    Unaligned, U128, U32,
+};
 
 //------------ InternalPrefixRecord -----------------------------------------
 
@@ -272,6 +275,44 @@ impl<M: std::fmt::Display> std::fmt::Display for PublicRecord<M> {
     }
 }
 
+#[derive(KnownLayout, Immutable, Unaligned, IntoBytes, TryFromBytes)]
+#[repr(C, packed)]
+pub struct ZeroCopyRecord<AF: AddressFamily> {
+    pub prefix: PrefixId<AF>,
+    pub multi_uniq_id: u32,
+    pub ltime: u64,
+    pub status: RouteStatus,
+    pub meta: [u8],
+}
+
+impl<AF: AddressFamily + std::fmt::Display> std::fmt::Display
+    for ZeroCopyRecord<AF>
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mui = self.multi_uniq_id;
+        let ltime = self.ltime;
+        write!(
+            f,
+            "{{ mui: {}, ltime: {}, status: {}, meta: {:?} }}",
+            mui, ltime, self.status, &self.meta
+        )
+    }
+}
+
+#[derive(KnownLayout, Immutable, Unaligned, IntoBytes, TryFromBytes)]
+#[repr(C, packed)]
+pub struct ValueHeader {
+    pub ltime: u64,
+    pub status: RouteStatus,
+}
+
+impl std::fmt::Display for ValueHeader {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let ltime = self.ltime;
+        write!(f, "{{ ltime: {}, status: {} }}", ltime, self.status,)
+    }
+}
+
 //------------ PublicPrefixRecord --------------------------------------------
 
 #[derive(Clone, Debug)]
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index a9ab37f3..906b4022 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -22,13 +22,17 @@ pub mod multi {
     };
     pub use crate::local_array::in_memory::iterators;
     pub use crate::local_array::in_memory::node::StrideNodeId;
+    pub use crate::local_array::persist::lsm_tree::KeySize;
+    pub use crate::local_array::rib::rib::{
+        MemoryOnlyConfig, PersistHistoryConfig, PersistOnlyConfig,
+        WriteAheadConfig,
+    };
     pub use crate::local_array::types::{PrefixId, RouteStatus};
     pub use crate::prefix_record::PublicRecord as Record;
 
     pub use crate::rib::Rib;
     pub use crate::rib::{
-        PersistStrategy, StoreConfig, StoreStats, UpsertCounters,
-        UpsertReport,
+        Config, PersistStrategy, StoreStats, UpsertCounters, UpsertReport,
     };
 
     pub use routecore::bgp::path_selection::TiebreakerInfo;
diff --git a/tests/best-path.rs b/tests/best-path.rs
index 4a99cff7..d1d95508 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -1,10 +1,11 @@
 use inetnum::addr::Prefix;
 use inetnum::asn::Asn;
+use log::trace;
 use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::Record;
 use rotonda_store::prelude::multi::RouteStatus;
+use rotonda_store::rib::MemoryOnlyConfig;
 use rotonda_store::rib::PersistStrategy;
-use rotonda_store::rib::StoreConfig;
 use rotonda_store::IncludeHistory;
 use rotonda_store::MatchOptions;
 use rotonda_store::Meta;
@@ -74,17 +75,13 @@ fn test_best_path_1(// tree_bitmap: MultiThreadedStore<Ipv4Route>,
 ) -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
-    let store_config = StoreConfig {
-        persist_strategy: PersistStrategy::MemoryOnly,
-        persist_path: "/tmp/rotonda/".into(),
-    };
-
     let tree_bitmap =
         std::sync::Arc::new(std::sync::Arc::new(MultiThreadedStore::<
             Ipv4Route,
-        >::new_with_config(
-            store_config
-        )?));
+            MemoryOnlyConfig,
+        >::try_default()?));
+
+    trace!("Done creating tree...");
 
     let pfx = Prefix::from_str("185.34.0.0/16")?;
     let mut asns = [
@@ -175,8 +172,11 @@ fn test_best_path_1(// tree_bitmap: MultiThreadedStore<Ipv4Route>,
             Ipv4Route(mui, pa_map.clone(), tbi),
         );
         tree_bitmap.insert(&pfx, rec, None)?;
+        trace!("inserted {}", pfx);
     }
 
+    trace!("done inserting prefixes...");
+
     let res = tree_bitmap.match_prefix(
         &pfx,
         &MatchOptions {
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index 825106f5..911a85ce 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -2,9 +2,11 @@ use std::{str::FromStr, sync::atomic::Ordering};
 
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
-    meta_examples::NoMeta, prelude::multi::RouteStatus, rib::StoreConfig,
-    test_types::BeBytesAsn, IncludeHistory, MatchOptions, MultiThreadedStore,
-    PublicRecord as Record,
+    meta_examples::NoMeta,
+    prelude::multi::RouteStatus,
+    rib::{Config, MemoryOnlyConfig},
+    test_types::BeBytesAsn,
+    IncludeHistory, MatchOptions, MultiThreadedStore, PublicRecord as Record,
 };
 
 mod common {
@@ -24,8 +26,8 @@ rotonda_store::all_strategies![
     BeBytesAsn
 ];
 
-fn test_concurrent_updates_1(
-    tree_bitmap: MultiThreadedStore<BeBytesAsn>,
+fn test_concurrent_updates_1<C: Config + Sync + Send + 'static>(
+    tree_bitmap: MultiThreadedStore<BeBytesAsn, C>,
 ) -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
@@ -415,13 +417,10 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
 
     let cur_ltime = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
 
-    let store_config = StoreConfig {
-        persist_strategy: rotonda_store::rib::PersistStrategy::MemoryOnly,
-        persist_path: "/tmp/rotonda/".into(),
-    };
-    let tree_bitmap = std::sync::Arc::new(
-        MultiThreadedStore::<BeBytesAsn>::new_with_config(store_config)?,
-    );
+    let tree_bitmap = std::sync::Arc::new(MultiThreadedStore::<
+        BeBytesAsn,
+        MemoryOnlyConfig,
+    >::try_default()?);
     let guard = &rotonda_store::epoch::pin();
 
     let _: Vec<_> =
@@ -735,13 +734,10 @@ fn more_specifics_short_lengths() -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
     println!("PersistOnly strategy starting...");
-    let store_config = StoreConfig {
-        persist_strategy: rotonda_store::rib::PersistStrategy::PersistOnly,
-        persist_path: "/tmp/rotonda/".into(),
-    };
-    let tree_bitmap = std::sync::Arc::new(
-        MultiThreadedStore::<NoMeta>::new_with_config(store_config)?,
-    );
+    let tree_bitmap = std::sync::Arc::new(MultiThreadedStore::<
+        NoMeta,
+        MemoryOnlyConfig,
+    >::try_default()?);
     let match_options = MatchOptions {
         match_type: rotonda_store::MatchType::EmptyMatch,
         include_withdrawn: false,
diff --git a/tests/full-table.rs b/tests/full-table.rs
index 27c4feeb..2a3b8950 100644
--- a/tests/full-table.rs
+++ b/tests/full-table.rs
@@ -59,8 +59,8 @@ mod tests {
     ];
 
     // #[test]
-    fn test_full_table_from_csv(
-        tree_bitmap: MultiThreadedStore<AsnList>,
+    fn test_full_table_from_csv<C: Config>(
+        tree_bitmap: MultiThreadedStore<AsnList, C>,
     ) -> Result<(), Box<dyn Error>> {
         // These constants are all contingent on the exact csv file,
         // being loaded!
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index a7d60812..e30700d5 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -5,7 +5,9 @@ use std::error::Error;
 use inetnum::addr::Prefix;
 
 use rotonda_store::{
-    meta_examples::PrefixAs, prelude::multi::RouteStatus, rib::StoreConfig,
+    meta_examples::PrefixAs,
+    prelude::multi::RouteStatus,
+    rib::{Config, MemoryOnlyConfig},
     IncludeHistory, MatchOptions, MatchType, MultiThreadedStore,
     PublicRecord as Record,
 };
@@ -27,8 +29,8 @@ rotonda_store::all_strategies![
 ];
 
 // #[test]
-fn test_more_specifics_without_less_specifics(
-    tree_bitmap: MultiThreadedStore<PrefixAs>,
+fn test_more_specifics_without_less_specifics<C: Config>(
+    tree_bitmap: MultiThreadedStore<PrefixAs, C>,
 ) -> Result<(), Box<dyn Error>> {
     crate::common::init();
 
@@ -127,7 +129,8 @@ fn test_more_specifics_without_less_specifics(
 fn test_more_specifics_with_less_specifics() -> Result<(), Box<dyn Error>> {
     crate::common::init();
 
-    let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
+    let tree_bitmap =
+        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18), // 0
         Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24), // 1
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 1dc6c050..785e7c91 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -23,8 +23,8 @@ rotonda_store::all_strategies![
 ];
 
 // #[test]
-fn test_more_specifics(
-    tree_bitmap: MultiThreadedStore<PrefixAs>,
+fn test_more_specifics<C: Config>(
+    tree_bitmap: MultiThreadedStore<PrefixAs, C>,
 ) -> Result<(), Box<dyn Error>> {
     crate::common::init();
 
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 7df45241..36e0eaae 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -27,8 +27,8 @@ mod tests {
     ];
 
     // #[test]
-    fn test_insert_extremes_ipv4(
-        trie: MultiThreadedStore<NoMeta>,
+    fn test_insert_extremes_ipv4<C: Config>(
+        trie: MultiThreadedStore<NoMeta, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         let min_pfx = Prefix::new_relaxed(
             std::net::Ipv4Addr::new(0, 0, 0, 0).into(),
@@ -105,8 +105,8 @@ mod tests {
     ];
 
     // #[test]
-    fn test_tree_ipv4(
-        tree_bitmap: MultiThreadedStore<PrefixAs>,
+    fn test_tree_ipv4<C: Config>(
+        tree_bitmap: MultiThreadedStore<PrefixAs, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
@@ -415,98 +415,95 @@ mod tests {
         Ok(())
     }
 
-    rotonda_store::all_strategies![
-        ranges_ipv4;
-        test_ranges_ipv4;
-        NoMeta
-    ];
+    // rotonda_store::all_strategies![
+    //     ranges_ipv4;
+    //     test_ranges_ipv4;
+    //     NoMeta
+    // ];
 
-    // #[test]
-    fn test_ranges_ipv4(
-        tree_bitmap: MultiThreadedStore<NoMeta>,
+    #[test]
+    fn test_ranges_ipv4(// tree_bitmap: MultiThreadedStore<NoMeta>,
     ) -> Result<(), Box<dyn std::error::Error>> {
-        // for persist_strategy in [
-        //     PersistStrategy::MemoryOnly,
-        //     PersistStrategy::PersistOnly,
-        //     PersistStrategy::WriteAhead,
-        //     PersistStrategy::PersistHistory,
-        // ] {
-        for i_net in 0..255 {
-            // let config = StoreConfig {
-            //     persist_strategy,
-            //     persist_path: "/tmp/rotonda".into(),
-            // };
-            // let tree_bitmap =
-            //     MultiThreadedStore::<NoMeta>::new_with_config(config)?;
-
-            let pfx_vec: Vec<Prefix> = (1..32)
-                .collect::<Vec<u8>>()
-                .into_iter()
-                .map(|i_len| {
-                    Prefix::new_relaxed(
-                        std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                        i_len,
-                    )
-                    .unwrap()
-                })
-                .collect();
-
-            let mut i_len_s = 0;
-            for pfx in pfx_vec {
-                i_len_s += 1;
-                tree_bitmap.insert(
-                    &pfx,
-                    Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
-                    None,
-                )?;
-
-                let res_pfx = Prefix::new_relaxed(
-                    std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                    i_len_s,
-                );
-
-                let guard = &epoch::pin();
-                for s_len in i_len_s..32 {
-                    let pfx = Prefix::new_relaxed(
-                        std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                        s_len,
-                    )?;
-                    let res = tree_bitmap.match_prefix(
+        for persist_strategy in [
+            PersistStrategy::MemoryOnly,
+            // PersistStrategy::PersistOnly,
+            // PersistStrategy::WriteAhead,
+            // PersistStrategy::PersistHistory,
+        ] {
+            for i_net in 0..255 {
+                let tree_bitmap = MultiThreadedStore::<
+                    NoMeta,
+                    MemoryOnlyConfig,
+                >::try_default()?;
+
+                let pfx_vec: Vec<Prefix> = (1..32)
+                    .collect::<Vec<u8>>()
+                    .into_iter()
+                    .map(|i_len| {
+                        Prefix::new_relaxed(
+                            std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
+                            i_len,
+                        )
+                        .unwrap()
+                    })
+                    .collect();
+
+                let mut i_len_s = 0;
+                for pfx in pfx_vec {
+                    i_len_s += 1;
+                    tree_bitmap.insert(
                         &pfx,
-                        &MatchOptions {
-                            match_type: MatchType::LongestMatch,
-                            include_withdrawn: false,
-                            include_less_specifics: false,
-                            include_more_specifics: false,
-                            mui: None,
-                            include_history: IncludeHistory::None,
-                        },
-                        guard,
+                        Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
+                        None,
+                    )?;
+
+                    let res_pfx = Prefix::new_relaxed(
+                        std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
+                        i_len_s,
                     );
-                    println!("{:?}", pfx);
 
-                    assert_eq!(res.prefix.unwrap(), res_pfx?);
+                    let guard = &epoch::pin();
+                    for s_len in i_len_s..32 {
+                        let pfx = Prefix::new_relaxed(
+                            std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
+                            s_len,
+                        )?;
+                        let res = tree_bitmap.match_prefix(
+                            &pfx,
+                            &MatchOptions {
+                                match_type: MatchType::LongestMatch,
+                                include_withdrawn: false,
+                                include_less_specifics: false,
+                                include_more_specifics: false,
+                                mui: None,
+                                include_history: IncludeHistory::None,
+                            },
+                            guard,
+                        );
+                        println!("{:?}", pfx);
+
+                        assert_eq!(res.prefix.unwrap(), res_pfx?);
+                    }
                 }
             }
-            // }
         }
 
         Ok(())
     }
 
-    rotonda_store::all_strategies![
-        multi_ranges;
-        test_multi_ranges_ipv4;
-        NoMeta
-    ];
+    // rotonda_store::all_strategies![
+    //     multi_ranges;
+    //     test_multi_ranges_ipv4;
+    //     NoMeta
+    // ];
 
-    // #[test]
-    fn test_multi_ranges_ipv4(
-        tree_bitmap: MultiThreadedStore<NoMeta>,
+    #[test]
+    fn test_multi_ranges_ipv4(// tree_bitmap: MultiThreadedStore<NoMeta>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        // let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
+        let tree_bitmap =
+            MultiThreadedStore::<NoMeta, MemoryOnlyConfig>::try_default()?;
         for mui in [1_u32, 2, 3, 4, 5] {
             println!("Multi Uniq ID {mui}");
 
diff --git a/tests/treebitmap_v6.rs b/tests/treebitmap_v6.rs
index 6c90a019..43dd9e9d 100644
--- a/tests/treebitmap_v6.rs
+++ b/tests/treebitmap_v6.rs
@@ -24,8 +24,8 @@ mod tests {
     ];
 
     // #[test]
-    fn test_arbitrary_insert_ipv6(
-        trie: MultiThreadedStore<NoMeta>,
+    fn test_arbitrary_insert_ipv6<C: Config>(
+        trie: MultiThreadedStore<NoMeta, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
         // let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
@@ -73,8 +73,8 @@ mod tests {
     ];
 
     // #[test]
-    fn test_insert_extremes_ipv6(
-        trie: MultiThreadedStore<NoMeta>,
+    fn test_insert_extremes_ipv6<C: Config>(
+        trie: MultiThreadedStore<NoMeta, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
@@ -167,8 +167,8 @@ mod tests {
     // level mapping, most notably if the exit condition is met (a zero at
     // the end of a prefix-length array).
     // #[test]
-    fn test_max_levels(
-        tree_bitmap: MultiThreadedStore<PrefixAs>,
+    fn test_max_levels<C: Config>(
+        tree_bitmap: MultiThreadedStore<PrefixAs, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
@@ -350,8 +350,8 @@ mod tests {
     ];
 
     // #[test]
-    fn test_tree_ipv6(
-        tree_bitmap: MultiThreadedStore<PrefixAs>,
+    fn test_tree_ipv6<C: Config>(
+        tree_bitmap: MultiThreadedStore<PrefixAs, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         // let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
@@ -670,18 +670,19 @@ mod tests {
         Ok(())
     }
 
-    rotonda_store::all_strategies![
-        ranges_ipv4;
-        test_ranges_ipv4;
-        NoMeta
-    ];
+    // rotonda_store::all_strategies![
+    //     ranges_ipv4;
+    //     test_ranges_ipv4;
+    //     NoMeta
+    // ];
 
-    // #[test]
-    fn test_ranges_ipv4(
-        tree_bitmap: MultiThreadedStore<NoMeta>,
+    #[test]
+    fn test_ranges_ipv4(// tree_bitmap: MultiThreadedStore<NoMeta>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         for i_net in 0..255 {
-            // let tree_bitmap = MultiThreadedStore::<NoMeta>::try_default()?;
+            let tree_bitmap =
+                MultiThreadedStore::<NoMeta, MemoryOnlyConfig>::try_default(
+                )?;
 
             let pfx_vec: Vec<Prefix> = (1..32)
                 .collect::<Vec<u8>>()

From 9e3a7c09c7bf2c3d197bafc28cc4189afd104602 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 25 Feb 2025 18:21:30 +0100
Subject: [PATCH 079/147] turn Some empty Vec into None for
 get_records_for_prefix in persist

---
 src/local_array/persist/lsm_tree.rs | 83 ++++++++++++++++++++++++-----
 src/local_array/query.rs            |  1 -
 2 files changed, 70 insertions(+), 14 deletions(-)

diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 8922e1dd..02c7e03c 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -3,6 +3,7 @@
 use std::marker::PhantomData;
 use std::path::Path;
 
+use inetnum::addr::Prefix;
 use log::trace;
 use lsm_tree::{AbstractTree, KvPair};
 use roaring::RoaringBitmap;
@@ -269,7 +270,7 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         withdrawn_muis_bmin: &RoaringBitmap,
-    ) -> lsm_tree::Result<Vec<Vec<u8>>> {
+    ) -> Option<Vec<Vec<u8>>> {
         match (mui, include_withdrawn) {
             // Specific mui, include withdrawn routes
             (Some(mui), true) => {
@@ -297,9 +298,19 @@ impl<
                             bytes
                         })
                     })
-                    .collect::<Vec<_>>()
+                    .collect::<Vec<lsm_tree::Result<Vec<u8>>>>()
                     .into_iter()
-                    .collect()
+                    .collect::<lsm_tree::Result<Vec<Vec<u8>>>>()
+                    .ok()
+                    .and_then(
+                        |recs| {
+                            if recs.is_empty() {
+                                None
+                            } else {
+                                Some(recs)
+                            }
+                        },
+                    )
             }
             // Al muis, include withdrawn routes
             (None, true) => {
@@ -340,9 +351,19 @@ impl<
                             bytes
                         })
                     })
-                    .collect::<Vec<_>>()
+                    .collect::<Vec<lsm_tree::Result<Vec<u8>>>>()
                     .into_iter()
-                    .collect()
+                    .collect::<lsm_tree::Result<Vec<Vec<u8>>>>()
+                    .ok()
+                    .and_then(
+                        |recs| {
+                            if recs.is_empty() {
+                                None
+                            } else {
+                                Some(recs)
+                            }
+                        },
+                    )
             }
             // All muis, exclude withdrawn routes
             (None, false) => {
@@ -359,22 +380,48 @@ impl<
                             //         bytes.as_mut_bytes(),
                             //     )
                             //     .unwrap();
-                            let key = K::header_mut(&mut bytes[..KEY_SIZE]);
+                            let header =
+                                K::header_mut(&mut bytes[..KEY_SIZE]);
                             // If mui is in the global withdrawn muis table,
                             // then skip this record
-                            if key.status == RouteStatus::Withdrawn
+                            trace!("header {}", Prefix::from(header.prefix));
+                            trace!(
+                                "status {}",
+                                header.status == RouteStatus::Withdrawn
+                            );
+                            if header.status == RouteStatus::Withdrawn
                                 || withdrawn_muis_bmin
-                                    .contains(key.mui.into())
+                                    .contains(header.mui.into())
                             {
+                                trace!(
+                                    "NOT returning {} {}",
+                                    Prefix::from(header.prefix),
+                                    header.mui
+                                );
                                 return None;
                             }
+                            trace!(
+                                "RETURNING {} {}",
+                                Prefix::from(header.prefix),
+                                header.mui
+                            );
                             Some(bytes)
                         })
                         .transpose()
                     })
-                    .collect::<Vec<_>>()
+                    .collect::<Vec<lsm_tree::Result<Vec<u8>>>>()
                     .into_iter()
-                    .collect()
+                    .collect::<lsm_tree::Result<Vec<Vec<u8>>>>()
+                    .ok()
+                    .and_then(
+                        |recs| {
+                            if recs.is_empty() {
+                                None
+                            } else {
+                                Some(recs)
+                            }
+                        },
+                    )
             }
             // Specific mui, exclude withdrawn routes
             (Some(mui), false) => {
@@ -416,9 +463,19 @@ impl<
                         })
                         .transpose()
                     })
-                    .collect::<Vec<_>>()
+                    .collect::<Vec<lsm_tree::Result<Vec<u8>>>>()
                     .into_iter()
-                    .collect()
+                    .collect::<lsm_tree::Result<Vec<Vec<u8>>>>()
+                    .ok()
+                    .and_then(
+                        |recs| {
+                            if recs.is_empty() {
+                                None
+                            } else {
+                                Some(recs)
+                            }
+                        },
+                    )
             }
         }
 
@@ -809,7 +866,7 @@ impl<
         header: ValueHeader,
         record_b: &[u8],
     ) {
-        let record = ZeroCopyRecord::<AF>::try_ref_from_prefix(&record_b)
+        let record = ZeroCopyRecord::<AF>::try_ref_from_prefix(record_b)
             .unwrap()
             .0;
         let key = ShortKey::from((record.prefix, record.multi_uniq_id));
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 93012a20..3c289a7a 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -66,7 +66,6 @@ where
                             })
                             .collect::<Vec<_>>()
                     })
-                    .ok()
                 })
             }
             _ => self.prefix_cht.get_records_for_prefix(

From 7140493327d3a28387522c14b6209a6705923266 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 25 Feb 2025 18:25:16 +0100
Subject: [PATCH 080/147] all tests for all strategies

---
 tests/more-more-specifics.rs |  15 +++-
 tests/treebitmap.rs          | 160 ++++++++++++++++++-----------------
 2 files changed, 93 insertions(+), 82 deletions(-)

diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index e30700d5..99b7da4b 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -125,12 +125,19 @@ fn test_more_specifics_without_less_specifics<C: Config>(
     Ok(())
 }
 
-#[test]
-fn test_more_specifics_with_less_specifics() -> Result<(), Box<dyn Error>> {
+rotonda_store::all_strategies![
+    test_ms_w_ls_2;
+    test_more_specifics_with_less_specifics;
+    PrefixAs
+];
+
+fn test_more_specifics_with_less_specifics<C: Config>(
+    tree_bitmap: MultiThreadedStore<PrefixAs, C>,
+) -> Result<(), Box<dyn Error>> {
     crate::common::init();
 
-    let tree_bitmap =
-        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
+    // let tree_bitmap =
+    //     MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18), // 0
         Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24), // 1
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 36e0eaae..5b38256e 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -14,6 +14,7 @@ mod tests {
     use std::str::FromStr;
 
     use inetnum::addr::Prefix;
+    use log::trace;
     use rotonda_store::{
         meta_examples::{NoMeta, PrefixAs},
         prelude::multi::*,
@@ -415,75 +416,72 @@ mod tests {
         Ok(())
     }
 
-    // rotonda_store::all_strategies![
-    //     ranges_ipv4;
-    //     test_ranges_ipv4;
-    //     NoMeta
-    // ];
+    rotonda_store::all_strategies![
+        ranges_ipv4;
+        test_ranges_ipv4;
+        NoMeta
+    ];
 
-    #[test]
-    fn test_ranges_ipv4(// tree_bitmap: MultiThreadedStore<NoMeta>,
+    // #[test]
+    fn test_ranges_ipv4<C: Config>(
+        _tree_bitmap: MultiThreadedStore<NoMeta, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
-        for persist_strategy in [
-            PersistStrategy::MemoryOnly,
-            // PersistStrategy::PersistOnly,
-            // PersistStrategy::WriteAhead,
-            // PersistStrategy::PersistHistory,
-        ] {
-            for i_net in 0..255 {
-                let tree_bitmap = MultiThreadedStore::<
-                    NoMeta,
-                    MemoryOnlyConfig,
-                >::try_default()?;
-
-                let pfx_vec: Vec<Prefix> = (1..32)
-                    .collect::<Vec<u8>>()
-                    .into_iter()
-                    .map(|i_len| {
-                        Prefix::new_relaxed(
-                            std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                            i_len,
-                        )
-                        .unwrap()
-                    })
-                    .collect();
-
-                let mut i_len_s = 0;
-                for pfx in pfx_vec {
-                    i_len_s += 1;
-                    tree_bitmap.insert(
-                        &pfx,
-                        Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
-                        None,
-                    )?;
-
-                    let res_pfx = Prefix::new_relaxed(
+        // for persist_strategy in [
+        //     PersistStrategy::MemoryOnly,
+        //     // PersistStrategy::PersistOnly,
+        //     // PersistStrategy::WriteAhead,
+        //     // PersistStrategy::PersistHistory,
+
+        for i_net in 0..255 {
+            let tree_bitmap = MultiThreadedStore::<NoMeta, C>::try_default()?;
+
+            let pfx_vec: Vec<Prefix> = (1..32)
+                .collect::<Vec<u8>>()
+                .into_iter()
+                .map(|i_len| {
+                    Prefix::new_relaxed(
                         std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                        i_len_s,
+                        i_len,
+                    )
+                    .unwrap()
+                })
+                .collect();
+
+            let mut i_len_s = 0;
+            for pfx in pfx_vec {
+                i_len_s += 1;
+                tree_bitmap.insert(
+                    &pfx,
+                    Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
+                    None,
+                )?;
+
+                let res_pfx = Prefix::new_relaxed(
+                    std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
+                    i_len_s,
+                );
+
+                let guard = &epoch::pin();
+                for s_len in i_len_s..32 {
+                    let pfx = Prefix::new_relaxed(
+                        std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
+                        s_len,
+                    )?;
+                    let res = tree_bitmap.match_prefix(
+                        &pfx,
+                        &MatchOptions {
+                            match_type: MatchType::LongestMatch,
+                            include_withdrawn: false,
+                            include_less_specifics: false,
+                            include_more_specifics: false,
+                            mui: None,
+                            include_history: IncludeHistory::None,
+                        },
+                        guard,
                     );
+                    println!("{:?}", pfx);
 
-                    let guard = &epoch::pin();
-                    for s_len in i_len_s..32 {
-                        let pfx = Prefix::new_relaxed(
-                            std::net::Ipv4Addr::new(i_net, 0, 0, 0).into(),
-                            s_len,
-                        )?;
-                        let res = tree_bitmap.match_prefix(
-                            &pfx,
-                            &MatchOptions {
-                                match_type: MatchType::LongestMatch,
-                                include_withdrawn: false,
-                                include_less_specifics: false,
-                                include_more_specifics: false,
-                                mui: None,
-                                include_history: IncludeHistory::None,
-                            },
-                            guard,
-                        );
-                        println!("{:?}", pfx);
-
-                        assert_eq!(res.prefix.unwrap(), res_pfx?);
-                    }
+                    assert_eq!(res.prefix.unwrap(), res_pfx?);
                 }
             }
         }
@@ -491,19 +489,20 @@ mod tests {
         Ok(())
     }
 
-    // rotonda_store::all_strategies![
-    //     multi_ranges;
-    //     test_multi_ranges_ipv4;
-    //     NoMeta
-    // ];
+    rotonda_store::all_strategies![
+        multi_ranges;
+        test_multi_ranges_ipv4;
+        NoMeta
+    ];
 
-    #[test]
-    fn test_multi_ranges_ipv4(// tree_bitmap: MultiThreadedStore<NoMeta>,
+    // #[test]
+    fn test_multi_ranges_ipv4<C: Config>(
+        tree_bitmap: MultiThreadedStore<NoMeta, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
-        let tree_bitmap =
-            MultiThreadedStore::<NoMeta, MemoryOnlyConfig>::try_default()?;
+        // let tree_bitmap =
+        //     MultiThreadedStore::<NoMeta, PersistOnlyConfig>::try_default()?;
         for mui in [1_u32, 2, 3, 4, 5] {
             println!("Multi Uniq ID {mui}");
 
@@ -898,9 +897,10 @@ mod tests {
 
         //----------------------
 
+        trace!("less_specifics match w/o withdrawn #4");
         // Change the requested prefix to the more specific from the former
         // queries.
-        let less_specifics = tree_bitmap.match_prefix(
+        let query = tree_bitmap.match_prefix(
             &Prefix::from_str("1.0.0.0/17")?,
             &MatchOptions {
                 match_type: MatchType::ExactMatch,
@@ -913,17 +913,21 @@ mod tests {
             guard,
         );
 
-        println!("less_specifics match w/o withdrawn #4 {}", less_specifics);
+        trace!("{:#?}", query);
 
-        assert_eq!(less_specifics.prefix_meta.len(), 5);
+        assert_eq!(query.prefix_meta.len(), 5);
+
+        let less_specifics = query.less_specifics.unwrap();
 
-        let less_specifics = less_specifics.less_specifics.unwrap();
         // All records for the less specific /16 are withdrawn, so this should
         // be empty.
         assert!(less_specifics.is_empty());
 
         //--------------------
 
+        println!("less_specifics match w/o withdrawn #5");
+
+        trace!("mark {} as active", wd_pfx);
         tree_bitmap.mark_mui_as_active_for_prefix(&wd_pfx, 5, 1)?;
 
         let less_specifics = tree_bitmap.match_prefix(
@@ -938,8 +942,8 @@ mod tests {
             },
             guard,
         );
-        println!("more_specifics match w/o withdrawn #5 {}", less_specifics);
         let less_specifics = less_specifics.less_specifics.unwrap();
+        println!("{:#?}", less_specifics);
 
         assert_eq!(less_specifics.v4.len(), 1);
         let less_specifics = &less_specifics.v4[0];

From 5a86a92cd31bcac34f78435e377cd9d6b6c1a5a6 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 25 Feb 2025 18:42:07 +0100
Subject: [PATCH 081/147] add bruno's test

---
 tests/more-specifics.rs | 46 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 785e7c91..7e8863df 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -257,3 +257,49 @@ fn test_more_specifics<C: Config>(
     }
     Ok(())
 }
+
+rotonda_store::all_strategies![
+    test_b_1;
+    test_brunos_more_specifics;
+    PrefixAs
+];
+
+fn test_brunos_more_specifics<C: Config>(
+    tree_bitmap: MultiThreadedStore<PrefixAs, C>,
+) -> Result<(), Box<dyn Error>> {
+    tree_bitmap.insert(
+        &Prefix::new(std::net::Ipv4Addr::new(168, 181, 224, 0).into(), 22)
+            .unwrap(),
+        Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(666)),
+        None,
+    )?;
+    tree_bitmap.insert(
+        &Prefix::new(std::net::Ipv4Addr::new(168, 181, 120, 0).into(), 24)?,
+        Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(666)),
+        None,
+    )?;
+    tree_bitmap.insert(
+        &Prefix::new(std::net::Ipv4Addr::new(168, 181, 121, 0).into(), 24)
+            .unwrap(),
+        Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(666)),
+        None,
+    )?;
+
+    let guard = &epoch::pin();
+    let found_result = tree_bitmap.match_prefix(
+        &Prefix::new(std::net::Ipv4Addr::new(168, 181, 224, 0).into(), 22)
+            .unwrap(),
+        &MatchOptions {
+            match_type: MatchType::ExactMatch,
+            include_withdrawn: false,
+            include_less_specifics: false,
+            include_more_specifics: true,
+            mui: None,
+            include_history: IncludeHistory::None,
+        },
+        guard,
+    );
+
+    assert!(found_result.more_specifics.unwrap().is_empty());
+    Ok(())
+}

From d4daae2fb44ee56aa910a4eafb9272a158c4fa69 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Feb 2025 10:25:02 +0100
Subject: [PATCH 082/147] cleanup

---
 examples/multi_single_thread.rs            |   4 +-
 examples/real_single_thread_24.rs          |   4 +-
 src/local_array/bit_span.rs                |   2 +-
 src/local_array/in_memory/atomic_stride.rs | 103 +---------------
 src/local_array/in_memory/tree.rs          |   2 +-
 src/local_array/persist/lsm_tree.rs        | 135 +++++----------------
 src/local_array/prefix_cht/iterators.rs    |   2 +-
 src/local_array/rib/default_store.rs       |  41 -------
 src/local_array/rib/macros.rs              |  38 ++----
 src/local_array/rib/rib.rs                 |   5 +-
 src/local_array/types.rs                   |   1 -
 src/local_vec/tree.rs                      |   3 -
 src/meta_examples.rs                       |   2 +-
 tests/best-path.rs                         |   1 -
 tests/more-more-specifics.rs               |   4 +-
 15 files changed, 53 insertions(+), 294 deletions(-)

diff --git a/examples/multi_single_thread.rs b/examples/multi_single_thread.rs
index bf93f82f..5de76bdb 100644
--- a/examples/multi_single_thread.rs
+++ b/examples/multi_single_thread.rs
@@ -33,7 +33,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .name(1_u8.to_string())
         .spawn(move || -> Result<(), Box<dyn std::error::Error + Send>> {
             // while !start_flag.load(std::sync::atomic::Ordering::Acquire) {
-            let mut rng = rand::thread_rng();
+            let mut rng = rand::rng();
 
             println!("park thread {}", 1);
             thread::park();
@@ -44,7 +44,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             loop {
                 // x += 1;
                 // print!("{}-", i);
-                let asn: u32 = rng.gen();
+                let asn: u32 = rng.random();
                 match tree_bitmap.insert(
                     &pfx.unwrap(),
                     Record::new(
diff --git a/examples/real_single_thread_24.rs b/examples/real_single_thread_24.rs
index b0a2b1a3..2ea23533 100644
--- a/examples/real_single_thread_24.rs
+++ b/examples/real_single_thread_24.rs
@@ -23,7 +23,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let thread = std::thread::Builder::new()
         .name(1_u8.to_string())
         .spawn(move || -> Result<(), Box<dyn std::error::Error + Send>> {
-            let mut rng = rand::thread_rng();
+            let mut rng = rand::rng();
 
             println!("park thread {}", 1);
             thread::park();
@@ -35,7 +35,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let pfx = Prefix::new_relaxed(pfx_int.into_ipaddr(), 32);
 
                 print!("{}-", pfx_int);
-                let asn: u32 = rng.gen();
+                let asn: u32 = rng.random();
                 match tree_bitmap
                     .insert(&pfx.unwrap(), PrefixAs::new_from_u32(asn))
                 {
diff --git a/src/local_array/bit_span.rs b/src/local_array/bit_span.rs
index d8b839d3..be1e03f0 100644
--- a/src/local_array/bit_span.rs
+++ b/src/local_array/bit_span.rs
@@ -1,5 +1,5 @@
 #[derive(Copy, Clone, Debug)]
-pub(crate) struct BitSpan {
+pub struct BitSpan {
     pub bits: u32,
     pub len: u8,
 }
diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 52eeaa66..66c94108 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -45,17 +45,12 @@ where
         + num_traits::PrimInt;
 
     fn new() -> Self;
-    fn inner(self) -> Self::InnerType;
-    fn is_set(&self, index: usize) -> bool;
     fn compare_exchange(
         &self,
         current: Self::InnerType,
         new: Self::InnerType,
     ) -> CasResult<Self::InnerType>;
     fn load(&self) -> Self::InnerType;
-    fn to_u64(&self) -> u64;
-    fn to_u32(&self) -> u32;
-    fn set(&self, value: Self::InnerType);
     fn merge_with(&self, node: Self::InnerType) {
         let mut spinwait = SpinWait::new();
         let current = self.load();
@@ -83,12 +78,7 @@ impl AtomicBitmap for AtomicStride2 {
     fn new() -> Self {
         AtomicStride2(AtomicU8::new(0))
     }
-    fn inner(self) -> Self::InnerType {
-        self.0.into_inner()
-    }
-    fn is_set(&self, bit: usize) -> bool {
-        self.load() & (1 << bit) != 0
-    }
+
     fn compare_exchange(
         &self,
         current: Self::InnerType,
@@ -101,21 +91,10 @@ impl AtomicBitmap for AtomicStride2 {
             Ordering::Relaxed,
         ))
     }
+
     fn load(&self) -> Self::InnerType {
         self.0.load(Ordering::SeqCst)
     }
-
-    fn set(&self, value: Self::InnerType) {
-        self.0.store(value, Ordering::Relaxed);
-    }
-
-    fn to_u32(&self) -> u32 {
-        self.0.load(Ordering::SeqCst) as u32
-    }
-
-    fn to_u64(&self) -> u64 {
-        self.0.load(Ordering::SeqCst) as u64
-    }
 }
 
 impl Zero for AtomicStride2 {
@@ -140,12 +119,6 @@ impl AtomicBitmap for AtomicStride3 {
     fn new() -> Self {
         AtomicStride3(AtomicU16::new(0))
     }
-    fn inner(self) -> Self::InnerType {
-        self.0.into_inner()
-    }
-    fn is_set(&self, bit: usize) -> bool {
-        self.load() & (1 << bit) != 0
-    }
     fn compare_exchange(
         &self,
         current: Self::InnerType,
@@ -162,18 +135,6 @@ impl AtomicBitmap for AtomicStride3 {
     fn load(&self) -> Self::InnerType {
         self.0.load(Ordering::Relaxed)
     }
-
-    fn set(&self, value: Self::InnerType) {
-        self.0.store(value, Ordering::Relaxed);
-    }
-
-    fn to_u32(&self) -> u32 {
-        self.0.load(Ordering::Relaxed) as u32
-    }
-
-    fn to_u64(&self) -> u64 {
-        self.0.load(Ordering::Relaxed) as u64
-    }
 }
 
 impl From<u16> for AtomicStride3 {
@@ -198,12 +159,6 @@ impl AtomicBitmap for AtomicStride4 {
     fn new() -> Self {
         AtomicStride4(AtomicU32::new(0))
     }
-    fn inner(self) -> Self::InnerType {
-        self.0.into_inner()
-    }
-    fn is_set(&self, bit: usize) -> bool {
-        self.load() & (1 << bit) != 0
-    }
     fn compare_exchange(
         &self,
         current: Self::InnerType,
@@ -219,18 +174,6 @@ impl AtomicBitmap for AtomicStride4 {
     fn load(&self) -> Self::InnerType {
         self.0.load(Ordering::Relaxed)
     }
-
-    fn set(&self, value: Self::InnerType) {
-        self.0.store(value, Ordering::Relaxed);
-    }
-
-    fn to_u32(&self) -> u32 {
-        self.0.load(Ordering::Relaxed)
-    }
-
-    fn to_u64(&self) -> u64 {
-        self.0.load(Ordering::Relaxed) as u64
-    }
 }
 
 impl From<u32> for AtomicStride4 {
@@ -254,12 +197,6 @@ impl AtomicBitmap for AtomicStride5 {
     fn new() -> Self {
         AtomicStride5(AtomicU64::new(0))
     }
-    fn inner(self) -> Self::InnerType {
-        self.0.into_inner()
-    }
-    fn is_set(&self, bit: usize) -> bool {
-        self.load() & (1 << bit) != 0
-    }
     fn compare_exchange(
         &self,
         current: Self::InnerType,
@@ -275,18 +212,6 @@ impl AtomicBitmap for AtomicStride5 {
     fn load(&self) -> Self::InnerType {
         self.0.load(Ordering::SeqCst)
     }
-
-    fn set(&self, value: Self::InnerType) {
-        self.0.store(value, Ordering::Relaxed);
-    }
-
-    fn to_u32(&self) -> u32 {
-        self.0.load(Ordering::SeqCst) as u32
-    }
-
-    fn to_u64(&self) -> u64 {
-        self.0.load(Ordering::SeqCst)
-    }
 }
 
 impl From<u64> for AtomicStride5 {
@@ -311,18 +236,6 @@ impl AtomicBitmap for AtomicStride6 {
     fn new() -> Self {
         AtomicStride6(AtomicU128::new(0))
     }
-    fn inner(self) -> Self::InnerType {
-        let hi = self.0 .0.into_inner().to_be_bytes();
-        let lo = self.0 .1.into_inner().to_be_bytes();
-
-        u128::from_be_bytes([
-            hi[0], hi[1], hi[2], hi[3], hi[4], hi[5], hi[6], hi[7], lo[0],
-            lo[1], lo[2], lo[3], lo[4], lo[5], lo[6], lo[7],
-        ])
-    }
-    fn is_set(&self, bit: usize) -> bool {
-        self.load() & (1 << bit) != 0
-    }
     fn compare_exchange(
         &self,
         current: Self::InnerType,
@@ -357,18 +270,6 @@ impl AtomicBitmap for AtomicStride6 {
         ])
     }
 
-    fn set(&self, _value: Self::InnerType) {
-        todo!()
-    }
-
-    fn to_u32(&self) -> u32 {
-        unimplemented!()
-    }
-
-    fn to_u64(&self) -> u64 {
-        unimplemented!()
-    }
-
     fn merge_with(&self, _node: Self::InnerType) {
         todo!()
     }
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index a77c158a..e9a58962 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -197,7 +197,7 @@ use std::{fmt::Debug, marker::PhantomData};
 
 use super::atomic_types::NodeBuckets;
 use crate::af::AddressFamily;
-use crate::rib::{Counters, UpsertReport};
+use crate::rib::Counters;
 use crate::{
     impl_search_level, impl_search_level_for_mui, insert_match,
     retrieve_node_mut_closure, store_node_closure,
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 02c7e03c..7c679993 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -43,17 +43,6 @@ pub trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
         Self::try_ref_from_bytes(bytes.as_bytes())
     }
 
-    // fn as_key_size_bytes(&self) -> [u8; KEY_SIZE] {
-    //     *self.as_bytes().first_chunk::<KEY_SIZE>().unwrap()
-    // }
-
-    fn new_write_key(
-        prefix_id: PrefixId<AF>,
-        mui: u32,
-        ltime: u64,
-        status: RouteStatus,
-    ) -> [u8; KEY_SIZE];
-
     fn header(bytes: &[u8]) -> &LongKey<AF> {
         LongKey::try_ref_from_bytes(bytes.as_bytes()).unwrap()
     }
@@ -70,76 +59,6 @@ pub trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
         trace!("short key {:?}", s_b);
         ShortKey::try_ref_from_prefix(bytes).unwrap().0
     }
-    // fn persistence_key(
-    //     // PREFIX_SIZE bytes
-    //     prefix_id: PrefixId<AF>,
-    //     // 4 bytes
-    //     mui: u32,
-    //     // 8 bytes
-    //     ltime: u64,
-    //     // 1 byte
-    //     status: RouteStatus,
-    // ) -> [u8; KEY_SIZE] {
-    //     assert!(KEY_SIZE > PREFIX_SIZE);
-    //     let key = &mut [0_u8; KEY_SIZE];
-
-    //     // prefix 5 or 17 bytes
-    //     *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() =
-    //         prefix_id.to_len_first_bytes();
-
-    //     // mui 4 bytes
-    //     *key[PREFIX_SIZE..PREFIX_SIZE + 4]
-    //         .first_chunk_mut::<4>()
-    //         .unwrap() = mui.to_le_bytes();
-
-    //     // ltime 8 bytes
-    //     *key[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
-    //         .first_chunk_mut::<8>()
-    //         .unwrap() = ltime.to_le_bytes();
-
-    //     // status 1 byte
-    //     key[PREFIX_SIZE + 12] = status.into();
-
-    //     *key
-    // }
-
-    // fn prefix_mui_persistence_key(
-    //     prefix_id: PrefixId<AF>,
-    //     mui: u32,
-    // ) -> Vec<u8> {
-    //     let mut key = vec![0; PREFIX_SIZE + 4];
-    //     // prefix 5 or 17 bytes
-    //     *key.first_chunk_mut::<PREFIX_SIZE>().unwrap() =
-    //         prefix_id.to_len_first_bytes();
-
-    //     // mui 4 bytes
-    //     *key[PREFIX_SIZE..PREFIX_SIZE + 4]
-    //         .first_chunk_mut::<4>()
-    //         .unwrap() = mui.to_le_bytes();
-
-    //     key
-    // }
-
-    // fn parse_key(bytes: &[u8]) -> (PrefixId<AF>, u32, u64, RouteStatus) {
-    //     (
-    //         // prefix 5 or 17 bytes
-    //         PrefixId::from(*bytes.first_chunk::<PREFIX_SIZE>().unwrap()),
-    //         // mui 4 bytes
-    //         u32::from_le_bytes(
-    //             *bytes[PREFIX_SIZE..PREFIX_SIZE + 4]
-    //                 .first_chunk::<4>()
-    //                 .unwrap(),
-    //         ),
-    //         // ltime 8 bytes
-    //         u64::from_le_bytes(
-    //             *bytes[PREFIX_SIZE + 4..PREFIX_SIZE + 12]
-    //                 .first_chunk::<8>()
-    //                 .unwrap(),
-    //         ),
-    //         // status 1 byte
-    //         RouteStatus::try_from(bytes[PREFIX_SIZE + 12]).unwrap(),
-    //     )
-    // }
 }
 
 #[derive(Debug, KnownLayout, Immutable, FromBytes, Unaligned, IntoBytes)]
@@ -161,26 +80,26 @@ pub struct ShortKey<AF: AddressFamily> {
 )]
 #[repr(C)]
 pub struct LongKey<AF: AddressFamily> {
-    prefix: PrefixId<AF>,
-    mui: U32<NativeEndian>,
-    ltime: U64<NativeEndian>,
-    status: RouteStatus,
-}
+    prefix: PrefixId<AF>,     // 4 or 16
+    mui: U32<NativeEndian>,   // 4
+    ltime: U64<NativeEndian>, // 8
+    status: RouteStatus,      // 1
+} // 17 or 29
 
 impl<AF: AddressFamily, const KEY_SIZE: usize> KeySize<AF, KEY_SIZE>
     for ShortKey<AF>
 {
-    fn new_write_key(
-        prefix: PrefixId<AF>,
-        mui: u32,
-        _ltime: u64,
-        _status: RouteStatus,
-    ) -> [u8; KEY_SIZE] {
-        *Self::from((prefix, mui))
-            .as_bytes()
-            .first_chunk::<KEY_SIZE>()
-            .unwrap()
-    }
+    // fn new_write_key(
+    //     prefix: PrefixId<AF>,
+    //     mui: u32,
+    //     _ltime: u64,
+    //     _status: RouteStatus,
+    // ) -> [u8; KEY_SIZE] {
+    //     *Self::from((prefix, mui))
+    //         .as_bytes()
+    //         .first_chunk::<KEY_SIZE>()
+    //         .unwrap()
+    // }
 }
 
 impl<AF: AddressFamily> From<(PrefixId<AF>, u32)> for ShortKey<AF> {
@@ -195,17 +114,17 @@ impl<AF: AddressFamily> From<(PrefixId<AF>, u32)> for ShortKey<AF> {
 impl<AF: AddressFamily, const KEY_SIZE: usize> KeySize<AF, KEY_SIZE>
     for LongKey<AF>
 {
-    fn new_write_key(
-        prefix: PrefixId<AF>,
-        mui: u32,
-        ltime: u64,
-        status: RouteStatus,
-    ) -> [u8; KEY_SIZE] {
-        *Self::from((prefix, mui, ltime, status))
-            .as_bytes()
-            .first_chunk::<KEY_SIZE>()
-            .unwrap()
-    }
+    // fn new_write_key(
+    //     prefix: PrefixId<AF>,
+    //     mui: u32,
+    //     ltime: u64,
+    //     status: RouteStatus,
+    // ) -> [u8; KEY_SIZE] {
+    //     *Self::from((prefix, mui, ltime, status))
+    //         .as_bytes()
+    //         .first_chunk::<KEY_SIZE>()
+    //         .unwrap()
+    // }
 }
 
 impl<AF: AddressFamily> From<(PrefixId<AF>, u32, u64, RouteStatus)>
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
index 2fec77f9..d5d7b12d 100644
--- a/src/local_array/prefix_cht/iterators.rs
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -11,7 +11,7 @@ use crate::{
     AddressFamily,
 };
 
-type Type<AF: AddressFamily> = SizedNodeMoreSpecificIter<AF>;
+type Type<AF> = SizedNodeMoreSpecificIter<AF>;
 
 pub(crate) struct MoreSpecificPrefixIter<
     'a,
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index e52ab0e8..2d0dec2f 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -14,7 +14,6 @@ use rand::prelude::*;
 ))]
 struct DefaultStore;
 
-/// Try some
 impl<M: Meta, C: Config> DefaultStore<M, C> {
     pub fn try_default() -> Result<Self, PrefixStoreError> {
         let config = C::default();
@@ -22,43 +21,3 @@ impl<M: Meta, C: Config> DefaultStore<M, C> {
             .map_err(|_| PrefixStoreError::StoreNotReadyError)
     }
 }
-
-// impl Default for StoreConfig {
-//     fn default() -> Self {
-//         Self {
-//             persist_strategy: PersistStrategy::MemoryOnly,
-//             persist_path: "/tmp/rotonda/".to_string(),
-//         }
-//     }
-// }
-
-// impl StoreConfig {
-//     fn persist_default() -> Self {
-//         Self {
-//             persist_strategy: PersistStrategy::PersistOnly,
-//             persist_path: "/tmp/rotonda/".to_string(),
-//         }
-//     }
-// }
-
-// pub mod persist_only_store {
-//     use crate::local_array::in_memory::atomic_types::NodeSet;
-//     use crate::local_array::persist::lsm_tree::ShortKey;
-//     use crate::prelude::multi::*;
-//     use crate::prelude::*;
-//     use rand::prelude::*;
-//     #[create_store((
-//         ([4], 5, 18, PersistOnlyConfig, ShortKey),
-//         ([4], 17, 30, PersistOnlyconfig, ShortKey)
-//     ))]
-//     struct PersistOnlyStore;
-
-//     /// Try some
-//     impl<M: Meta, C: Config> PersistOnlyStore<M, C> {
-//         pub fn try_default() -> Result<Self, PrefixStoreError> {
-//             let config = C::default();
-//             Self::new_with_config(config)
-//                 .map_err(|_| PrefixStoreError::StoreNotReadyError)
-//         }
-//     }
-// }
diff --git a/src/local_array/rib/macros.rs b/src/local_array/rib/macros.rs
index 52a2b54b..5448b30e 100644
--- a/src/local_array/rib/macros.rs
+++ b/src/local_array/rib/macros.rs
@@ -16,17 +16,11 @@ macro_rules! impl_search_level {
                     // HASHING FUNCTION
                     let index = Self::hash_node_id($id, level);
 
-                    // Read the node from the block pointed to by the Atomic
-                    // pointer.
-                    // let stored_node = unsafe {
-                    //     &mut nodes.0[index]
-                    // };
-                    // let this_node = stored_node.load(Ordering::Acquire, guard);
-
                     match nodes.read().get(index) {
                         None => None,
                         Some(stored_node) => {
-                            let StoredNode { node_id, node, node_set, .. } = stored_node;
+                            let StoredNode {
+                                node_id, node, node_set, .. } = stored_node;
                             if $id == *node_id {
                                 // YES, It's the one we're looking for!
                                 return Some(SizedStrideRef::$stride(&node));
@@ -34,14 +28,15 @@ macro_rules! impl_search_level {
                             // Meh, it's not, but we can a go to the next
                             // level and see if it lives there.
                             level += 1;
-                            match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
+                            match <NB as NodeBuckets<AF>>::len_to_store_bits(
+                                $id.get_id().1, level
+                            ) {
                                 // on to the next level!
                                 next_bit_shift if next_bit_shift > 0 => {
                                     (search_level.f)(
                                         search_level,
                                         &node_set,
                                         level,
-                                        // guard,
                                     )
                                 }
                                 // There's no next level, we found nothing.
@@ -73,23 +68,17 @@ macro_rules! impl_search_level_for_mui {
                     // HASHING FUNCTION
                     let index = Self::hash_node_id($id, level);
 
-                    // Read the node from the block pointed to by the Atomic
-                    // pointer.
-                    // let stored_node = unsafe {
-                    //     &mut nodes.0[index].assume_init_ref()
-                    // };
-                    // let this_node = stored_node.load(Ordering::Acquire, guard);
-
                     match nodes.read().get(index) {
                         None => None,
                         Some(this_node) => {
-                            let StoredNode { node_id, node, node_set, .. } = this_node;
+                            let StoredNode {
+                                node_id, node, node_set, .. } = this_node;
 
                             // early return if the mui is not in the index
                             // stored in this node, meaning the mui does not
                             // appear anywhere in the sub-tree formed from
                             // this node.
-                            let bmin = node_set.rbm().read().unwrap(); // load(Ordering::Acquire, guard).deref()
+                            let bmin = node_set.rbm().read().unwrap();
                             if !bmin.contains($mui) {
                                 return None;
                             }
@@ -108,7 +97,6 @@ macro_rules! impl_search_level_for_mui {
                                         search_level,
                                         &node_set,
                                         level,
-                                        // guard,
                                     )
                                 }
                                 // There's no next level, we found nothing.
@@ -157,11 +145,11 @@ macro_rules! retrieve_node_mut_closure {
 
                 match nodes.read().get(index) {
                     // This arm only ever gets called in multi-threaded code
-                    // where our thread (running this code *now*), andgot ahead
-                    // of another thread: After the other thread created the
-                    // TreeBitMapNode first, it was overtaken by our thread
-                    // running this method, so our thread enounters an empty node
-                    // in the store.
+                    // where our thread (running this code *now*), andgot
+                    // ahead of another thread: After the other thread created
+                    // the TreeBitMapNode first, it was overtaken by our
+                    // thread running this method, so our thread enounters an
+                    // empty node in the store.
                     None => {
                         let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
                             $id.get_id().1, level
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 7dae0d64..a2b0187f 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -10,7 +10,7 @@ use epoch::{Guard, Owned};
 use zerocopy::TryFromBytes;
 
 use crate::local_array::in_memory::tree::TreeBitMap;
-use crate::local_array::persist::lsm_tree::{KeySize, LongKey, ShortKey};
+use crate::local_array::persist::lsm_tree::KeySize;
 use crate::local_array::prefix_cht::cht::PrefixCHT;
 use crate::local_array::types::PrefixId;
 use crate::prefix_record::{ValueHeader, ZeroCopyRecord};
@@ -31,7 +31,6 @@ pub use crate::local_array::query;
 use crate::{IPv4, IPv6, Meta};
 
 use crate::AddressFamily;
-use zerocopy::IntoBytes;
 
 //------------ Config --------------------------------------------------------
 
@@ -673,7 +672,7 @@ impl<
                 // let stored_prefixes = p_tree
                 //     .get_records_with_keys_for_prefix_mui::<M>(prefix, mui)
 
-                if let Some(mut record_b) =
+                if let Some(record_b) =
                     p_tree.get_most_recent_record_for_prefix_mui(prefix, mui)
                 {
                     // let new_key: [u8; KEY_SIZE] = PersistTree::<
diff --git a/src/local_array/types.rs b/src/local_array/types.rs
index 958c195b..ce3e2e4b 100644
--- a/src/local_array/types.rs
+++ b/src/local_array/types.rs
@@ -1,4 +1,3 @@
-use log::trace;
 use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes, Unaligned};
 
 use crate::AddressFamily;
diff --git a/src/local_vec/tree.rs b/src/local_vec/tree.rs
index 9d68e8f3..031bd658 100644
--- a/src/local_vec/tree.rs
+++ b/src/local_vec/tree.rs
@@ -16,9 +16,6 @@ use crate::{
     rib::UpsertReport,
 };
 
-#[cfg(feature = "cli")]
-use crate::node_id::InMemNodeId;
-
 #[cfg(feature = "cli")]
 use ansi_term::Colour;
 
diff --git a/src/meta_examples.rs b/src/meta_examples.rs
index 1828fea2..4cc482ba 100644
--- a/src/meta_examples.rs
+++ b/src/meta_examples.rs
@@ -2,7 +2,7 @@
 
 use inetnum::asn::Asn;
 
-use crate::{Meta, PublicRecord};
+use crate::Meta;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub struct PrefixAs([u8; 4]);
diff --git a/tests/best-path.rs b/tests/best-path.rs
index d1d95508..ef28ab34 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -5,7 +5,6 @@ use rotonda_store::prelude::multi::PrefixStoreError;
 use rotonda_store::prelude::multi::Record;
 use rotonda_store::prelude::multi::RouteStatus;
 use rotonda_store::rib::MemoryOnlyConfig;
-use rotonda_store::rib::PersistStrategy;
 use rotonda_store::IncludeHistory;
 use rotonda_store::MatchOptions;
 use rotonda_store::Meta;
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index 99b7da4b..a3efc793 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -5,9 +5,7 @@ use std::error::Error;
 use inetnum::addr::Prefix;
 
 use rotonda_store::{
-    meta_examples::PrefixAs,
-    prelude::multi::RouteStatus,
-    rib::{Config, MemoryOnlyConfig},
+    meta_examples::PrefixAs, prelude::multi::RouteStatus, rib::Config,
     IncludeHistory, MatchOptions, MatchType, MultiThreadedStore,
     PublicRecord as Record,
 };

From e3918339273be3801f3863b1ba6edab1c0431bcc Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Feb 2025 10:52:43 +0100
Subject: [PATCH 083/147] use dev branch for routecore

---
 Cargo.toml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 5081473f..95262c6c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -31,8 +31,7 @@ inetnum   = "0.1"
 log       = "^0.4"
 roaring 	= "0.10.3"
 rotonda-macros = { path = "proc_macros", version = "0.4.0" }
-routecore       = { version = "0.5", features = ["bgp", "bmp", "fsm", "serde", "mrt"] }
-
+routecore       = { git = "https://github.com/nlnetlabs/routecore", branch = "dev", version = "0.5.2-dev", features = ["bgp", "bmp", "fsm", "serde", "mrt"] }
 ansi_term   = { version = "0.12", optional = true }
 csv         = { version = "1", optional = true }
 rustyline   = { version = "13", optional = true }

From c879c1ee27186da042aea2f121fc27bf02a05dc4 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Feb 2025 13:50:23 +0100
Subject: [PATCH 084/147] prelim SizedStref removal

---
 src/af.rs                                  |  26 +-
 src/lib.rs                                 |   2 +-
 src/local_array/in_memory/atomic_stride.rs |   7 +-
 src/local_array/in_memory/macros.rs        |  36 +-
 src/local_array/in_memory/node.rs          |   9 +-
 src/local_array/in_memory/tree.rs          | 463 ++++++++++++++++++---
 src/local_array/persist/lsm_tree.rs        |   4 +-
 src/local_array/prefix_cht/iterators.rs    |   2 +-
 src/macros.rs                              |   8 +-
 src/rotonda_store.rs                       |   2 +-
 src/stride.rs                              |  21 +-
 11 files changed, 466 insertions(+), 114 deletions(-)

diff --git a/src/af.rs b/src/af.rs
index bd3609e9..a6ffb7ab 100644
--- a/src/af.rs
+++ b/src/af.rs
@@ -54,7 +54,7 @@ pub trait AddressFamily:
     fn fmt_net(net: Self) -> String;
     // returns the specified nibble from `start_bit` to (and including)
     // `start_bit + len` and shifted to the right.
-    fn get_nibble(net: Self, start_bit: u8, len: u8) -> u32;
+    fn into_bit_span(net: Self, start_bit: u8, len: u8) -> BitSpan;
 
     /// Treat self as a prefix and append the given nibble to it.
     fn add_bit_span(self, len: u8, bs: BitSpan) -> (Self, u8);
@@ -111,10 +111,13 @@ impl AddressFamily for IPv4 {
         std::net::Ipv4Addr::from(u32::from(net)).to_string()
     }
 
-    fn get_nibble(net: Self, start_bit: u8, len: u8) -> u32 {
-        ((net << <U32<NetworkEndian>>::from(start_bit as u32))
-            >> <U32<NetworkEndian>>::from(((32 - len) % 32) as u32))
-        .into()
+    fn into_bit_span(net: Self, start_bit: u8, len: u8) -> BitSpan {
+        BitSpan {
+            bits: ((net << <U32<NetworkEndian>>::from(start_bit as u32))
+                >> <U32<NetworkEndian>>::from(((32 - len) % 32) as u32))
+            .into(),
+            len,
+        }
     }
 
     // You can't shift with the number of bits of self, so we'll just return
@@ -242,11 +245,14 @@ impl AddressFamily for IPv6 {
         std::net::Ipv6Addr::from(u128::from(net)).to_string()
     }
 
-    fn get_nibble(net: Self, start_bit: u8, len: u8) -> u32 {
-        u128::from(
-            (net << <U128<NetworkEndian>>::from(start_bit as u128))
-                >> (<U128<NetworkEndian>>::from(128 - len as u128) % 128),
-        ) as u32
+    fn into_bit_span(net: Self, start_bit: u8, len: u8) -> BitSpan {
+        BitSpan {
+            bits: u128::from(
+                (net << <U128<NetworkEndian>>::from(start_bit as u128))
+                    >> (<U128<NetworkEndian>>::from(128 - len as u128) % 128),
+            ) as u32,
+            len,
+        }
     }
 
     /// Treat self as a prefix and append the given nibble to it.
diff --git a/src/lib.rs b/src/lib.rs
index 1ee6b348..740eadb0 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -12,7 +12,7 @@
 //! Read more about the data-structure in this [blog post](https://blog.nlnetlabs.nl/donkeys-mules-horses/).
 mod af;
 mod local_array;
-mod local_vec;
+// mod local_vec;
 mod node_id;
 mod prefix_record;
 mod stride;
diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 66c94108..ff8ce2c3 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -416,8 +416,7 @@ where
     // *both* the bitmap part, we're considering here *and* the position
     // relative to the nibble length offset in the bitmap.
     fn get_bit_pos(
-        nibble: u32,
-        len: u8,
+        bit_pos: BitSpan,
     ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
 
     fn get_bit_pos_as_u8(nibble: u32, len: u8) -> u8;
@@ -432,7 +431,7 @@ where
 
     fn cursor_from_bit_span(bs: BitSpan) -> u8;
 
-    fn ptr_cursor_from_bit_span(bs: BitSpan) -> u8;
+    // fn ptr_cursor_from_bit_span(bs: BitSpan) -> u8;
 
     fn ptr_range(
         ptrbitarr: <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::
@@ -465,7 +464,7 @@ where
 
     // get_pfx_index only needs nibble and len for fixed-layout bitarrays,
     // since the index can be deducted from them.
-    fn get_pfx_index(nibble: u32, len: u8) -> usize;
+    fn get_pfx_index(bit_span: BitSpan) -> usize;
 
     // Clear the bitmap to the right of the pointer and count the number of
     // ones. This number represents the index to the corresponding child node
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index 22a629ca..02f50529 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -7,8 +7,9 @@ macro_rules! insert_match {
     (
         $self: ident;
         $guard: ident;
-        $nibble_len: expr;
-        $nibble: expr; // nibble is a variable-length bitarray (1,2,4,8,etc)
+        $bit_span: expr;
+        // $nibble_len: expr;
+        // $nibble: expr; // nibble is a variable-length bitarray (1,2,4,8,etc)
         $is_last_stride: expr;
         $pfx: ident; // the whole search prefix
         $mui: ident; // the reccord holding the metadata
@@ -55,8 +56,7 @@ macro_rules! insert_match {
                             // eval_node_or_prefix_at mutates the node to
                             // reflect changes in the ptrbitarr & pfxbitarr.
                             match current_node.eval_node_or_prefix_at(
-                                $nibble,
-                                $nibble_len,
+                                $bit_span,
                                 // All the bits of the search prefix, but with
                                 // a length set to the start of the current
                                 // stride.
@@ -80,7 +80,7 @@ macro_rules! insert_match {
                                     let new_id =
                                         StrideNodeId::new_with_cleaned_id(
                                             $pfx.get_net(),
-                                            $truncate_len + $nibble_len
+                                            $truncate_len + $bit_span.len
                                        );
 
                                     // store the new node in the in_memory
@@ -183,11 +183,11 @@ macro_rules! impl_primitive_atomic_stride {
                 const BITS: u8 = $bits;
                 const STRIDE_LEN: u8 = $len;
 
-                fn get_bit_pos(nibble: u32, len: u8) -> $pfxsize {
+                fn get_bit_pos(bs: BitSpan) -> $pfxsize {
                     // trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
                     1 << (
-                            <Self as Stride>::BITS - ((1 << len) - 1) as u8
-                            - nibble as u8 - 1
+                            <Self as Stride>::BITS - ((1 << bs.len) - 1) as u8
+                            - bs.bits as u8 - 1
                     )
                 }
 
@@ -203,7 +203,7 @@ macro_rules! impl_primitive_atomic_stride {
                 }
 
                 fn cursor_from_bit_span(bs: BitSpan) -> u8 {
-                    Self::get_bit_pos(bs.bits, bs.len)
+                    Self::get_bit_pos(bs)
                     .leading_zeros() as u8
                 }
 
@@ -242,13 +242,13 @@ macro_rules! impl_primitive_atomic_stride {
                 // Ptrbitarr searches are only done in the last half of
                 // the bitarray, in the len = S::STRIDE_LEN part. We need a
                 // complete BitSpan still to figure when to stop.
-                fn ptr_cursor_from_bit_span(bs: BitSpan) -> u8 {
-                    let p = Self::get_bit_pos(bs.bits << (4 - bs.len), 4)
-                        .leading_zeros() as u8;
-                    trace!("bs in {:?}", bs);
-                    trace!("pos {}", p);
-                    p
-                }
+                // fn ptr_cursor_from_bit_span(bs: BitSpan) -> u8 {
+                //     let p = Self::get_bit_pos(bs.bits << (4 - bs.len), 4)
+                //         .leading_zeros() as u8;
+                //     trace!("bs in {:?}", bs);
+                //     trace!("pos {}", p);
+                //     p
+                // }
 
                 fn get_bit_pos_as_u8(nibble: u32, len: u8) -> u8 {
                     1 << (
@@ -257,9 +257,9 @@ macro_rules! impl_primitive_atomic_stride {
                     )
                 }
 
-                fn get_pfx_index(nibble: u32, len: u8)
+                fn get_pfx_index(bs: BitSpan)
                 -> usize {
-                    (Self::get_bit_pos(nibble, len).leading_zeros() - 1) as usize
+                    (Self::get_bit_pos(bs).leading_zeros() - 1) as usize
 
                 }
 
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 7808ba2b..e895bb37 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -135,8 +135,9 @@ where
     // compare_exchange of both ptrbitarr and pfxbitarr.
     pub(crate) fn eval_node_or_prefix_at(
         &self,
-        nibble: u32,
-        nibble_len: u8,
+        bit_span: BitSpan,
+        // nibble: u32,
+        // nibble_len: u8,
         // all the bits of the search prefix, but with the length set to
         // the length of this stride. So bits are set beyond its length.
         base_prefix: StrideNodeId<AF>,
@@ -158,7 +159,7 @@ where
         let mut retry_count = 0;
         let ptrbitarr = self.ptrbitarr.load();
         let pfxbitarr = self.pfxbitarr.load();
-        let bit_pos = S::get_bit_pos(nibble, nibble_len);
+        let bit_pos = S::get_bit_pos(bit_span);
         let new_node: SizedStrideNode<AF>;
 
         // Check that we're not at the last stride (pfx.len <= stride_end),
@@ -1123,7 +1124,7 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
             S::cursor_from_bit_span(bs)
         );
         trace!("pfx {:032b}", self.pfxbitarr);
-        let bit_pos = S::get_bit_pos(bs.bits, bs.len);
+        let bit_pos = S::get_bit_pos(bs);
         let prefix_id: PrefixId<AF> = self
             .base_prefix
             .add_bit_span(BitSpan::from_bit_pos_index(
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index e9a58962..803f38e3 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -283,14 +283,6 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             return self
                 .update_default_route_prefix_meta(mui)
                 .map(|(rc, _mui_exists)| (rc, !prefix_new));
-            // return self.update_default_route_prefix_meta(mui).map(
-            //     |(rc, mui_exists)| UpsertReport {
-            //         cas_count: rc as usize,
-            //         prefix_new,
-            //         mui_new: !mui_exists,
-            //         mui_count: 0,
-            //     },
-            // );
         }
 
         let mut stride_end: u8 = 0;
@@ -307,7 +299,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                 stride
             };
 
-            let nibble = AF::get_nibble(
+            let bit_span = AF::into_bit_span(
                 pfx.get_net(),
                 stride_end - stride,
                 nibble_len,
@@ -315,28 +307,111 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             let is_last_stride = pfx.get_len() <= stride_end;
             let stride_start = stride_end - stride;
 
-            // insert_match! returns the node_id of the next node to be
-            // traversed. It was created if it did not exist.
-            let node_result = insert_match![
-                // applicable to the whole outer match in the macro
-                self;
-                guard;
-                nibble_len;
-                nibble;
-                is_last_stride;
-                pfx;
-                mui;
-                // the length at the start of the stride a.k.a. start_bit
-                stride_start;
-                stride;
-                cur_i;
-                level;
-                acc_retry_count;
-                // Strides to create match arm for; stats level
-                Stride3; 0,
-                Stride4; 1,
-                Stride5; 2
-            ];
+            // this counts the number of retry_count for this loop only,
+            // but ultimately we will return the accumulated count of all
+            // retry_count from this macro.
+            let node_result =
+                {
+                    let local_retry_count = 0;
+                    // retrieve_node_mut updates the bitmap index if
+                    // necessary.
+                    if let Some(current_node) =
+                        self.retrieve_node_mut(cur_i, mui)
+                    {
+                        match current_node {
+                            SizedStrideRef::Stride4(current_node) => {
+                                // eval_node_or_prefix_at mutates the node
+                                // to reflect changes in the ptrbitarr &
+                                // pfxbitarr.
+                                match current_node.eval_node_or_prefix_at(
+                        bit_span,
+                        // All the bits of the search prefix, but with
+                        // a length set to the start of the current
+                        // stride.
+                        StrideNodeId::dangerously_new_with_id_as_is(
+                            pfx.get_net(),
+                            stride_start,
+                        ),
+                        // the length of THIS stride
+                        stride,
+                        // the length of the next stride
+                        self.get_stride_sizes().get((level + 1) as usize),
+                        is_last_stride,
+                    ) {
+                        (NewNodeOrIndex::NewNode(n), retry_count) => {
+                            // Stride3 logs to stats[0], Stride4 logs
+                            // to stats[1], etc.
+                            // $self.stats[$stats_level].inc($level);
+
+                            // get a new identifier for the node we're
+                            // going to create.
+                            let new_id = StrideNodeId::new_with_cleaned_id(
+                                pfx.get_net(),
+                                stride_start + bit_span.len,
+                            );
+
+                            // store the new node in the in_memory
+                            // part of the RIB. It returns the created
+                            // id and the number of retries before
+                            // success.
+                            match self.store_node(new_id, mui, n) {
+                                Ok((node_id, s_retry_count)) => Ok((
+                                    node_id,
+                                    acc_retry_count
+                                        + s_retry_count
+                                        + retry_count,
+                                )),
+                                Err(err) => Err(err),
+                            }
+                        }
+                        (
+                            NewNodeOrIndex::ExistingNode(node_id),
+                            retry_count,
+                        ) => {
+                            if log_enabled!(log::Level::Trace)
+                                && local_retry_count > 0
+                            {
+                                trace!(
+                                    "{} contention: Node \
+                                             already exists {}",
+                                    std::thread::current()
+                                        .name()
+                                        .unwrap_or("unnamed-thread"),
+                                    node_id
+                                )
+                            }
+                            Ok((
+                                node_id,
+                                acc_retry_count
+                                    + local_retry_count
+                                    + retry_count,
+                            ))
+                        }
+                        (NewNodeOrIndex::NewPrefix, retry_count) => {
+                            break (
+                                acc_retry_count
+                                    + local_retry_count
+                                    + retry_count,
+                                false,
+                            )
+                        }
+                        (NewNodeOrIndex::ExistingPrefix, retry_count) => {
+                            break (
+                                acc_retry_count
+                                    + local_retry_count
+                                    + retry_count,
+                                true,
+                            )
+                        }
+                    } // end of eval_node_or_prefix_at
+                            }
+                            SizedStrideRef::Stride3(_) => todo!(),
+                            SizedStrideRef::Stride5(_) => todo!(),
+                        }
+                    } else {
+                        Err(PrefixStoreError::NodeCreationMaxRetryError)
+                    }
+                };
 
             match node_result {
                 Ok((next_id, retry_count)) => {
@@ -376,15 +451,12 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         match self.retrieve_node(node_id) {
             Some(SizedStrideRef::Stride4(n)) => {
                 let pfxbitarr = n.pfxbitarr.load();
-                pfxbitarr & Stride4::get_bit_pos(bs.bits, bs.len) > 0
+                pfxbitarr & Stride4::get_bit_pos(bs) > 0
             }
-            None => false,
-            Some(n) => {
-                panic!(
-                    "unsupported stride length: {:?} node_id {}",
-                    n, node_id
-                );
+            Some(_) => {
+                panic!("bla");
             }
+            None => false,
         }
     }
 
@@ -399,15 +471,12 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         match self.retrieve_node_for_mui(node_id, mui) {
             Some(SizedStrideRef::Stride4(n)) => {
                 let pfxbitarr = n.pfxbitarr.load();
-                pfxbitarr & Stride4::get_bit_pos(bs.bits, bs.len) > 0
+                pfxbitarr & Stride4::get_bit_pos(bs) > 0
             }
-            None => false,
-            Some(n) => {
-                panic!(
-                    "unsupported stride length: {:?} node_id {}",
-                    n, node_id
-                );
+            Some(_) => {
+                panic!("man o man");
             }
+            None => false,
         }
     }
     // pub fn prefix_exists_legacy(&self, prefix_id: PrefixId<AF>) -> bool {
@@ -612,20 +681,20 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         if let Some(root_node) =
             self.retrieve_node_mut(self.get_root_node_id(), mui)
         {
-            match root_node {
-                SizedStrideRef::Stride3(_) => self
-                    .node_buckets
-                    .get_store3(self.get_root_node_id())
-                    .update_rbm_index(mui),
-                SizedStrideRef::Stride4(_) => self
-                    .node_buckets
-                    .get_store4(self.get_root_node_id())
-                    .update_rbm_index(mui),
-                SizedStrideRef::Stride5(_) => self
-                    .node_buckets
-                    .get_store5(self.get_root_node_id())
-                    .update_rbm_index(mui),
-            }
+            // match root_node {
+            //     SizedStrideRef::Stride3(_) => self
+            //         .node_buckets
+            //         .get_store3(self.get_root_node_id())
+            //         .update_rbm_index(mui),
+            // SizedStrideRef::Stride4(_) => self
+            self.node_buckets
+                .get_store4(self.get_root_node_id())
+                .update_rbm_index(mui)
+            // SizedStrideRef::Stride5(_) => self
+            //     .node_buckets
+            //     .get_store5(self.get_root_node_id())
+            //     .update_rbm_index(mui),
+            // }
         } else {
             Err(PrefixStoreError::StoreNotReadyError)
         }
@@ -767,6 +836,282 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         }
     }
 
+    pub fn _new_retrieve_node_mut(
+        &self,
+        id: StrideNodeId<AF>,
+        mui: u32,
+    ) -> Option<&TreeBitMapNode<AF, Stride4>> {
+        // HASHING FUNCTION
+        let mut level = 0;
+        let mut node;
+        let mut nodes = self.node_buckets.get_store4(id);
+
+        loop {
+            let index = Self::hash_node_id(id, level);
+            match nodes.read().get(index) {
+                // This arm only ever gets called in multi-threaded code
+                // where our thread (running this code *now*), andgot
+                // ahead of another thread: After the other thread created
+                // the TreeBitMapNode first, it was overtaken by our
+                // thread running this method, so our thread enounters an
+                // empty node in the store.
+                None => {
+                    let this_level =
+                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            id.get_id().1,
+                            level,
+                        );
+                    let next_level =
+                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            id.get_id().1,
+                            level + 1,
+                        );
+                    let node_set = NodeSet::init(next_level - this_level);
+
+                    // See if we can create the node
+                    (node, _) =
+                        nodes.read().get_or_init(index, || StoredNode {
+                            node_id: id,
+                            node: TreeBitMapNode::new(),
+                            node_set,
+                        });
+
+                    // We may have lost, and a different node than we
+                    // intended could live here, if so go a level deeper
+                    if id == node.node_id {
+                        // Nope, its ours or at least the node we need.
+                        let _retry_count =
+                            node.node_set.update_rbm_index(mui).ok();
+
+                        return Some(&node.node);
+                    };
+                }
+                Some(this_node) => {
+                    node = this_node;
+                    if id == this_node.node_id {
+                        // YES, It's the one we're looking for!
+
+                        // Update the rbm_index in this node with the
+                        // multi_uniq_id that the caller specified. This
+                        // is the only atomic operation we need to do
+                        // here. The NodeSet that the index is attached
+                        // to, does not need to be written to, it's part
+                        // of a trie, so it just needs to "exist" (and it
+                        // already does).
+                        let retry_count =
+                            this_node.node_set.update_rbm_index(mui).ok();
+
+                        trace!("Retry_count rbm index {:?}", retry_count);
+                        trace!(
+                        "add multi uniq id to bitmap index {} for node {}",
+                        mui,
+                        this_node.node
+                    );
+                        return Some(&this_node.node);
+                    };
+                }
+            }
+            // It isn't ours. Move one level deeper.
+            level += 1;
+            match <NB as NodeBuckets<AF>>::len_to_store_bits(
+                id.get_id().1,
+                level,
+            ) {
+                // on to the next level!
+                next_bit_shift if next_bit_shift > 0 => {
+                    nodes = &node.node_set;
+                }
+                // There's no next level, we found nothing.
+                _ => return None,
+            }
+        }
+    }
+
+    pub fn _new_retrieve_node(
+        &self,
+        id: StrideNodeId<AF>,
+    ) -> Option<&TreeBitMapNode<AF, Stride4>> {
+        // HASHING FUNCTION
+        let mut level = 0;
+        let mut node;
+        let mut nodes = self.node_buckets.get_store4(id);
+
+        loop {
+            let index = Self::hash_node_id(id, level);
+            match nodes.read().get(index) {
+                // This arm only ever gets called in multi-threaded code
+                // where our thread (running this code *now*), andgot
+                // ahead of another thread: After the other thread created
+                // the TreeBitMapNode first, it was overtaken by our
+                // thread running this method, so our thread enounters an
+                // empty node in the store.
+                None => {
+                    let this_level =
+                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            id.get_id().1,
+                            level,
+                        );
+                    let next_level =
+                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            id.get_id().1,
+                            level + 1,
+                        );
+                    let node_set = NodeSet::init(next_level - this_level);
+
+                    // See if we can create the node
+                    (node, _) =
+                        nodes.read().get_or_init(index, || StoredNode {
+                            node_id: id,
+                            node: TreeBitMapNode::new(),
+                            node_set,
+                        });
+
+                    // We may have lost, and a different node than we
+                    // intended could live here, if so go a level deeper
+                    if id == node.node_id {
+                        // Nope, its ours or at least the node we need.
+
+                        return Some(&node.node);
+                    };
+                }
+                Some(this_node) => {
+                    node = this_node;
+                    if id == this_node.node_id {
+                        // YES, It's the one we're looking for!
+
+                        // Update the rbm_index in this node with the
+                        // multi_uniq_id that the caller specified. This
+                        // is the only atomic operation we need to do
+                        // here. The NodeSet that the index is attached
+                        // to, does not need to be written to, it's part
+                        // of a trie, so it just needs to "exist" (and it
+                        // already does).
+                        // let retry_count =
+                        //     this_node.node_set.update_rbm_index(mui).ok();
+
+                        //     trace!("Retry_count rbm index {:?}", retry_count);
+                        //     trace!(
+                        //     "add multi uniq id to bitmap index {} for node {}",
+                        //     mui,
+                        //     this_node.node
+                        // );
+                        return Some(&this_node.node);
+                    };
+                }
+            }
+            // It isn't ours. Move one level deeper.
+            level += 1;
+            match <NB as NodeBuckets<AF>>::len_to_store_bits(
+                id.get_id().1,
+                level,
+            ) {
+                // on to the next level!
+                next_bit_shift if next_bit_shift > 0 => {
+                    nodes = &node.node_set;
+                }
+                // There's no next level, we found nothing.
+                _ => return None,
+            }
+        }
+    }
+    pub fn _new_retrieve_node_for_mui(
+        &self,
+        id: StrideNodeId<AF>,
+        mui: u32,
+    ) -> Option<&TreeBitMapNode<AF, Stride4>> {
+        // HASHING FUNCTION
+        let mut level = 0;
+        let mut node;
+        let mut nodes = self.node_buckets.get_store4(id);
+
+        loop {
+            let index = Self::hash_node_id(id, level);
+            match nodes.read().get(index) {
+                // This arm only ever gets called in multi-threaded code
+                // where our thread (running this code *now*), andgot
+                // ahead of another thread: After the other thread created
+                // the TreeBitMapNode first, it was overtaken by our
+                // thread running this method, so our thread enounters an
+                // empty node in the store.
+                None => {
+                    let this_level =
+                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            id.get_id().1,
+                            level,
+                        );
+                    let next_level =
+                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            id.get_id().1,
+                            level + 1,
+                        );
+                    let node_set = NodeSet::init(next_level - this_level);
+
+                    // See if we can create the node
+                    (node, _) =
+                        nodes.read().get_or_init(index, || StoredNode {
+                            node_id: id,
+                            node: TreeBitMapNode::new(),
+                            node_set,
+                        });
+
+                    // We may have lost, and a different node than we
+                    // intended could live here, if so go a level deeper
+                    if id == node.node_id {
+                        // Nope, its ours or at least the node we need.
+
+                        return Some(&node.node);
+                    };
+                }
+                Some(this_node) => {
+                    node = this_node;
+                    // early return if the mui is not in the index
+                    // stored in this node, meaning the mui does not
+                    // appear anywhere in the sub-tree formed from
+                    // this node.
+                    let bmin = nodes.rbm().read().unwrap();
+                    if !bmin.contains(mui) {
+                        return None;
+                    }
+
+                    if id == this_node.node_id {
+                        // YES, It's the one we're looking for!
+
+                        // Update the rbm_index in this node with the
+                        // multi_uniq_id that the caller specified. This
+                        // is the only atomic operation we need to do
+                        // here. The NodeSet that the index is attached
+                        // to, does not need to be written to, it's part
+                        // of a trie, so it just needs to "exist" (and it
+                        // already does).
+                        // let retry_count =
+                        //     this_node.node_set.update_rbm_index(mui).ok();
+
+                        //     trace!("Retry_count rbm index {:?}", retry_count);
+                        //     trace!(
+                        //     "add multi uniq id to bitmap index {} for node {}",
+                        //     mui,
+                        //     this_node.node
+                        // );
+                        return Some(&this_node.node);
+                    };
+                }
+            }
+            // It isn't ours. Move one level deeper.
+            level += 1;
+            match <NB as NodeBuckets<AF>>::len_to_store_bits(
+                id.get_id().1,
+                level,
+            ) {
+                // on to the next level!
+                next_bit_shift if next_bit_shift > 0 => {
+                    nodes = &node.node_set;
+                }
+                // There's no next level, we found nothing.
+                _ => return None,
+            }
+        }
+    }
+
     #[allow(clippy::type_complexity)]
     pub(crate) fn retrieve_node(
         &self,
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 7c679993..266ada9c 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -80,11 +80,11 @@ pub struct ShortKey<AF: AddressFamily> {
 )]
 #[repr(C)]
 pub struct LongKey<AF: AddressFamily> {
-    prefix: PrefixId<AF>,     // 4 or 16
+    prefix: PrefixId<AF>,     // 1 + (4 or 16)
     mui: U32<NativeEndian>,   // 4
     ltime: U64<NativeEndian>, // 8
     status: RouteStatus,      // 1
-} // 17 or 29
+} // 18 or 30
 
 impl<AF: AddressFamily, const KEY_SIZE: usize> KeySize<AF, KEY_SIZE>
     for ShortKey<AF>
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
index d5d7b12d..2fec77f9 100644
--- a/src/local_array/prefix_cht/iterators.rs
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -11,7 +11,7 @@ use crate::{
     AddressFamily,
 };
 
-type Type<AF> = SizedNodeMoreSpecificIter<AF>;
+type Type<AF: AddressFamily> = SizedNodeMoreSpecificIter<AF>;
 
 pub(crate) struct MoreSpecificPrefixIter<
     'a,
diff --git a/src/macros.rs b/src/macros.rs
index 57e90774..5920aa65 100644
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -11,12 +11,12 @@ macro_rules! impl_primitive_stride {
                     const BITS: u8 = $bits;
                     const STRIDE_LEN: u8 = $len;
 
-                    fn get_bit_pos(nibble: u32, len: u8) -> $pfxsize {
-                        1 << (<Self as Stride>::BITS - ((1 << len) - 1) as u8 - nibble as u8 - 1)
+                    fn get_bit_pos(bs: BitSpan) -> $pfxsize {
+                        1 << (<Self as Stride>::BITS - ((1 << bs.len) - 1) as u8 - bs.bits as u8 - 1)
                     }
 
-                    fn get_pfx_index(bitmap: $pfxsize, nibble: u32, len: u8) -> usize {
-                        (bitmap >> ((<Self as Stride>::BITS - ((1 << len) - 1) as u8 - nibble as u8 - 1) as usize))
+                    fn get_pfx_index(bitmap: $pfxsize, bs: BitSpan) -> usize {
+                        (bitmap >> ((<Self as Stride>::BITS - ((1 << bs.len) - 1) as u8 - bs.bits as u8 - 1) as usize))
                             .count_ones() as usize
                             - 1
                     }
diff --git a/src/rotonda_store.rs b/src/rotonda_store.rs
index 098e7082..14e74ac1 100644
--- a/src/rotonda_store.rs
+++ b/src/rotonda_store.rs
@@ -17,7 +17,7 @@ pub const RECORDS_MAX_NUM: usize = 3;
 //------------ The publicly available Rotonda Stores ------------------------
 
 pub use crate::local_array::rib::DefaultStore as MultiThreadedStore;
-pub use crate::local_vec::store::Store as SingleThreadedStore;
+// pub use crate::local_vec::store::Store as SingleThreadedStore;
 
 //------------ Types for strides displaying/monitoring ----------------------
 
diff --git a/src/stride.rs b/src/stride.rs
index e3e4f771..f13911d1 100644
--- a/src/stride.rs
+++ b/src/stride.rs
@@ -1,4 +1,5 @@
 use crate::impl_primitive_stride;
+use crate::local_array::bit_span::BitSpan;
 use crate::synth_int::{U256, U512};
 use std::fmt::{Binary, Debug};
 
@@ -34,7 +35,7 @@ pub trait Stride:
     // length, this follows from the fact that the `nibble` value represents
     // *both* the bitmap part, we're considering here *and* the position
     // relative to the nibble length offset in the bitmap.
-    fn get_bit_pos(nibble: u32, len: u8) -> Self;
+    fn get_bit_pos(bs: BitSpan) -> Self;
 
     // Clear the bitmap to the right of the pointer and count the number of ones.
     // This numbder represents the index to the corresponding prefix in the pfx_vec.
@@ -48,7 +49,7 @@ pub trait Stride:
     // `nibble`
     // The bit position relative to the offset for the nibble length, this index
     // is only used at the last (relevant) stride, so the offset is always 0.
-    fn get_pfx_index(bitmap: Self, nibble: u32, len: u8) -> usize;
+    fn get_pfx_index(bitmap: Self, bs: BitSpan) -> usize;
 
     // Clear the bitmap to the right of the pointer and count the number of ones.
     // This number represents the index to the corresponding child node in the ptr_vec.
@@ -97,15 +98,15 @@ impl Stride for Stride7 {
     const BITS: u8 = 255;
     const STRIDE_LEN: u8 = 7;
 
-    fn get_bit_pos(nibble: u32, len: u8) -> Self {
-        match 256 - ((1 << len) - 1) as u16 - nibble as u16 - 1 {
+    fn get_bit_pos(bs: BitSpan) -> Self {
+        match 256 - ((1 << bs.len) - 1) as u16 - bs.bits as u16 - 1 {
             n if n < 128 => U256(0, 1 << n),
             n => U256(1 << (n - 128), 0),
         }
     }
 
-    fn get_pfx_index(bitmap: Self, nibble: u32, len: u8) -> usize {
-        let n = 256 - ((1 << len) - 1) as u16 - nibble as u16 - 1;
+    fn get_pfx_index(bitmap: Self, bs: BitSpan) -> usize {
+        let n = 256 - ((1 << bs.len) - 1) as u16 - bs.bits as u16 - 1;
         match n {
             // if we move less than 128 bits to the right,
             // all of bitmap.0 and a part of bitmap.1 will be used for counting zeros
@@ -159,8 +160,8 @@ impl Stride for Stride8 {
     const BITS: u8 = 255; // bogus
     const STRIDE_LEN: u8 = 8;
 
-    fn get_bit_pos(nibble: u32, len: u8) -> Self {
-        match 512 - ((1 << len) - 1) as u16 - nibble as u16 - 1 {
+    fn get_bit_pos(bs: BitSpan) -> Self {
+        match 512 - ((1 << bs.len) - 1) as u16 - bs.bits as u16 - 1 {
             n if n < 128 => U512(0, 0, 0, 1 << n),
             n if n < 256 => U512(0, 0, 1 << (n - 128), 0),
             n if n < 384 => U512(0, 1 << (n - 256), 0, 0),
@@ -168,8 +169,8 @@ impl Stride for Stride8 {
         }
     }
 
-    fn get_pfx_index(bitmap: Self, nibble: u32, len: u8) -> usize {
-        let n = 512 - ((1 << len) - 1) as u16 - nibble as u16 - 1;
+    fn get_pfx_index(bitmap: Self, bs: BitSpan) -> usize {
+        let n = 512 - ((1 << bs.len) - 1) as u16 - bs.bits as u16 - 1;
         match n {
             // if we move less than 128 bits to the right, all of bitmap.2
             // and a part of bitmap.3 will be used for counting zeros.

From 83d1f95a0eb04fb420e60978ff5455be1fd9318a Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Feb 2025 14:29:04 +0100
Subject: [PATCH 085/147] get rid of retrieve_* closure macros

---
 src/local_array/in_memory/iterators.rs  | 127 +++++++++++----------
 src/local_array/in_memory/tree.rs       | 143 +++++++++++-------------
 src/local_array/prefix_cht/iterators.rs |   5 +-
 3 files changed, 131 insertions(+), 144 deletions(-)

diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 9033cea7..ae563334 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -146,30 +146,30 @@ impl<'a, AF: AddressFamily + 'a, NB: NodeBuckets<AF>> Iterator
                 let node = self.tree.retrieve_node(next_ptr);
 
                 match node {
-                    Some(SizedStrideRef::Stride3(next_node)) => {
-                        // copy the current iterator into the parent vec and create
-                        // a new ptr iterator for this node
-                        self.parent_and_position.push(self.cur_ptr_iter);
-                        let ptr_iter = next_node.more_specific_ptr_iter(
-                            next_ptr,
-                            BitSpan { bits: 0, len: 0 },
-                        );
-                        self.cur_ptr_iter = ptr_iter.wrap();
-
-                        // trace!(
-                        //     "next stride new iterator stride 3 {:?} start \
-                        // bit_span {:?}",
-                        //     self.cur_ptr_iter,
-                        //     self.start_bit_span
-                        // );
-                        self.cur_pfx_iter = next_node
-                            .more_specific_pfx_iter(
-                                next_ptr,
-                                BitSpan::new(0, 0),
-                            )
-                            .wrap();
-                    }
-                    Some(SizedStrideRef::Stride4(next_node)) => {
+                    // Some(next_node) => {
+                    //     // copy the current iterator into the parent vec and create
+                    //     // a new ptr iterator for this node
+                    //     self.parent_and_position.push(self.cur_ptr_iter);
+                    //     let ptr_iter = next_node.more_specific_ptr_iter(
+                    //         next_ptr,
+                    //         BitSpan { bits: 0, len: 0 },
+                    //     );
+                    //     self.cur_ptr_iter = ptr_iter.wrap();
+
+                    //     // trace!(
+                    //     //     "next stride new iterator stride 3 {:?} start \
+                    //     // bit_span {:?}",
+                    //     //     self.cur_ptr_iter,
+                    //     //     self.start_bit_span
+                    //     // );
+                    //     self.cur_pfx_iter = next_node
+                    //         .more_specific_pfx_iter(
+                    //             next_ptr,
+                    //             BitSpan::new(0, 0),
+                    //         )
+                    //         .wrap();
+                    // }
+                    Some(next_node) => {
                         // create new ptr iterator for this node.
                         self.parent_and_position.push(self.cur_ptr_iter);
                         let ptr_iter = next_node.more_specific_ptr_iter(
@@ -190,28 +190,28 @@ impl<'a, AF: AddressFamily + 'a, NB: NodeBuckets<AF>> Iterator
                             )
                             .wrap();
                     }
-                    Some(SizedStrideRef::Stride5(next_node)) => {
-                        // create new ptr iterator for this node.
-                        self.parent_and_position.push(self.cur_ptr_iter);
-                        let ptr_iter = next_node.more_specific_ptr_iter(
-                            next_ptr,
-                            BitSpan { bits: 0, len: 0 },
-                        );
-                        self.cur_ptr_iter = ptr_iter.wrap();
-
-                        // trace!(
-                        //     "next stride new iterator stride 5 {:?} start \
-                        // bit_span {:?}",
-                        //     self.cur_ptr_iter,
-                        //     self.start_bit_span
-                        // );
-                        self.cur_pfx_iter = next_node
-                            .more_specific_pfx_iter(
-                                next_ptr,
-                                BitSpan::new(0, 0),
-                            )
-                            .wrap();
-                    }
+                    // Some(SizedStrideRef::Stride5(next_node)) => {
+                    //     // create new ptr iterator for this node.
+                    //     self.parent_and_position.push(self.cur_ptr_iter);
+                    //     let ptr_iter = next_node.more_specific_ptr_iter(
+                    //         next_ptr,
+                    //         BitSpan { bits: 0, len: 0 },
+                    //     );
+                    //     self.cur_ptr_iter = ptr_iter.wrap();
+
+                    //     // trace!(
+                    //     //     "next stride new iterator stride 5 {:?} start \
+                    //     // bit_span {:?}",
+                    //     //     self.cur_ptr_iter,
+                    //     //     self.start_bit_span
+                    //     // );
+                    //     self.cur_pfx_iter = next_node
+                    //         .more_specific_pfx_iter(
+                    //             next_ptr,
+                    //             BitSpan::new(0, 0),
+                    //         )
+                    //         .wrap();
+                    // }
                     None => {
                         println!("no node here.");
                         return None;
@@ -353,15 +353,15 @@ impl<
 
             if let Some(node) = node {
                 match node {
-                    SizedStrideRef::Stride3(n) => {
-                        cur_pfx_iter = SizedPrefixIter::Stride3(
-                            n.more_specific_pfx_iter(start_node_id, start_bs),
-                        );
-                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
-                            n.more_specific_ptr_iter(start_node_id, start_bs),
-                        );
-                    }
-                    SizedStrideRef::Stride4(n) => {
+                    // SizedStrideRef::Stride3(n) => {
+                    //     cur_pfx_iter = SizedPrefixIter::Stride3(
+                    //         n.more_specific_pfx_iter(start_node_id, start_bs),
+                    //     );
+                    //     cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
+                    //         n.more_specific_ptr_iter(start_node_id, start_bs),
+                    //     );
+                    // }
+                    n => {
                         cur_pfx_iter = SizedPrefixIter::Stride4(
                             n.more_specific_pfx_iter(start_node_id, start_bs),
                         );
@@ -370,15 +370,14 @@ impl<
                         cur_ptr_iter = SizedNodeMoreSpecificIter::Stride4(
                             n.more_specific_ptr_iter(start_node_id, start_bs),
                         );
-                    }
-                    SizedStrideRef::Stride5(n) => {
-                        cur_pfx_iter = SizedPrefixIter::Stride5(
-                            n.more_specific_pfx_iter(start_node_id, start_bs),
-                        );
-                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
-                            n.more_specific_ptr_iter(start_node_id, start_bs),
-                        );
-                    }
+                    } // SizedStrideRef::Stride5(n) => {
+                      //     cur_pfx_iter = SizedPrefixIter::Stride5(
+                      //         n.more_specific_pfx_iter(start_node_id, start_bs),
+                      //     );
+                      //     cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
+                      //         n.more_specific_ptr_iter(start_node_id, start_bs),
+                      //     );
+                      // }
                 };
 
                 Some(MoreSpecificPrefixIter {
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 803f38e3..6f10940f 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -310,20 +310,14 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             // this counts the number of retry_count for this loop only,
             // but ultimately we will return the accumulated count of all
             // retry_count from this macro.
-            let node_result =
+            let node_result = {
+                let local_retry_count = 0;
+                // retrieve_node_mut updates the bitmap index if
+                // necessary.
+                if let Some(current_node) = self.retrieve_node_mut(cur_i, mui)
                 {
-                    let local_retry_count = 0;
-                    // retrieve_node_mut updates the bitmap index if
-                    // necessary.
-                    if let Some(current_node) =
-                        self.retrieve_node_mut(cur_i, mui)
-                    {
-                        match current_node {
-                            SizedStrideRef::Stride4(current_node) => {
-                                // eval_node_or_prefix_at mutates the node
-                                // to reflect changes in the ptrbitarr &
-                                // pfxbitarr.
-                                match current_node.eval_node_or_prefix_at(
+                    // let current_node = current_node;
+                    match current_node.eval_node_or_prefix_at(
                         bit_span,
                         // All the bits of the search prefix, but with
                         // a length set to the start of the current
@@ -373,7 +367,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                             {
                                 trace!(
                                     "{} contention: Node \
-                                             already exists {}",
+                                                 already exists {}",
                                     std::thread::current()
                                         .name()
                                         .unwrap_or("unnamed-thread"),
@@ -403,15 +397,11 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                                 true,
                             )
                         }
-                    } // end of eval_node_or_prefix_at
-                            }
-                            SizedStrideRef::Stride3(_) => todo!(),
-                            SizedStrideRef::Stride5(_) => todo!(),
-                        }
-                    } else {
-                        Err(PrefixStoreError::NodeCreationMaxRetryError)
                     }
-                };
+                } else {
+                    return Err(PrefixStoreError::NodeCreationMaxRetryError);
+                }
+            };
 
             match node_result {
                 Ok((next_id, retry_count)) => {
@@ -449,13 +439,10 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         let (node_id, bs) = self.get_node_id_for_prefix(&prefix_id);
 
         match self.retrieve_node(node_id) {
-            Some(SizedStrideRef::Stride4(n)) => {
+            Some(n) => {
                 let pfxbitarr = n.pfxbitarr.load();
                 pfxbitarr & Stride4::get_bit_pos(bs) > 0
             }
-            Some(_) => {
-                panic!("bla");
-            }
             None => false,
         }
     }
@@ -469,13 +456,10 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         let (node_id, bs) = self.get_node_id_for_prefix(&prefix_id);
 
         match self.retrieve_node_for_mui(node_id, mui) {
-            Some(SizedStrideRef::Stride4(n)) => {
+            Some(n) => {
                 let pfxbitarr = n.pfxbitarr.load();
                 pfxbitarr & Stride4::get_bit_pos(bs) > 0
             }
-            Some(_) => {
-                panic!("man o man");
-            }
             None => false,
         }
     }
@@ -786,7 +770,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
     }
 
     #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_node_mut(
+    pub(crate) fn _retrieve_node_mut(
         &self,
         id: StrideNodeId<AF>,
         multi_uniq_id: u32,
@@ -836,7 +820,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         }
     }
 
-    pub fn _new_retrieve_node_mut(
+    pub fn retrieve_node_mut(
         &self,
         id: StrideNodeId<AF>,
         mui: u32,
@@ -927,7 +911,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         }
     }
 
-    pub fn _new_retrieve_node(
+    pub fn retrieve_node(
         &self,
         id: StrideNodeId<AF>,
     ) -> Option<&TreeBitMapNode<AF, Stride4>> {
@@ -1014,7 +998,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             }
         }
     }
-    pub fn _new_retrieve_node_for_mui(
+    pub fn retrieve_node_for_mui(
         &self,
         id: StrideNodeId<AF>,
         mui: u32,
@@ -1063,12 +1047,13 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                     };
                 }
                 Some(this_node) => {
-                    node = this_node;
+                    // node = this_node;
                     // early return if the mui is not in the index
                     // stored in this node, meaning the mui does not
                     // appear anywhere in the sub-tree formed from
                     // this node.
-                    let bmin = nodes.rbm().read().unwrap();
+                    node = this_node;
+                    let bmin = this_node.node_set.rbm().read().unwrap();
                     if !bmin.contains(mui) {
                         return None;
                     }
@@ -1112,56 +1097,56 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         }
     }
 
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_node(
-        &self,
-        id: StrideNodeId<AF>,
-    ) -> Option<SizedStrideRef<'_, AF>> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, S>,
-                &'a NodeSet<AF, S>,
-                u8,
-            )
-                -> Option<SizedStrideRef<'a, AF>>,
-        }
+    // #[allow(clippy::type_complexity)]
+    // pub(crate) fn retrieve_node(
+    //     &self,
+    //     id: StrideNodeId<AF>,
+    // ) -> Option<SizedStrideRef<'_, AF>> {
+    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+    //         f: &'s dyn for<'a> Fn(
+    //             &SearchLevel<AF, S>,
+    //             &'a NodeSet<AF, S>,
+    //             u8,
+    //         )
+    //             -> Option<SizedStrideRef<'a, AF>>,
+    //     }
 
-        let search_level_3 = impl_search_level![Stride3; id;];
-        let search_level_4 = impl_search_level![Stride4; id;];
-        let search_level_5 = impl_search_level![Stride5; id;];
+    //     let search_level_3 = impl_search_level![Stride3; id;];
+    //     let search_level_4 = impl_search_level![Stride4; id;];
+    //     let search_level_5 = impl_search_level![Stride5; id;];
 
-        if log_enabled!(log::Level::Trace) {
-            trace!(
-                "{} store: Retrieve node {} from l{}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                id.get_id().1
-            );
-        }
+    //     if log_enabled!(log::Level::Trace) {
+    //         trace!(
+    //             "{} store: Retrieve node {} from l{}",
+    //             std::thread::current().name().unwrap_or("unnamed-thread"),
+    //             id,
+    //             id.get_id().1
+    //         );
+    //     }
 
-        match self.get_stride_for_id(id) {
-            3 => (search_level_3.f)(
-                &search_level_3,
-                self.node_buckets.get_store3(id),
-                0,
-            ),
-            4 => (search_level_4.f)(
-                &search_level_4,
-                self.node_buckets.get_store4(id),
-                0,
-            ),
-            _ => (search_level_5.f)(
-                &search_level_5,
-                self.node_buckets.get_store5(id),
-                0,
-            ),
-        }
-    }
+    //     match self.get_stride_for_id(id) {
+    //         3 => (search_level_3.f)(
+    //             &search_level_3,
+    //             self.node_buckets.get_store3(id),
+    //             0,
+    //         ),
+    //         4 => (search_level_4.f)(
+    //             &search_level_4,
+    //             self.node_buckets.get_store4(id),
+    //             0,
+    //         ),
+    //         _ => (search_level_5.f)(
+    //             &search_level_5,
+    //             self.node_buckets.get_store5(id),
+    //             0,
+    //         ),
+    //     }
+    // }
 
     // retrieve a node, but only if its bitmap index contains the specified
     // mui. Used for iterators per mui.
     #[allow(clippy::type_complexity)]
-    pub(crate) fn retrieve_node_for_mui(
+    pub(crate) fn _old_retrieve_node_for_mui(
         &self,
         id: StrideNodeId<AF>,
         // The mui that is tested to be present in the nodes bitmap index
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
index 2fec77f9..511af05c 100644
--- a/src/local_array/prefix_cht/iterators.rs
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -79,10 +79,13 @@ impl<
             if let Some(next_ptr) = next_ptr {
                 let node = if self.mui.is_none() {
                     trace!("let's retriev node {}", next_ptr);
-                    self.store.retrieve_node(next_ptr)
+                    self.store
+                        .retrieve_node(next_ptr)
+                        .map(|n| SizedStrideRef::Stride4(n))
                 } else {
                     self.store
                         .retrieve_node_for_mui(next_ptr, self.mui.unwrap())
+                        .map(|n| SizedStrideRef::Stride4(n))
                 };
 
                 match node {

From b60213564f79d0ad25aa77e043939e36995812ee Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Feb 2025 16:04:56 +0100
Subject: [PATCH 086/147] consolidate retrieve_* and store_* w/o closures

---
 src/local_array/in_memory/tree.rs | 467 ++++++++++++++++++++++++++----
 1 file changed, 413 insertions(+), 54 deletions(-)

diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 6f10940f..e53c4bdb 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -188,6 +188,7 @@ use crate::local_array::in_memory::atomic_types::{NodeSet, StoredNode};
 use crate::prelude::multi::PrefixId;
 use crossbeam_epoch::{Atomic, Guard};
 use log::{debug, error, log_enabled, trace};
+use rayon::iter::MultiZip;
 use roaring::RoaringBitmap;
 
 use std::sync::atomic::{
@@ -234,27 +235,29 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         };
 
         let root_node = match Self::get_first_stride_size() {
-            3 => SizedStrideNode::Stride3(TreeBitMapNode {
-                ptrbitarr: AtomicStride2(AtomicU8::new(0)),
-                pfxbitarr: AtomicStride3(AtomicU16::new(0)),
-                _af: PhantomData,
-            }),
-            4 => SizedStrideNode::Stride4(TreeBitMapNode {
+            // 3 => SizedStrideNode::Stride3(TreeBitMapNode {
+            //     ptrbitarr: AtomicStride2(AtomicU8::new(0)),
+            //     pfxbitarr: AtomicStride3(AtomicU16::new(0)),
+            //     _af: PhantomData,
+            // }),
+            4 => TreeBitMapNode {
                 ptrbitarr: AtomicStride3(AtomicU16::new(0)),
                 pfxbitarr: AtomicStride4(AtomicU32::new(0)),
                 _af: PhantomData,
-            }),
-            5 => SizedStrideNode::Stride5(TreeBitMapNode {
-                ptrbitarr: AtomicStride4(AtomicU32::new(0)),
-                pfxbitarr: AtomicStride5(AtomicU64::new(0)),
-                _af: PhantomData,
-            }),
-            unknown_stride_size => {
-                panic!(
-                    "unknown stride size {} encountered in STRIDES array",
-                    unknown_stride_size
-                );
-            }
+            },
+            _ => {
+                panic!("wut?");
+            } // 5 => SizedStrideNode::Stride5(TreeBitMapNode {
+              //     ptrbitarr: AtomicStride4(AtomicU32::new(0)),
+              //     pfxbitarr: AtomicStride5(AtomicU64::new(0)),
+              //     _af: PhantomData,
+              // }),
+              // unknown_stride_size => {
+              //     panic!(
+              //         "unknown stride size {} encountered in STRIDES array",
+              //         unknown_stride_size
+              //     );
+              // }
         };
 
         let _retry_count = tree_bitmap.store_node(
@@ -348,14 +351,18 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                             // part of the RIB. It returns the created
                             // id and the number of retries before
                             // success.
-                            match self.store_node(new_id, mui, n) {
-                                Ok((node_id, s_retry_count)) => Ok((
-                                    node_id,
-                                    acc_retry_count
-                                        + s_retry_count
-                                        + retry_count,
-                                )),
-                                Err(err) => Err(err),
+                            if let SizedStrideNode::Stride4(n) = n {
+                                match self.store_node(new_id, mui, n) {
+                                    Ok((node_id, s_retry_count)) => Ok((
+                                        node_id,
+                                        acc_retry_count
+                                            + s_retry_count
+                                            + retry_count,
+                                    )),
+                                    Err(err) => Err(err),
+                                }
+                            } else {
+                                panic!("ugh");
                             }
                         }
                         (
@@ -695,6 +702,358 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         }
     }
 
+    fn _new_store_node(
+        &self,
+        id: StrideNodeId<AF>,
+        multi_uniq_id: u32,
+        next_node: TreeBitMapNode<AF, Stride4>,
+    ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
+        if log_enabled!(log::Level::Trace) {
+            debug!(
+                "{} store: Store node {}: {:?} mui {}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                id,
+                next_node,
+                multi_uniq_id
+            );
+        }
+        self.counters.inc_nodes_count();
+        let mut level = 0;
+        let mut retry_count = 0;
+        let mut nodes = self.node_buckets.get_store4(id);
+
+        let this_level =
+            <NB as NodeBuckets<AF>>::len_to_store_bits(id.get_id().1, level);
+        trace!("{:032b}", id.get_id().0);
+        trace!("id {:?}", id.get_id());
+        trace!("multi_uniq_id {}", multi_uniq_id);
+
+        loop {
+            // HASHING FUNCTION
+            let index = Self::hash_node_id(id, level);
+
+            match nodes.read().get(index) {
+                None => {
+                    // No node exists, so we create one here.
+                    let next_level =
+                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            id.get_id().1,
+                            level + 1,
+                        );
+
+                    if log_enabled!(log::Level::Trace) {
+                        trace!(
+                        "Empty node found, creating new node {} len{} lvl{}",
+                        id,
+                        id.get_id().1,
+                        level + 1
+                    );
+                        trace!("Next level {}", next_level);
+                        trace!(
+                            "Creating space for {} nodes",
+                            if next_level >= this_level {
+                                1 << (next_level - this_level)
+                            } else {
+                                1
+                            }
+                        );
+                    }
+
+                    trace!("multi uniq id {}", multi_uniq_id);
+
+                    let node_set = NodeSet::init(next_level - this_level);
+
+                    let ptrbitarr = next_node.ptrbitarr.load();
+                    let pfxbitarr = next_node.pfxbitarr.load();
+
+                    let (stored_node, its_us) =
+                        nodes.read().get_or_init(index, || StoredNode {
+                            node_id: id,
+                            node: next_node,
+                            node_set,
+                        });
+
+                    if stored_node.node_id == id {
+                        stored_node
+                            .node_set
+                            .update_rbm_index(multi_uniq_id)?;
+
+                        if !its_us && ptrbitarr != 0 {
+                            retry_count += 1;
+                            stored_node.node.ptrbitarr.merge_with(ptrbitarr);
+                        }
+
+                        if !its_us && pfxbitarr != 0 {
+                            retry_count += 1;
+                            stored_node.node.pfxbitarr.merge_with(pfxbitarr);
+                        }
+                    }
+
+                    return Ok((id, retry_count));
+                }
+                Some(stored_node) => {
+                    // A node exists, might be ours, might be
+                    // another one.
+
+                    if log_enabled!(log::Level::Trace) {
+                        trace!(
+                            "
+                        {} store: Node here exists {:?}",
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread"),
+                            stored_node.node_id
+                        );
+                        trace!("node_id {:?}", stored_node.node_id.get_id());
+                        trace!(
+                            "node_id {:032b}",
+                            stored_node.node_id.get_id().0
+                        );
+                        trace!("id {}", id);
+                        trace!("     id {:032b}", id.get_id().0);
+                    }
+
+                    // See if somebody beat us to creating our
+                    // node already, if so, we still need to do
+                    // work: we have to update the bitmap index
+                    // with the multi_uniq_id we've got from the
+                    // caller.
+                    if id == stored_node.node_id {
+                        stored_node
+                            .node_set
+                            .update_rbm_index(multi_uniq_id)?;
+
+                        if next_node.ptrbitarr.load() != 0 {
+                            stored_node
+                                .node
+                                .ptrbitarr
+                                .merge_with(next_node.ptrbitarr.load());
+                        }
+                        if next_node.pfxbitarr.load() != 0 {
+                            stored_node
+                                .node
+                                .pfxbitarr
+                                .merge_with(next_node.pfxbitarr.load());
+                        }
+
+                        return Ok((id, retry_count));
+                    } else {
+                        // it's not "our" node, make a (recursive)
+                        // call to create it.
+                        level += 1;
+                        trace!(
+                            "Collision with node_id {}, move to next level:
+                            {} len{} next_lvl{} index {}",
+                            stored_node.node_id,
+                            id,
+                            id.get_id().1,
+                            level,
+                            index
+                        );
+
+                        level += 1;
+                        match <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            id.get_id().1,
+                            level,
+                        ) {
+                            // on to the next level!
+                            next_bit_shift if next_bit_shift > 0 => {
+                                nodes = &stored_node.node_set;
+                                // (search_level.f)(
+                                //     search_level,
+                                //     &stored_node.node_set,
+                                //     new_node,
+                                //     multi_uniq_id,
+                                //     level,
+                                //     retry_count,
+                                // )
+                            }
+                            // There's no next level!
+                            _ => panic!(
+                                "out of storage levels, current level is {}",
+                                level
+                            ),
+                        };
+                    }
+                }
+            }
+        }
+    }
+
+    fn store_node(
+        &self,
+        id: StrideNodeId<AF>,
+        multi_uniq_id: u32,
+        next_node: TreeBitMapNode<AF, Stride4>,
+    ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
+        if log_enabled!(log::Level::Trace) {
+            debug!(
+                "{} store: Store node {}: {:?} mui {}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                id,
+                next_node,
+                multi_uniq_id
+            );
+        }
+        self.counters.inc_nodes_count();
+
+        let mut nodes = self.node_buckets.get_store4(id);
+        let new_node = next_node;
+        let mut level = 0;
+        let mut retry_count = 0;
+
+        loop {
+            let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
+                id.get_id().1,
+                level,
+            );
+            trace!("{:032b}", id.get_id().0);
+            trace!("id {:?}", id.get_id());
+            trace!("multi_uniq_id {}", multi_uniq_id);
+
+            // HASHING FUNCTION
+            let index = Self::hash_node_id(id, level);
+
+            match nodes.read().get(index) {
+                None => {
+                    // No node exists, so we create one here.
+                    let next_level =
+                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            id.get_id().1,
+                            level + 1,
+                        );
+
+                    if log_enabled!(log::Level::Trace) {
+                        trace!(
+                            "Empty node found,
+                            creating new node {} len{} lvl{}",
+                            id,
+                            id.get_id().1,
+                            level + 1
+                        );
+                        trace!("Next level {}", next_level);
+                        trace!(
+                            "Creating space for {} nodes",
+                            if next_level >= this_level {
+                                1 << (next_level - this_level)
+                            } else {
+                                1
+                            }
+                        );
+                    }
+
+                    trace!("multi uniq id {}", multi_uniq_id);
+
+                    let node_set = NodeSet::init(next_level - this_level);
+
+                    let ptrbitarr = new_node.ptrbitarr.load();
+                    let pfxbitarr = new_node.pfxbitarr.load();
+
+                    let (stored_node, its_us) =
+                        nodes.read().get_or_init(index, || StoredNode {
+                            node_id: id,
+                            node: new_node,
+                            node_set,
+                        });
+
+                    if stored_node.node_id == id {
+                        stored_node
+                            .node_set
+                            .update_rbm_index(multi_uniq_id)?;
+
+                        if !its_us && ptrbitarr != 0 {
+                            stored_node.node.ptrbitarr.merge_with(ptrbitarr);
+                        }
+
+                        if !its_us && pfxbitarr != 0 {
+                            stored_node.node.pfxbitarr.merge_with(pfxbitarr);
+                        }
+                    }
+
+                    return Ok((id, retry_count));
+                }
+                Some(stored_node) => {
+                    // A node exists, might be ours, might be
+                    // another one.
+
+                    if log_enabled!(log::Level::Trace) {
+                        trace!(
+                            "
+                                {} store: Node here exists {:?}",
+                            std::thread::current()
+                                .name()
+                                .unwrap_or("unnamed-thread"),
+                            stored_node.node_id
+                        );
+                        trace!("node_id {:?}", stored_node.node_id.get_id());
+                        trace!(
+                            "node_id {:032b}",
+                            stored_node.node_id.get_id().0
+                        );
+                        trace!("id {}", id);
+                        trace!("     id {:032b}", id.get_id().0);
+                    }
+
+                    // See if somebody beat us to creating our
+                    // node already, if so, we still need to do
+                    // work: we have to update the bitmap index
+                    // with the multi_uniq_id we've got from the
+                    // caller.
+                    if id == stored_node.node_id {
+                        stored_node
+                            .node_set
+                            .update_rbm_index(multi_uniq_id)?;
+
+                        if new_node.ptrbitarr.load() != 0 {
+                            stored_node
+                                .node
+                                .ptrbitarr
+                                .merge_with(new_node.ptrbitarr.load());
+                        }
+                        if new_node.pfxbitarr.load() != 0 {
+                            stored_node
+                                .node
+                                .pfxbitarr
+                                .merge_with(new_node.pfxbitarr.load());
+                        }
+
+                        return Ok((id, retry_count));
+                    } else {
+                        // it's not "our" node, make a (recursive)
+                        // call to create it.
+                        level += 1;
+                        trace!(
+                            "Collision with node_id {},
+                                 move to next level: {} len{} next_lvl{}
+                                 index {}",
+                            stored_node.node_id,
+                            id,
+                            id.get_id().1,
+                            level,
+                            index
+                        );
+
+                        match <NB as NodeBuckets<AF>>::len_to_store_bits(
+                            id.get_id().1,
+                            level,
+                        ) {
+                            // on to the next level!
+                            next_bit_shift if next_bit_shift > 0 => {
+                                nodes = &stored_node.node_set;
+                            }
+                            // There's no next level!
+                            _ => panic!(
+                                "out of storage levels,
+                                    current level is {}",
+                                level
+                            ),
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     // Create a new node in the store with payload `next_node`.
     //
     // Next node will be ignored if a node with the same `id` already exists,
@@ -703,11 +1062,11 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
     // Returns: a tuple with the node_id of the created node and the number of
     // retry_count
     #[allow(clippy::type_complexity)]
-    fn store_node(
+    fn _old_old_store_node(
         &self,
         id: StrideNodeId<AF>,
         multi_uniq_id: u32,
-        next_node: SizedStrideNode<AF>,
+        next_node: TreeBitMapNode<AF, Stride4>,
     ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
         struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
             f: &'s dyn Fn(
@@ -741,32 +1100,32 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         }
         self.counters.inc_nodes_count();
 
-        match next_node {
-            SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
-                &search_level_3,
-                self.node_buckets.get_store3(id),
-                new_node,
-                multi_uniq_id,
-                0,
-                0,
-            ),
-            SizedStrideNode::Stride4(new_node) => (search_level_4.f)(
-                &search_level_4,
-                self.node_buckets.get_store4(id),
-                new_node,
-                multi_uniq_id,
-                0,
-                0,
-            ),
-            SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
-                &search_level_5,
-                self.node_buckets.get_store5(id),
-                new_node,
-                multi_uniq_id,
-                0,
-                0,
-            ),
-        }
+        // match next_node {
+        // SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
+        //     &search_level_3,
+        //     self.node_buckets.get_store3(id),
+        //     new_node,
+        //     multi_uniq_id,
+        //     0,
+        //     0,
+        // ),
+        (search_level_4.f)(
+            &search_level_4,
+            self.node_buckets.get_store4(id),
+            next_node,
+            multi_uniq_id,
+            0,
+            0,
+        )
+        // SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
+        //     &search_level_5,
+        //     self.node_buckets.get_store5(id),
+        //     new_node,
+        //     multi_uniq_id,
+        //     0,
+        //     0,
+        // ),
+        // }
     }
 
     #[allow(clippy::type_complexity)]

From b9de32f400a7bbf5a2c2521544a147bc6a7a4e1d Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Feb 2025 17:23:48 +0100
Subject: [PATCH 087/147] remove Sized* enum wrappers

---
 proc_macros/src/lib.rs                    |  42 +-
 src/local_array/in_memory/atomic_types.rs |  29 +-
 src/local_array/in_memory/iterators.rs    | 101 +--
 src/local_array/in_memory/macros.rs       | 305 ++++-----
 src/local_array/in_memory/node.rs         | 627 +----------------
 src/local_array/in_memory/tree.rs         | 788 ++++------------------
 src/local_array/prefix_cht/iterators.rs   |  80 +--
 src/local_array/rib/default_store.rs      |   2 +
 8 files changed, 350 insertions(+), 1624 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index f860a504..70d0963d 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -44,14 +44,6 @@ pub fn stride_sizes(
         l => panic!("Expected Key Size for Address Family, got {:?}", l),
     };
 
-    // let _config = match attrs
-    //     .get(4)
-    //     .unwrap_or_else(|| panic!("Missing config type for store"))
-    // {
-    //     syn::Expr::Path(p) => p,
-    //     p => panic!("Expected Config type, got {:?}", p),
-    // };
-
     let key_type = match attrs
         .get(4)
         .unwrap_or_else(|| panic!("Missing Key Type for Persist Strategy"))
@@ -145,7 +137,6 @@ pub fn stride_sizes(
         _ => panic!("Expected an array"),
     };
     let strides_len = attrs_s.elems.len() as u8;
-    let first_stride_size = &attrs_s.elems[0];
 
     for (len, stride) in attrs_s.elems.iter().enumerate() {
         strides_all_len.push(format_ident!("l{}", len));
@@ -280,18 +271,7 @@ pub fn stride_sizes(
                 }
             }
 
-            fn get_store3(&self, id: StrideNodeId<#ip_af>) -> &NodeSet<#ip_af, Stride3> {
-                match id.get_id().1 as usize {
-                    #( #strides_len3 => &self.#strides_len3_l, )*
-                    _ => panic!(
-                        "unexpected sub prefix length {} in stride size 3 ({})",
-                        id.get_id().1,
-                        id
-                    ),
-                }
-            }
-
-            fn get_store4(&self, id: StrideNodeId<#ip_af>) -> &NodeSet<#ip_af, Stride4> {
+            fn get_store(&self, id: StrideNodeId<#ip_af>) -> &NodeSet<#ip_af, Stride4> {
                 match id.get_id().1 as usize {
                     #( #strides_len4 => &self.#strides_len4_l, )*
                     // ex.:
@@ -304,20 +284,6 @@ pub fn stride_sizes(
                 }
             }
 
-            fn get_store5(&self, id: StrideNodeId<#ip_af>) -> &NodeSet<#ip_af, Stride5> {
-                match id.get_id().1 as usize {
-                    #( #strides_len5 => &self.#strides_len5_l, )*
-                    // ex.:
-                    // 0 => &self.l0,
-                    // 5 => &self.l5,
-                    _ => panic!(
-                        "unexpected sub prefix length {} in stride size 5 ({})",
-                        id.get_id().1,
-                        id
-                    ),
-                }
-            }
-
             #[inline]
             fn get_stride_sizes(&self) -> &[u8] {
                 &self.stride_sizes[0..self.strides_len as usize]
@@ -335,9 +301,9 @@ pub fn stride_sizes(
                 #strides_len
             }
 
-            fn get_first_stride_size() -> u8 {
-                #first_stride_size
-            }
+            // fn get_first_stride_size() -> u8 {
+            //     #first_stride_size
+            // }
         }
 
     };
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 75702e9f..8916ca77 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -23,7 +23,7 @@ use super::super::errors::PrefixStoreError;
 use super::atomic_stride;
 use super::node::{StrideNodeId, TreeBitMapNode};
 use super::oncebox::OnceBoxSlice;
-use super::tree::{Stride, Stride3, Stride4, Stride5};
+use super::tree::{Stride, Stride4};
 
 // ----------- Node related structs -----------------------------------------
 
@@ -255,14 +255,6 @@ impl<AF: AddressFamily, M: crate::prefix_record::Meta> StoredPrefix<AF, M> {
 
         Ok(path_selection_muis)
     }
-
-    pub(crate) fn get_next_bucket(&self) -> Option<&PrefixSet<AF, M>> {
-        if self.next_bucket.is_empty() {
-            None
-        } else {
-            Some(&self.next_bucket)
-        }
-    }
 }
 
 #[derive(Clone, Debug)]
@@ -587,11 +579,11 @@ pub trait NodeBuckets<AF: AddressFamily> {
     fn len_to_store_bits(len: u8, level: u8) -> u8;
     fn get_stride_sizes(&self) -> &[u8];
     fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8;
-    fn get_store3(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride3>;
-    fn get_store4(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride4>;
-    fn get_store5(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride5>;
+    // fn get_store3(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride3>;
+    fn get_store(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride4>;
+    // fn get_store5(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride5>;
     fn get_strides_len() -> u8;
-    fn get_first_stride_size() -> u8;
+    // fn get_first_stride_size() -> u8;
 }
 
 pub trait PrefixBuckets<AF: AddressFamily, M: Meta>
@@ -624,15 +616,4 @@ impl<AF: AddressFamily, M: Meta> PrefixSet<AF, M> {
     pub fn init(p2_size: u8) -> Self {
         PrefixSet(OnceBoxSlice::new(p2_size))
     }
-
-    pub(crate) fn is_empty(&self) -> bool {
-        self.0.is_null()
-    }
-
-    pub(crate) fn get_by_index(
-        &self,
-        index: usize,
-    ) -> Option<&StoredPrefix<AF, M>> {
-        self.0.get(index)
-    }
 }
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index ae563334..d019074c 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -20,10 +20,8 @@
 // in the TreeBitMap.
 
 use crate::local_array::in_memory::atomic_types::NodeBuckets;
-use crate::local_array::in_memory::node::{SizedStrideRef, StrideNodeId};
-use crate::local_array::in_memory::tree::{
-    Stride3, Stride4, Stride5, TreeBitMap,
-};
+use crate::local_array::in_memory::tree::Stride4;
+use crate::local_array::in_memory::tree::TreeBitMap;
 use crate::{
     af::AddressFamily,
     local_array::{
@@ -38,45 +36,6 @@ use crate::{
 use inetnum::addr::Prefix;
 use log::{log_enabled, trace};
 
-// ----------- Sized Wrappers -----------------------------------------------
-
-// These are enums to abstract over the Stride Size of the iterators. Each
-// iterator in here need to go over iterators that have different underlying
-// stride sizes. To facilitate this these wrapper enums exist.
-
-#[derive(Copy, Clone, Debug)]
-pub(crate) enum SizedNodeMoreSpecificIter<AF: AddressFamily> {
-    Stride3(NodeMoreSpecificChildIter<AF, Stride3>),
-    Stride4(NodeMoreSpecificChildIter<AF, Stride4>),
-    Stride5(NodeMoreSpecificChildIter<AF, Stride5>),
-}
-
-impl<AF: AddressFamily> SizedNodeMoreSpecificIter<AF> {
-    pub(crate) fn next(&mut self) -> Option<StrideNodeId<AF>> {
-        match self {
-            SizedNodeMoreSpecificIter::Stride3(iter) => iter.next(),
-            SizedNodeMoreSpecificIter::Stride4(iter) => iter.next(),
-            SizedNodeMoreSpecificIter::Stride5(iter) => iter.next(),
-        }
-    }
-}
-
-pub(crate) enum SizedPrefixIter<AF: AddressFamily> {
-    Stride3(NodeMoreSpecificsPrefixIter<AF, Stride3>),
-    Stride4(NodeMoreSpecificsPrefixIter<AF, Stride4>),
-    Stride5(NodeMoreSpecificsPrefixIter<AF, Stride5>),
-}
-
-impl<AF: AddressFamily> SizedPrefixIter<AF> {
-    pub(crate) fn next(&mut self) -> Option<PrefixId<AF>> {
-        match self {
-            SizedPrefixIter::Stride3(iter) => iter.next(),
-            SizedPrefixIter::Stride4(iter) => iter.next(),
-            SizedPrefixIter::Stride5(iter) => iter.next(),
-        }
-    }
-}
-
 // ----------- MoreSpecificPrefixIter ------------------------------------
 
 // A iterator over all the more-specifics for a given prefix.
@@ -101,9 +60,9 @@ pub(crate) struct MoreSpecificPrefixIter<
     NB: NodeBuckets<AF>,
 > {
     tree: &'a TreeBitMap<AF, NB>,
-    cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
-    cur_pfx_iter: SizedPrefixIter<AF>,
-    parent_and_position: Vec<SizedNodeMoreSpecificIter<AF>>,
+    cur_ptr_iter: NodeMoreSpecificChildIter<AF, Stride4>,
+    cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF, Stride4>,
+    parent_and_position: Vec<NodeMoreSpecificChildIter<AF, Stride4>>,
 }
 
 impl<'a, AF: AddressFamily + 'a, NB: NodeBuckets<AF>> Iterator
@@ -176,19 +135,17 @@ impl<'a, AF: AddressFamily + 'a, NB: NodeBuckets<AF>> Iterator
                             next_ptr,
                             BitSpan { bits: 0, len: 0 },
                         );
-                        self.cur_ptr_iter = ptr_iter.wrap();
+                        self.cur_ptr_iter = ptr_iter;
 
                         trace!(
                             "next stride new iterator stride 4 {:?} start \
                         bit_span 0 0",
                             self.cur_ptr_iter,
                         );
-                        self.cur_pfx_iter = next_node
-                            .more_specific_pfx_iter(
-                                next_ptr,
-                                BitSpan::new(0, 0),
-                            )
-                            .wrap();
+                        self.cur_pfx_iter = next_node.more_specific_pfx_iter(
+                            next_ptr,
+                            BitSpan::new(0, 0),
+                        );
                     }
                     // Some(SizedStrideRef::Stride5(next_node)) => {
                     //     // create new ptr iterator for this node.
@@ -347,37 +304,19 @@ impl<
                 start_bs.len
             );
 
-            let cur_pfx_iter: SizedPrefixIter<AF>;
-            let cur_ptr_iter: SizedNodeMoreSpecificIter<AF>;
+            let cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF, Stride4>;
+            let cur_ptr_iter: NodeMoreSpecificChildIter<AF, Stride4>;
             let node = self.retrieve_node(start_node_id);
 
             if let Some(node) = node {
-                match node {
-                    // SizedStrideRef::Stride3(n) => {
-                    //     cur_pfx_iter = SizedPrefixIter::Stride3(
-                    //         n.more_specific_pfx_iter(start_node_id, start_bs),
-                    //     );
-                    //     cur_ptr_iter = SizedNodeMoreSpecificIter::Stride3(
-                    //         n.more_specific_ptr_iter(start_node_id, start_bs),
-                    //     );
-                    // }
-                    n => {
-                        cur_pfx_iter = SizedPrefixIter::Stride4(
-                            n.more_specific_pfx_iter(start_node_id, start_bs),
-                        );
-                        trace!("---------------------");
-                        trace!("start iterating nodes");
-                        cur_ptr_iter = SizedNodeMoreSpecificIter::Stride4(
-                            n.more_specific_ptr_iter(start_node_id, start_bs),
-                        );
-                    } // SizedStrideRef::Stride5(n) => {
-                      //     cur_pfx_iter = SizedPrefixIter::Stride5(
-                      //         n.more_specific_pfx_iter(start_node_id, start_bs),
-                      //     );
-                      //     cur_ptr_iter = SizedNodeMoreSpecificIter::Stride5(
-                      //         n.more_specific_ptr_iter(start_node_id, start_bs),
-                      //     );
-                      // }
+                let n = node;
+                {
+                    cur_pfx_iter =
+                        n.more_specific_pfx_iter(start_node_id, start_bs);
+                    trace!("---------------------");
+                    trace!("start iterating nodes");
+                    cur_ptr_iter =
+                        n.more_specific_ptr_iter(start_node_id, start_bs);
                 };
 
                 Some(MoreSpecificPrefixIter {
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index 02f50529..162b212f 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -1,163 +1,158 @@
-#[macro_export]
-// This macro expands into a match node {}
-// with match arms for all SizedStrideNode::Stride[3-8]
-// for use in insert()
-#[doc(hidden)]
-macro_rules! insert_match {
-    (
-        $self: ident;
-        $guard: ident;
-        $bit_span: expr;
-        // $nibble_len: expr;
-        // $nibble: expr; // nibble is a variable-length bitarray (1,2,4,8,etc)
-        $is_last_stride: expr;
-        $pfx: ident; // the whole search prefix
-        $mui: ident; // the reccord holding the metadata
-        // $update_path_selections: ident; // boolean indicate whether to update the path selections for this route
-        $truncate_len: ident; // the start of the length of this stride
-        $stride_len: ident; // the length of this stride
-        $cur_i: expr; // the id of the current node in this stride
-        $level: expr;
-        $acc_retry_count: expr;
-        // $enum: ident;
-        // The strides to generate match arms for,
-        // $variant is the name of the enum varian (Stride[3..8]) and
-        // $len is the index of the stats level, so 0..5
-        $( $variant: ident; $stats_level: expr ), *
-    ) => {
-        // Look up the current node in the store. This should never fail,
-        // since we're starting at the root node and retrieve that. If a node
-        // does not exist, it is created here. BUT, BUT, in a multi-threaded
-        // context, one thread creating the node may be outpaced by a thread
-        // reading the same node. Because the creation of a node actually
-        // consists of two independent atomic operations (first setting the
-        // right bit in the parent bitarray, second storing the node in the
-        // store with the meta-data), a thread creating a new node may have
-        // altered the parent bitarray, but not it didn't create the node
-        // in the store yet. The reading thread, however, saw the bit in the
-        // parent and wants to read the node in the store, but that doesn't
-        // exist yet. In that case, the reader thread needs to try again
-        // until it is actually created
-
-        // This macro counts the number of retries and adds that to the
-        // $acc_retry_count variable, to be used by the incorporating
-        // function.
-        {
-            // this counts the number of retry_count for this loop only,
-            // but ultimately we will return the accumulated count of all
-            // retry_count from this macro.
-            let local_retry_count = 0;
-            // retrieve_node_mut updates the bitmap index if necessary.
-            if let Some(current_node) = $self.retrieve_node_mut(
-                $cur_i, $mui) {
-                match current_node {
-                    $(
-                        SizedStrideRef::$variant(current_node) => {
-                            // eval_node_or_prefix_at mutates the node to
-                            // reflect changes in the ptrbitarr & pfxbitarr.
-                            match current_node.eval_node_or_prefix_at(
-                                $bit_span,
-                                // All the bits of the search prefix, but with
-                                // a length set to the start of the current
-                                // stride.
-                                StrideNodeId::dangerously_new_with_id_as_is(
-                                    $pfx.get_net(), $truncate_len),
-                                // the length of THIS stride
-                                $stride_len,
-                                // the length of the next stride
-                                $self
-                                    .get_stride_sizes()
-                                    .get(($level + 1) as usize),
-                                $is_last_stride,
-                            ) {
-                                (NewNodeOrIndex::NewNode(n), retry_count) => {
-                                    // Stride3 logs to stats[0], Stride4 logs
-                                    // to stats[1], etc.
-                                    // $self.stats[$stats_level].inc($level);
+// #[macro_export]
+// // This macro expands into a match node {}
+// // with match arms for all SizedStrideNode::Stride[3-8]
+// // for use in insert()
+// #[doc(hidden)]
+// macro_rules! insert_match {
+//     (
+//         $self: ident;
+//         $guard: ident;
+//         $bit_span: expr;
+//         $is_last_stride: expr;
+//         $pfx: ident; // the whole search prefix
+//         $mui: ident; // the reccord holding the metadata
+//         $truncate_len: ident; // the start of the length of this stride
+//         $stride_len: ident; // the length of this stride
+//         $cur_i: expr; // the id of the current node in this stride
+//         $level: expr;
+//         $acc_retry_count: expr;
+//         // The strides to generate match arms for,
+//         // $variant is the name of the enum varian (Stride[3..8]) and
+//         // $len is the index of the stats level, so 0..5
+//         $( $variant: ident; $stats_level: expr ), *
+//     ) => {
+//         // Look up the current node in the store. This should never fail,
+//         // since we're starting at the root node and retrieve that. If a node
+//         // does not exist, it is created here. BUT, BUT, in a multi-threaded
+//         // context, one thread creating the node may be outpaced by a thread
+//         // reading the same node. Because the creation of a node actually
+//         // consists of two independent atomic operations (first setting the
+//         // right bit in the parent bitarray, second storing the node in the
+//         // store with the meta-data), a thread creating a new node may have
+//         // altered the parent bitarray, but not it didn't create the node
+//         // in the store yet. The reading thread, however, saw the bit in the
+//         // parent and wants to read the node in the store, but that doesn't
+//         // exist yet. In that case, the reader thread needs to try again
+//         // until it is actually created
 
-                                    // get a new identifier for the node we're
-                                    // going to create.
-                                    let new_id =
-                                        StrideNodeId::new_with_cleaned_id(
-                                            $pfx.get_net(),
-                                            $truncate_len + $bit_span.len
-                                       );
+//         // This macro counts the number of retries and adds that to the
+//         // $acc_retry_count variable, to be used by the incorporating
+//         // function.
+//         {
+//             // this counts the number of retry_count for this loop only,
+//             // but ultimately we will return the accumulated count of all
+//             // retry_count from this macro.
+//             let local_retry_count = 0;
+//             // retrieve_node_mut updates the bitmap index if necessary.
+//             if let Some(current_node) = $self.retrieve_node_mut(
+//                 $cur_i, $mui) {
+//                 match current_node {
+//                     $(
+//                         SizedStrideRef::$variant(current_node) => {
+//                             // eval_node_or_prefix_at mutates the node to
+//                             // reflect changes in the ptrbitarr & pfxbitarr.
+//                             match current_node.eval_node_or_prefix_at(
+//                                 $bit_span,
+//                                 // All the bits of the search prefix, but with
+//                                 // a length set to the start of the current
+//                                 // stride.
+//                                 StrideNodeId::dangerously_new_with_id_as_is(
+//                                     $pfx.get_net(), $truncate_len),
+//                                 // the length of THIS stride
+//                                 $stride_len,
+//                                 // the length of the next stride
+//                                 $self
+//                                     .get_stride_sizes()
+//                                     .get(($level + 1) as usize),
+//                                 $is_last_stride,
+//                             ) {
+//                                 (NewNodeOrIndex::NewNode(n), retry_count) => {
+//                                     // Stride3 logs to stats[0], Stride4 logs
+//                                     // to stats[1], etc.
+//                                     // $self.stats[$stats_level].inc($level);
 
-                                    // store the new node in the in_memory
-                                    // part of the RIB. It returns the created
-                                    // id and the number of retries before
-                                    // success.
-                                    match $self.store_node(
-                                        new_id,
-                                        $mui, n
-                                    ) {
-                                        Ok((node_id, s_retry_count)) => {
-                                            Ok((
-                                                node_id,
-                                                $acc_retry_count +
-                                                s_retry_count +
-                                                retry_count
-                                            ))
-                                        },
-                                        Err(err) => {
-                                            Err(err)
-                                        }
-                                    }
-                                }
-                                (NewNodeOrIndex::ExistingNode(node_id),
-                                    retry_count
-                                ) => {
-                                    if log_enabled!(log::Level::Trace) {
-                                        if local_retry_count > 0 {
-                                            trace!("{} contention: Node \
-                                                 already exists {}",
-                                            std::thread::current()
-                                                .name()
-                                                .unwrap_or("unnamed-thread"),
-                                                node_id
-                                            )
-                                        }
-                                    }
-                                    Ok((
-                                        node_id,
-                                        $acc_retry_count +
-                                        local_retry_count +
-                                        retry_count
-                                    ))
-                                },
-                                (NewNodeOrIndex::NewPrefix, retry_count) => {
-                                    break (
-                                        $acc_retry_count +
-                                        local_retry_count +
-                                        retry_count,
-                                        false
-                                    )
+//                                     // get a new identifier for the node we're
+//                                     // going to create.
+//                                     let new_id =
+//                                         StrideNodeId::new_with_cleaned_id(
+//                                             $pfx.get_net(),
+//                                             $truncate_len + $bit_span.len
+//                                        );
 
-                                },
-                                (
-                                    NewNodeOrIndex::ExistingPrefix,
-                                     retry_count
-                                 ) =>
-                                     {
-                                    break (
-                                        $acc_retry_count +
-                                        local_retry_count +
-                                        retry_count,
-                                        true
-                                    )
+//                                     // store the new node in the in_memory
+//                                     // part of the RIB. It returns the created
+//                                     // id and the number of retries before
+//                                     // success.
+//                                     match $self.store_node(
+//                                         new_id,
+//                                         $mui, n
+//                                     ) {
+//                                         Ok((node_id, s_retry_count)) => {
+//                                             Ok((
+//                                                 node_id,
+//                                                 $acc_retry_count +
+//                                                 s_retry_count +
+//                                                 retry_count
+//                                             ))
+//                                         },
+//                                         Err(err) => {
+//                                             Err(err)
+//                                         }
+//                                     }
+//                                 }
+//                                 (NewNodeOrIndex::ExistingNode(node_id),
+//                                     retry_count
+//                                 ) => {
+//                                     if log_enabled!(log::Level::Trace) {
+//                                         if local_retry_count > 0 {
+//                                             trace!("{} contention: Node \
+//                                                  already exists {}",
+//                                             std::thread::current()
+//                                                 .name()
+//                                                 .unwrap_or("unnamed-thread"),
+//                                                 node_id
+//                                             )
+//                                         }
+//                                     }
+//                                     Ok((
+//                                         node_id,
+//                                         $acc_retry_count +
+//                                         local_retry_count +
+//                                         retry_count
+//                                     ))
+//                                 },
+//                                 (NewNodeOrIndex::NewPrefix, retry_count) => {
+//                                     break (
+//                                         $acc_retry_count +
+//                                         local_retry_count +
+//                                         retry_count,
+//                                         false
+//                                     )
 
+//                                 },
+//                                 (
+//                                     NewNodeOrIndex::ExistingPrefix,
+//                                      retry_count
+//                                  ) =>
+//                                      {
+//                                     break (
+//                                         $acc_retry_count +
+//                                         local_retry_count +
+//                                         retry_count,
+//                                         true
+//                                     )
 
-                                }
-                            }   // end of eval_node_or_prefix_at
-                        }
-                    )*,
-                }
-            } else {
-                Err(PrefixStoreError::NodeCreationMaxRetryError)
-            }
-        }
-    }
-}
+//                                 }
+//                             }   // end of eval_node_or_prefix_at
+//                         }
+//                     )*,
+//                 }
+//             } else {
+//                 Err(PrefixStoreError::NodeCreationMaxRetryError)
+//             }
+//         }
+//     }
+// }
 
 #[macro_export]
 // This macro only works for stride with bitmaps that are <= u128,
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index e895bb37..558bbd64 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -1,15 +1,13 @@
 use num_traits::PrimInt;
-use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, AtomicU8};
+use std::sync::atomic::{AtomicU16, AtomicU32};
 use std::{fmt::Debug, marker::PhantomData};
 
 use log::{log_enabled, trace};
 use parking_lot_core::SpinWait;
 
 use super::super::bit_span::BitSpan;
-use super::iterators::{SizedNodeMoreSpecificIter, SizedPrefixIter};
 use super::tree::{
-    AtomicBitmap, AtomicStride2, AtomicStride3, AtomicStride4, AtomicStride5,
-    CasResult, Stride, Stride3, Stride4, Stride5,
+    AtomicBitmap, AtomicStride3, AtomicStride4, CasResult, Stride, Stride4,
 };
 
 use crate::af::AddressFamily;
@@ -136,13 +134,10 @@ where
     pub(crate) fn eval_node_or_prefix_at(
         &self,
         bit_span: BitSpan,
-        // nibble: u32,
-        // nibble_len: u8,
         // all the bits of the search prefix, but with the length set to
         // the length of this stride. So bits are set beyond its length.
         base_prefix: StrideNodeId<AF>,
-        stride_len: u8,
-        next_stride: Option<&u8>,
+        // stride_len: u8,
         is_last_stride: bool,
     ) -> (NewNodeOrIndex<AF>, u32) {
         // THE CRITICAL SECTION
@@ -160,7 +155,7 @@ where
         let ptrbitarr = self.ptrbitarr.load();
         let pfxbitarr = self.pfxbitarr.load();
         let bit_pos = S::get_bit_pos(bit_span);
-        let new_node: SizedStrideNode<AF>;
+        let new_node: TreeBitMapNode<AF, Stride4>;
 
         // Check that we're not at the last stride (pfx.len <= stride_end),
         // Note that next_stride may have a value, but we still don't want to
@@ -177,35 +172,10 @@ where
                 <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::
                     InnerType>::zero() {
                 // Nope, set it and create a child node
-
-                match next_stride.unwrap() {
-                    3_u8 => {
-                        new_node = SizedStrideNode::Stride3(TreeBitMapNode {
-                            ptrbitarr: AtomicStride2(AtomicU8::new(0)),
-                            pfxbitarr: AtomicStride3(AtomicU16::new(0)),
-                            // pfx_vec: PrefixSet::empty(14),
-                            _af: PhantomData,
-                        });
-                    }
-                    4_u8 => {
-                        new_node = SizedStrideNode::Stride4(TreeBitMapNode {
-                            ptrbitarr: AtomicStride3(AtomicU16::new(0)),
-                            pfxbitarr: AtomicStride4(AtomicU32::new(0)),
-                            // pfx_vec: PrefixSet::empty(30),
-                            _af: PhantomData,
-                        });
-                    }
-                    5_u8 => {
-                        new_node = SizedStrideNode::Stride5(TreeBitMapNode {
-                            ptrbitarr: AtomicStride4(AtomicU32::new(0)),
-                            pfxbitarr: AtomicStride5(AtomicU64::new(0)),
-                            // pfx_vec: PrefixSet::empty(62),
-                            _af: PhantomData,
-                        });
-                    }
-                    _ => {
-                        panic!("can't happen");
-                    }
+                new_node = TreeBitMapNode {
+                    ptrbitarr: AtomicStride3(AtomicU16::new(0)),
+                    pfxbitarr: AtomicStride4(AtomicU32::new(0)),
+                    _af: PhantomData,
                 };
 
                 // THE CRITICAL SECTION
@@ -290,412 +260,11 @@ where
         // base_prefix (which is always the start length of the stride).
         (
             NewNodeOrIndex::ExistingNode(
-                base_prefix.add_to_len(stride_len).truncate_to_len(),
+                base_prefix.add_to_len(4).truncate_to_len(),
             ),
             retry_count,
         )
     }
-
-    //-------- Search nibble functions --------------------------------------
-
-    // This function looks for the longest marching prefix in the provided
-    // nibble, by iterating over all the bits in it and comparing that with
-    // the appropriate bytes from the requested prefix. It mutates the
-    // `less_specifics_vec` that was passed in to hold all the prefixes found
-    // along the way.
-    // pub(crate) fn search_stride_for_longest_match_at(
-    //     &self,
-    //     search_pfx: PrefixId<AF>,
-    //     mut nibble: u32,
-    //     nibble_len: u8,
-    //     start_bit: u8,
-    //     less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
-    // ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
-    //     let pfxbitarr = self.pfxbitarr.load();
-    //     let ptrbitarr = self.ptrbitarr.load();
-    //     let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-    //     let mut found_pfx = None;
-
-    //     trace!("start longest_match search");
-    //     for n_l in 1..(nibble_len + 1) {
-    //         // Move the bit in the right position.
-    //         nibble = AddressFamily::get_nibble(
-    //             search_pfx.get_net(),
-    //             start_bit,
-    //             n_l,
-    //         );
-    //         bit_pos = S::get_bit_pos(nibble, n_l);
-
-    //         // Check if the prefix has been set, if so select the prefix.
-    //         // This is not necessarily the final prefix that will be
-    //         // returned.
-
-    //         // Check it there's a prefix matching in this bitmap for this
-    //         // nibble.
-    //         trace!("pfxbitarr {:032b}", pfxbitarr);
-
-    //         if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize
-    //             as AtomicBitmap>::InnerType::zero() {
-    //             let f_pfx = PrefixId::new(search_pfx.get_net()
-    //                 .truncate_to_len(start_bit + n_l), start_bit + n_l);
-    //             // f_pfx.set_serial(self.get_pfx_serial(f_pfx, nibble, n_l, guard).load(Ordering::Relaxed));
-
-    //             // Receiving a less_specifics_vec means that the user wants
-    //             // to have all the last-specific prefixes returned, so add
-    //             // the found prefix.
-    //             trace!("gather pfx in less_specifics {:?}", f_pfx);
-    //             trace!("ls_vec {:?}", less_specifics_vec);
-    //             if let Some(ls_vec) = less_specifics_vec {
-    //                 trace!("len {}", search_pfx.get_len());
-    //                 trace!("start_bit {}", start_bit);
-    //                 trace!("n_l {}", n_l);
-    //                 trace!("smaller length? {}",
-    //                     search_pfx.get_len() > start_bit + n_l);
-    //                 trace!("{}", (S::into_stride_size(ptrbitarr)
-    //                         & bit_pos)
-    //                         == <<S as Stride>::AtomicPfxSize
-    //                         as AtomicBitmap>::InnerType::zero());
-    //                 if search_pfx.get_len() > start_bit + n_l
-    //                     && (S::into_stride_size(ptrbitarr)
-    //                         & bit_pos)
-    //                         == <<S as Stride>::AtomicPfxSize
-    //                             as AtomicBitmap>::InnerType::zero()
-    //                 {
-    //                     ls_vec.push(f_pfx);
-    //                 }
-    //             }
-
-    //             found_pfx = Some(f_pfx);
-    //         }
-    //     }
-
-    //     let base_prefix = StrideNodeId::new_with_cleaned_id(
-    //         search_pfx.get_net(),
-    //         start_bit,
-    //     );
-
-    //     // Check if this the last stride, or if they're no more children to
-    //     // go to, if so return what we found up until now.
-    //     if search_pfx.get_len() <= start_bit + nibble_len
-    //         || (S::into_stride_size(ptrbitarr) & bit_pos) ==
-    //             <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType
-    //                 ::zero()
-    //     // No children or at the end, return the definitive LMP we found.
-    //     {
-    //         return (
-    //             None,      /* no more children */
-    //             found_pfx, /* The definitive LMP if any */
-    //         );
-    //     }
-
-    //     // There's another child, return it together with the preliminary LMP
-    //     // we found.
-    //     (
-    //         // The identifier of the node that has children of the next
-    //         // stride.
-    //         Some(base_prefix.add_bit_span(BitSpan {
-    //             bits: nibble,
-    //             len: nibble_len,
-    //         })),
-    //         found_pfx,
-    //     )
-    // }
-
-    // This function looks for the exactly matching prefix in the provided
-    // nibble. It doesn't need to iterate over anything it just compares
-    // the complete nibble, with the appropriate bits in the requested
-    // prefix. Although this is rather efficient, there's no way to collect
-    // less-specific prefixes from the search prefix.
-    // pub(crate) fn search_stride_for_exact_match_at(
-    //     &'_ self,
-    //     search_pfx: PrefixId<AF>,
-    //     nibble: u32,
-    //     nibble_len: u8,
-    //     start_bit: u8,
-    //     _: &mut Option<Vec<PrefixId<AF>>>,
-    // ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
-    //     let pfxbitarr = self.pfxbitarr.load();
-    //     let ptrbitarr = self.ptrbitarr.load();
-    //     // This is an exact match, so we're only considering the position of
-    //     // the full nibble.
-    //     let bit_pos = S::get_bit_pos(nibble, nibble_len);
-    //     let mut found_pfx = None;
-    //     let mut found_child = None;
-
-    //     // Is this the last nibble?
-    //     // Otherwise we're not looking for a prefix (exact matching only
-    //     // lives at last nibble)
-    //     match search_pfx.get_len() <= start_bit + nibble_len {
-    //         // We're at the last nibble.
-    //         true => {
-    //             // Check for an actual prefix at the right position, i.e.
-    //             // consider the complete nibble.
-    //             if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-    //                 let f_pfx = PrefixId::new(search_pfx.get_net().truncate_to_len(start_bit + nibble_len), start_bit + nibble_len);
-    //                 found_pfx = Some(f_pfx);
-    //             }
-    //         }
-    //         // We're not at the last nibble.
-    //         false => {
-    //             // Check for a child node at the right position, i.e.
-    //             // consider the complete nibble.
-    //             if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-    //             {
-    //                 found_child = Some(
-    //                     StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)
-    //                 );
-    //             }
-    //         }
-    //     }
-
-    //     (
-    //         found_child, /* The node that has children in the next stride, if
-    //                      any */
-    //         found_pfx, /* The exactly matching prefix, if any */
-    //     )
-    // }
-
-    // pub(crate) fn prefix_exists_in_stride(
-    //     &'_ self,
-    //     search_pfx: PrefixId<AF>,
-    //     nibble: u32,
-    //     nibble_len: u8,
-    //     start_bit: u8,
-    // ) -> (Option<StrideNodeId<AF>>, bool) {
-    //     let pfxbitarr = self.pfxbitarr.load();
-    //     let ptrbitarr = self.ptrbitarr.load();
-    //     // This is an exact match, so we're only considering the position of
-    //     // the full nibble.
-    //     let bit_pos = S::get_bit_pos(nibble, nibble_len);
-    //     let mut found_pfx = false;
-    //     let mut found_child = None;
-
-    //     // Is this the last nibble?
-    //     // Otherwise we're not looking for a prefix (exact matching only
-    //     // lives at last nibble)
-    //     match search_pfx.get_len() <= start_bit + nibble_len {
-    //         // We're at the last nibble.
-    //         true => {
-    //             // Check for an actual prefix at the right position, i.e.
-    //             // consider the complete nibble.
-    //             if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-    //                 found_pfx = true;
-    //             }
-    //         }
-    //         // We're not at the last nibble.
-    //         false => {
-    //             // Check for a child node at the right position, i.e.
-    //             // consider the complete nibble.
-    //             if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-    //             {
-    //                 found_child = Some(
-    //                     StrideNodeId::new_with_cleaned_id(search_pfx.get_net(), start_bit + nibble_len)
-    //                 );
-    //             }
-    //         }
-    //     }
-
-    //     (
-    //         found_child, /* The node that has children in the next stride, if
-    //                      any */
-    //         found_pfx, /* The exactly matching prefix, if any */
-    //     )
-    // }
-
-    // This function looks for the exactly matching prefix in the provided
-    // nibble, just like the one above, but this *does* iterate over all the
-    // bytes in the nibble to collect the less-specific prefixes of the the
-    // search prefix. This is of course slower, so it should only be used
-    // when the user explicitly requests less-specifics.
-    // pub(crate) fn search_stride_for_exact_match_with_less_specifics_at(
-    //     &self,
-    //     search_pfx: PrefixId<AF>,
-    //     mut nibble: u32,
-    //     nibble_len: u8,
-    //     start_bit: u8,
-    //     less_specifics_vec: &mut Option<Vec<PrefixId<AF>>>,
-    // ) -> (Option<StrideNodeId<AF>>, Option<PrefixId<AF>>) {
-    //     let pfxbitarr = self.pfxbitarr.load();
-    //     let ptrbitarr = self.ptrbitarr.load();
-    //     let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-    //     let mut found_pfx = None;
-
-    //     let ls_vec = less_specifics_vec.as_mut().expect(concat!(
-    //         "You shouldn't call this function without",
-    //         "a `less_specifics_vec` buffer. Supply one when calling this
-    //          function",
-    //         "or use `search_stride_for_exact_match_at`"
-    //     ));
-
-    //     for n_l in 1..(nibble_len + 1) {
-    //         // Move the bit in the right position.
-    //         nibble = AddressFamily::get_nibble(
-    //             search_pfx.get_net(),
-    //             start_bit,
-    //             n_l,
-    //         );
-    //         bit_pos = S::get_bit_pos(nibble, n_l);
-
-    //         // Check if the prefix has been set, if so select the prefix.
-    //         // This is not necessarily the final prefix that will be
-    //         // returned.
-
-    //         // Check it there's a prefix matching in this bitmap for this
-    //         // nibble.
-    //         if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as
-    //             AtomicBitmap>::InnerType::zero() {
-    //             // since we want an exact match only, we will fill the prefix
-    //             // field only if we're exactly at the last bit of the nibble
-    //             if n_l == nibble_len {
-    //                 let f_pfx =
-    //                     PrefixId::new(
-    //                         search_pfx.get_net().truncate_to_len(start_bit +
-    //                             n_l), start_bit + n_l);
-    //                     found_pfx = Some(f_pfx);
-    //             }
-
-    //             // Receiving a less_specifics_vec means that the user wants to
-    //             // have all the last-specific prefixes returned, so add the
-    //             // found prefix.
-    //             let f_pfx = PrefixId::new(search_pfx.get_net().
-    //                 truncate_to_len(start_bit + n_l), start_bit + n_l);
-    //             ls_vec.push(f_pfx);
-    //         }
-    //     }
-
-    //     if found_pfx.is_none() {
-    //         // no prefix here, clear out all of the prefixes we found along
-    //         // the way, since it doesn't make sense to return less-specifics
-    //         // if we don't have a exact match.
-    //         ls_vec.clear();
-    //     }
-
-    //     // Check if this the last stride, or if they're no more children to
-    //     // go to, if so return what we found up until now.
-    //     match search_pfx.get_len() <= start_bit + nibble_len
-    //         || (S::into_stride_size(ptrbitarr) & bit_pos)
-    //             == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType
-    //                 as std::ops::BitAnd>::Output::zero()
-    //     {
-    //         // No children or at the end, return the definitive LMP we found.
-    //         true => {
-    //             println!("found {:?}", found_pfx);
-    //             (
-    //             None,      /* no more children */
-    //             found_pfx, /* The definitive LMP if any */
-    //             )
-    //         },
-    //         // There's another child, we won't return the found_pfx, since
-    //         // we're not at the last nibble and we want an exact match only.
-    //         false => (
-    //             Some(StrideNodeId::new_with_cleaned_id(search_pfx.get_net(),
-    //                 start_bit + nibble_len)),
-    //             None,
-    //         ),
-    //     }
-    // }
-
-    // Search a stride for more-specific prefixes and child nodes containing
-    // more specifics for `search_prefix`.
-    // pub(crate) fn add_more_specifics_at(
-    //     &self,
-    //     nibble: u32,
-    //     nibble_len: u8,
-    //     base_prefix: StrideNodeId<AF>,
-    // ) -> (
-    //     Vec<StrideNodeId<AF>>, /* child nodes with more more-specifics in
-    //                            this stride */
-    //     Vec<PrefixId<AF>>, /* more-specific prefixes in this stride */
-    // ) {
-    //     trace!("start adding more specifics");
-    //     let pfxbitarr = self.pfxbitarr.load();
-    //     let ptrbitarr = self.ptrbitarr.load();
-    //     trace!("ptrbitarr {:032b}", ptrbitarr);
-    //     trace!("pfxbitarr {:032b}", pfxbitarr);
-    //     let mut found_children_with_more_specifics = vec![];
-    //     let mut found_more_specifics_vec: Vec<PrefixId<AF>> = vec![];
-
-    //     // This is an exact match, so we're only considering the position of
-    //     // the full nibble.
-    //     let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-    //     let mut found_child = None;
-
-    //     // Is there also a child node here?
-    //     // Note that even without a child node, there may be more specifics
-    //     // further up in this pfxbitarr or children in this ptrbitarr.
-    //     if (S::into_stride_size(ptrbitarr) & bit_pos)
-    //         > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero(
-    //         )
-    //     {
-    //         found_child = Some(base_prefix.add_bit_span(BitSpan {
-    //             bits: nibble,
-    //             len: nibble_len,
-    //         }));
-    //     }
-
-    //     if let Some(child) = found_child {
-    //         found_children_with_more_specifics.push(child);
-    //     }
-
-    //     // We're expanding the search for more-specifics bit-by-bit.
-    //     // `ms_nibble_len` is the number of bits including the original
-    //     // nibble we're considering, e.g. if our prefix has a length of 25
-    //     // and we've all strides sized 4, we would end up with a last
-    //     // nibble_len of 1. `ms_nibble_len` will expand then from 2 up and
-    //     // till 4.
-    //     //
-    //     // ex.:
-    //     // nibble: 1 , (nibble_len: 1)
-    //     // Iteration:
-    //     // ms_nibble_len=1,n_l=0: 10, n_l=1: 11
-    //     // ms_nibble_len=2,n_l=0: 100, n_l=1: 101, n_l=2: 110, n_l=3: 111
-    //     // ms_nibble_len=3,n_l=0: 1000, n_l=1: 1001, n_l=2: 1010, ...,
-    //     // n_l=7: 1111
-
-    //     for ms_nibble_len in nibble_len + 1..=S::STRIDE_LEN {
-    //         // iterate over all the possible values for this `ms_nibble_len`,
-    //         // e.g. two bits can have 4 different values.
-    //         for n_l in 0..(1 << (ms_nibble_len - nibble_len)) {
-    //             // move the nibble left with the amount of bits we're going
-    //             // to loop over. e.g. a stride of size 4 with a nibble 0000
-    //             // 0000 0000 0011 becomes 0000 0000 0000 1100, then it will
-    //             // iterate over ...1100,...1101,...1110,...1111
-    //             let ms_nibble =
-    //                 (nibble << (ms_nibble_len - nibble_len)) + n_l as u32;
-    //             bit_pos = S::get_bit_pos(ms_nibble, ms_nibble_len);
-
-    //             if (S::into_stride_size(ptrbitarr) & bit_pos) > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero()
-    //             {
-    //                 found_children_with_more_specifics.push(
-    //                 base_prefix.add_bit_span(BitSpan {
-    //                     bits: ms_nibble, len: ms_nibble_len})
-    //                 );
-    //             }
-
-    //             if pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType::zero() {
-    //                 found_more_specifics_vec.push(
-    //                     base_prefix.add_bit_span(BitSpan {
-    //                         bits: ms_nibble,
-    //                         len: ms_nibble_len
-    //                     }).into())
-    //             }
-    //         }
-    //     }
-
-    //     trace!(
-    //         "found_children_with_more_specifics {:?}",
-    //         found_children_with_more_specifics
-    //     );
-    //     trace!("found_more_specifics_vec {:?}", found_more_specifics_vec);
-
-    //     (
-    //         // We're done here, the caller should now go over all nodes in
-    //         // found_children_with_more_specifics vec and add ALL prefixes
-    //         // found in there.
-    //         found_children_with_more_specifics,
-    //         found_more_specifics_vec,
-    //     )
-    // }
 }
 
 // ------------ Iterator methods --------------------------------------------
@@ -731,42 +300,6 @@ where
 type PtrBitArr<S> = <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
 type PfxBitArr<S> = <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
 
-// #[derive(Debug, Copy, Clone)]
-// pub(crate) struct NodeChildIter<AF: AddressFamily, S: Stride> {
-//     base_prefix: StrideNodeId<AF>,
-//     ptrbitarr: PtrBitArr<S>,
-//     bit_span: BitSpan, // start with 0
-//     _af: PhantomData<AF>,
-// }
-
-// impl<AF: AddressFamily, S: Stride> std::iter::Iterator
-//     for NodeChildIter<AF, S>
-// {
-//     type Item = StrideNodeId<AF>;
-//     fn next(&mut self) -> Option<Self::Item> {
-//         // iterate over all the possible values for this stride length, e.g.
-//         // two bits can have 4 different values.
-//         for cursor in self.bit_span.bits..(1 << S::STRIDE_LEN) {
-//             // move the bit_span left with the amount of bits we're going to
-//             // loop over.
-//             // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011
-//             // becomes 0000 0000 0000 1100, then it will iterate over
-//             // ...1100,...1101,...1110,...1111
-//             let bit_pos = S::get_bit_pos(cursor, S::STRIDE_LEN);
-//             if (S::into_stride_size(self.ptrbitarr) & bit_pos)
-//                 > PfxBitArr::<S>::zero()
-//             {
-//                 self.bit_span.bits = cursor + 1;
-//                 return Some(self.base_prefix.add_bit_span(BitSpan {
-//                     bits: cursor,
-//                     len: S::STRIDE_LEN,
-//                 }));
-//             }
-//         }
-//         None
-//     }
-// }
-
 // ----------- NodeMoreSpecificChildIter ------------------------------------
 
 // Create an iterator over all the child nodes that hold a more specific
@@ -889,24 +422,6 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
     }
 }
 
-impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride3> {
-    pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
-        SizedNodeMoreSpecificIter::<AF>::Stride3(self)
-    }
-}
-
-impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride4> {
-    pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
-        SizedNodeMoreSpecificIter::<AF>::Stride4(self)
-    }
-}
-
-impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride5> {
-    pub fn wrap(self) -> SizedNodeMoreSpecificIter<AF> {
-        SizedNodeMoreSpecificIter::<AF>::Stride5(self)
-    }
-}
-
 // ----------- NodePrefixIter -----------------------------------------------
 
 // Create an iterator of all prefix ids hosted by this node.
@@ -943,75 +458,6 @@ impl<AF: AddressFamily> NodeMoreSpecificChildIter<AF, Stride5> {
 //     _s: PhantomData<S>,
 // }
 
-// impl<AF: AddressFamily, S: Stride> std::iter::Iterator
-//     for NodePrefixIter<AF, S>
-// {
-//     type Item = PrefixId<AF>;
-
-//     fn next(&mut self) -> Option<Self::Item> {
-//         if self.pfxbitarr == PfxBitArr::<S>::zero() {
-//             return None;
-//         }
-//         if log_enabled!(log::Level::Trace) {
-//             trace!("---- NodePrefixIter -----");
-//             match S::STRIDE_LEN {
-//                 3 => {
-//                     trace!(
-//                         "pfxbitarr  {:016b} ({})",
-//                         self.pfxbitarr,
-//                         S::STRIDE_LEN
-//                     );
-//                 }
-//                 4 => {
-//                     trace!(
-//                         "pfxbitarr  {:032b} ({})",
-//                         self.pfxbitarr,
-//                         S::STRIDE_LEN
-//                     );
-//                 }
-//                 5 => {
-//                     trace!(
-//                         "pfxbitarr  {:064b} ({})",
-//                         self.pfxbitarr,
-//                         S::STRIDE_LEN
-//                     );
-//                 }
-//                 _ => {
-//                     trace!("UNKNOWN STRIDE SIZE ENCOUNTERED.");
-//                 }
-//             };
-//         }
-
-//         let cursor = self.cursor;
-
-//         for i in cursor..S::BITS - 1 {
-//             let bit_pos = S::bit_pos_from_index(i);
-//             self.cursor += 1;
-
-//             if log_enabled!(log::Level::Trace) {
-//                 let bs = BitSpan::from_bit_pos_index(i);
-//                 trace!(
-//                     "{:02}: {:06b} {:064b} bit_span: {:05b} (len: {})",
-//                     i,
-//                     i - 1,
-//                     bit_pos,
-//                     bs.bits,
-//                     bs.len
-//                 );
-//             }
-
-//             if self.pfxbitarr & bit_pos > <<S as Stride>::AtomicPfxSize
-//                 as AtomicBitmap>::InnerType::zero() {
-//                 let bs = BitSpan::from_bit_pos_index(i);
-//                 return Some(self.base_prefix.add_bit_span(bs).into());
-//             }
-//         }
-
-//         trace!("end of iterator NodePrefixIter for this node");
-//         None
-//     }
-// }
-
 pub const fn ms_prefix_mask_arr(bs: BitSpan) -> u32 {
     [
         0b_01111111111111111111111111111110, // bits = 0, len = 0
@@ -1136,37 +582,6 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
     }
 }
 
-impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride3> {
-    pub fn wrap(self) -> SizedPrefixIter<AF> {
-        SizedPrefixIter::<AF>::Stride3(self)
-    }
-}
-
-impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride4> {
-    pub fn wrap(self) -> SizedPrefixIter<AF> {
-        SizedPrefixIter::<AF>::Stride4(self)
-    }
-}
-
-impl<AF: AddressFamily> NodeMoreSpecificsPrefixIter<AF, Stride5> {
-    pub fn wrap(self) -> SizedPrefixIter<AF> {
-        SizedPrefixIter::Stride5(self)
-    }
-}
-
-//------------------- Sized Node Enums ------------------------------------
-
-// No, no, NO, NO, no, no! We're not going to Box this, because that's slow!
-// This enum is never used to store nodes/prefixes, it's only to be used in
-// generic code.
-#[allow(clippy::large_enum_variant)]
-#[derive(Debug)]
-pub enum SizedStrideNode<AF: AddressFamily> {
-    Stride3(TreeBitMapNode<AF, Stride3>),
-    Stride4(TreeBitMapNode<AF, Stride4>),
-    Stride5(TreeBitMapNode<AF, Stride5>),
-}
-
 impl<AF, S> Default for TreeBitMapNode<AF, S>
 where
     AF: AddressFamily,
@@ -1181,29 +596,8 @@ where
     }
 }
 
-impl<AF> Default for SizedStrideNode<AF>
-where
-    AF: AddressFamily,
-{
-    fn default() -> Self {
-        SizedStrideNode::Stride3(TreeBitMapNode {
-            ptrbitarr: AtomicStride2(AtomicU8::new(0)),
-            pfxbitarr: AtomicStride3(AtomicU16::new(0)),
-            _af: PhantomData,
-        })
-    }
-}
-
-// Used to create a public iterator over all nodes.
-#[derive(Debug)]
-pub enum SizedStrideRef<'a, AF: AddressFamily> {
-    Stride3(&'a TreeBitMapNode<AF, Stride3>),
-    Stride4(&'a TreeBitMapNode<AF, Stride4>),
-    Stride5(&'a TreeBitMapNode<AF, Stride5>),
-}
-
 pub(crate) enum NewNodeOrIndex<AF: AddressFamily> {
-    NewNode(SizedStrideNode<AF>),
+    NewNode(TreeBitMapNode<AF, Stride4>),
     ExistingNode(StrideNodeId<AF>),
     NewPrefix,
     ExistingPrefix,
@@ -1240,6 +634,7 @@ impl<AF: AddressFamily> StrideNodeId<AF> {
     pub(crate) fn get_len(&self) -> u8 {
         self.len
     }
+
     pub fn set_len(mut self, len: u8) -> Self {
         self.len = len;
         self
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index e53c4bdb..d4df7d7d 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -185,30 +185,24 @@
 
 use crate::local_array::bit_span::BitSpan;
 use crate::local_array::in_memory::atomic_types::{NodeSet, StoredNode};
+use crate::local_array::rib::default_store::STRIDE_SIZE;
 use crate::prelude::multi::PrefixId;
 use crossbeam_epoch::{Atomic, Guard};
 use log::{debug, error, log_enabled, trace};
-use rayon::iter::MultiZip;
 use roaring::RoaringBitmap;
 
-use std::sync::atomic::{
-    AtomicBool, AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering,
-};
+use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering};
 use std::{fmt::Debug, marker::PhantomData};
 
 use super::atomic_types::NodeBuckets;
 use crate::af::AddressFamily;
 use crate::rib::Counters;
-use crate::{
-    impl_search_level, impl_search_level_for_mui, insert_match,
-    retrieve_node_mut_closure, store_node_closure,
-};
 
 use super::super::errors::PrefixStoreError;
 pub(crate) use super::atomic_stride::*;
 use crate::local_array::in_memory::node::{NewNodeOrIndex, StrideNodeId};
 
-use super::node::{SizedStrideNode, SizedStrideRef, TreeBitMapNode};
+use super::node::TreeBitMapNode;
 
 #[cfg(feature = "cli")]
 use ansi_term::Colour;
@@ -234,36 +228,14 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             _af: PhantomData,
         };
 
-        let root_node = match Self::get_first_stride_size() {
-            // 3 => SizedStrideNode::Stride3(TreeBitMapNode {
-            //     ptrbitarr: AtomicStride2(AtomicU8::new(0)),
-            //     pfxbitarr: AtomicStride3(AtomicU16::new(0)),
-            //     _af: PhantomData,
-            // }),
-            4 => TreeBitMapNode {
+        let _retry_count = tree_bitmap.store_node(
+            StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0),
+            0_u32,
+            TreeBitMapNode {
                 ptrbitarr: AtomicStride3(AtomicU16::new(0)),
                 pfxbitarr: AtomicStride4(AtomicU32::new(0)),
                 _af: PhantomData,
             },
-            _ => {
-                panic!("wut?");
-            } // 5 => SizedStrideNode::Stride5(TreeBitMapNode {
-              //     ptrbitarr: AtomicStride4(AtomicU32::new(0)),
-              //     pfxbitarr: AtomicStride5(AtomicU64::new(0)),
-              //     _af: PhantomData,
-              // }),
-              // unknown_stride_size => {
-              //     panic!(
-              //         "unknown stride size {} encountered in STRIDES array",
-              //         unknown_stride_size
-              //     );
-              // }
-        };
-
-        let _retry_count = tree_bitmap.store_node(
-            StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0),
-            0_u32,
-            root_node,
         )?;
 
         Ok(tree_bitmap)
@@ -290,25 +262,24 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
 
         let mut stride_end: u8 = 0;
         let mut cur_i = self.get_root_node_id();
-        let mut level: u8 = 0;
+        // let mut level: u8 = 0;
         let mut acc_retry_count = 0;
 
         let retry_and_exists = loop {
-            let stride = self.get_stride_sizes()[level as usize];
-            stride_end += stride;
+            stride_end += STRIDE_SIZE;
             let nibble_len = if pfx.get_len() < stride_end {
-                stride + pfx.get_len() - stride_end
+                STRIDE_SIZE + pfx.get_len() - stride_end
             } else {
-                stride
+                STRIDE_SIZE
             };
 
             let bit_span = AF::into_bit_span(
                 pfx.get_net(),
-                stride_end - stride,
+                stride_end - STRIDE_SIZE,
                 nibble_len,
             );
             let is_last_stride = pfx.get_len() <= stride_end;
-            let stride_start = stride_end - stride;
+            let stride_start = stride_end - STRIDE_SIZE;
 
             // this counts the number of retry_count for this loop only,
             // but ultimately we will return the accumulated count of all
@@ -319,7 +290,6 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                 // necessary.
                 if let Some(current_node) = self.retrieve_node_mut(cur_i, mui)
                 {
-                    // let current_node = current_node;
                     match current_node.eval_node_or_prefix_at(
                         bit_span,
                         // All the bits of the search prefix, but with
@@ -330,9 +300,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                             stride_start,
                         ),
                         // the length of THIS stride
-                        stride,
-                        // the length of the next stride
-                        self.get_stride_sizes().get((level + 1) as usize),
+                        // stride,
                         is_last_stride,
                     ) {
                         (NewNodeOrIndex::NewNode(n), retry_count) => {
@@ -351,18 +319,14 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                             // part of the RIB. It returns the created
                             // id and the number of retries before
                             // success.
-                            if let SizedStrideNode::Stride4(n) = n {
-                                match self.store_node(new_id, mui, n) {
-                                    Ok((node_id, s_retry_count)) => Ok((
-                                        node_id,
-                                        acc_retry_count
-                                            + s_retry_count
-                                            + retry_count,
-                                    )),
-                                    Err(err) => Err(err),
-                                }
-                            } else {
-                                panic!("ugh");
+                            match self.store_node(new_id, mui, n) {
+                                Ok((node_id, s_retry_count)) => Ok((
+                                    node_id,
+                                    acc_retry_count
+                                        + s_retry_count
+                                        + retry_count,
+                                )),
+                                Err(err) => Err(err),
                             }
                         }
                         (
@@ -413,7 +377,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             match node_result {
                 Ok((next_id, retry_count)) => {
                     cur_i = next_id;
-                    level += 1;
+                    // level += 1;
                     acc_retry_count += retry_count;
                 }
                 Err(err) => {
@@ -470,176 +434,6 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             None => false,
         }
     }
-    // pub fn prefix_exists_legacy(&self, prefix_id: PrefixId<AF>) -> bool {
-    //     // if prefix_id.get_len() == 0 {
-    //     //     return self.update_default_route_prefix_meta(mui);
-    //     // }
-    //     trace!("exists {}?", Prefix::from(prefix_id));
-    //     if prefix_id.get_len() == 0 {
-    //         return false;
-    //     }
-
-    //     let mut stride_end: u8 = 0;
-    //     let mut cur_i = self.get_root_node_id();
-    //     let mut level: u8 = 0;
-
-    //     loop {
-    //         let stride = self.get_stride_sizes()[level as usize];
-    //         stride_end += stride;
-    //         trace!("stride end {}", stride_end);
-    //         assert!(stride_end <= AF::BITS);
-    //         let nibble_len = if prefix_id.get_len() < stride_end {
-    //             stride + prefix_id.get_len() - stride_end
-    //         } else {
-    //             stride
-    //         };
-
-    //         let nibble = AF::get_nibble(
-    //             prefix_id.get_net(),
-    //             stride_end - stride,
-    //             nibble_len,
-    //         );
-    //         let stride_start = stride_end - stride;
-
-    //         let node_result = match self.retrieve_node(cur_i) {
-    //             Some(node) => match node {
-    //                 SizedStrideRef::Stride3(n) => {
-    //                     let (child_node, found) = n.prefix_exists_in_stride(
-    //                         prefix_id,
-    //                         nibble,
-    //                         nibble_len,
-    //                         stride_start,
-    //                     );
-    //                     if found {
-    //                         return true;
-    //                     } else {
-    //                         child_node
-    //                     }
-    //                 }
-    //                 SizedStrideRef::Stride4(n) => {
-    //                     let (child_node, found) = n.prefix_exists_in_stride(
-    //                         prefix_id,
-    //                         nibble,
-    //                         nibble_len,
-    //                         stride_start,
-    //                     );
-    //                     if found {
-    //                         return true;
-    //                     } else {
-    //                         child_node
-    //                     }
-    //                 }
-    //                 SizedStrideRef::Stride5(n) => {
-    //                     let (child_node, found) = n.prefix_exists_in_stride(
-    //                         prefix_id,
-    //                         nibble,
-    //                         nibble_len,
-    //                         stride_start,
-    //                     );
-    //                     if found {
-    //                         return true;
-    //                     } else {
-    //                         child_node
-    //                     }
-    //                 }
-    //             },
-    //             None => {
-    //                 return false;
-    //             }
-    //         };
-
-    //         if let Some(next_id) = node_result {
-    //             cur_i = next_id;
-    //             level += 1;
-    //             continue;
-    //         }
-
-    //         return false;
-    //     }
-    // }
-
-    // pub fn prefix_exists_for_mui(
-    //     &self,
-    //     prefix_id: PrefixId<AF>,
-    //     mui: u32,
-    // ) -> bool {
-    //     // if prefix_id.get_len() == 0 {
-    //     //     return self.update_default_route_prefix_meta(mui);
-    //     // }
-
-    //     let mut stride_end: u8 = 0;
-    //     let mut cur_i = self.get_root_node_id();
-    //     let mut level: u8 = 0;
-
-    //     loop {
-    //         let stride = self.get_stride_sizes()[level as usize];
-    //         stride_end += stride;
-    //         let nibble_len = if prefix_id.get_len() < stride_end {
-    //             stride + prefix_id.get_len() - stride_end
-    //         } else {
-    //             stride
-    //         };
-
-    //         let nibble = AF::get_nibble(
-    //             prefix_id.get_net(),
-    //             stride_end - stride,
-    //             nibble_len,
-    //         );
-    //         let stride_start = stride_end - stride;
-
-    //         let node_result = match self.retrieve_node_for_mui(cur_i, mui) {
-    //             Some(node) => match node {
-    //                 SizedStrideRef::Stride3(n) => {
-    //                     let (child_node, found) = n.prefix_exists_in_stride(
-    //                         prefix_id,
-    //                         nibble,
-    //                         nibble_len,
-    //                         stride_start,
-    //                     );
-    //                     if found {
-    //                         return true;
-    //                     } else {
-    //                         child_node
-    //                     }
-    //                 }
-    //                 SizedStrideRef::Stride4(n) => {
-    //                     let (child_node, found) = n.prefix_exists_in_stride(
-    //                         prefix_id,
-    //                         nibble,
-    //                         nibble_len,
-    //                         stride_start,
-    //                     );
-    //                     if found {
-    //                         return true;
-    //                     } else {
-    //                         child_node
-    //                     }
-    //                 }
-    //                 SizedStrideRef::Stride5(n) => {
-    //                     let (child_node, found) = n.prefix_exists_in_stride(
-    //                         prefix_id,
-    //                         nibble,
-    //                         nibble_len,
-    //                         stride_start,
-    //                     );
-    //                     if found {
-    //                         return true;
-    //                     } else {
-    //                         child_node
-    //                     }
-    //                 }
-    //             },
-    //             None => {
-    //                 return false;
-    //             }
-    //         };
-
-    //         if let Some(next_id) = node_result {
-    //             cur_i = next_id;
-    //             level += 1;
-    //         }
-    //     }
-    // }
 
     // Yes, we're hating this. But, the root node has no room for a serial of
     // the prefix 0/0 (the default route), which doesn't even matter, unless,
@@ -669,23 +463,12 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
     ) -> Result<(u32, bool), PrefixStoreError> {
         trace!("Updating the default route...");
 
-        if let Some(root_node) =
+        if let Some(_root_node) =
             self.retrieve_node_mut(self.get_root_node_id(), mui)
         {
-            // match root_node {
-            //     SizedStrideRef::Stride3(_) => self
-            //         .node_buckets
-            //         .get_store3(self.get_root_node_id())
-            //         .update_rbm_index(mui),
-            // SizedStrideRef::Stride4(_) => self
             self.node_buckets
-                .get_store4(self.get_root_node_id())
+                .get_store(self.get_root_node_id())
                 .update_rbm_index(mui)
-            // SizedStrideRef::Stride5(_) => self
-            //     .node_buckets
-            //     .get_store5(self.get_root_node_id())
-            //     .update_rbm_index(mui),
-            // }
         } else {
             Err(PrefixStoreError::StoreNotReadyError)
         }
@@ -702,184 +485,6 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         }
     }
 
-    fn _new_store_node(
-        &self,
-        id: StrideNodeId<AF>,
-        multi_uniq_id: u32,
-        next_node: TreeBitMapNode<AF, Stride4>,
-    ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
-        if log_enabled!(log::Level::Trace) {
-            debug!(
-                "{} store: Store node {}: {:?} mui {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                next_node,
-                multi_uniq_id
-            );
-        }
-        self.counters.inc_nodes_count();
-        let mut level = 0;
-        let mut retry_count = 0;
-        let mut nodes = self.node_buckets.get_store4(id);
-
-        let this_level =
-            <NB as NodeBuckets<AF>>::len_to_store_bits(id.get_id().1, level);
-        trace!("{:032b}", id.get_id().0);
-        trace!("id {:?}", id.get_id());
-        trace!("multi_uniq_id {}", multi_uniq_id);
-
-        loop {
-            // HASHING FUNCTION
-            let index = Self::hash_node_id(id, level);
-
-            match nodes.read().get(index) {
-                None => {
-                    // No node exists, so we create one here.
-                    let next_level =
-                        <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            id.get_id().1,
-                            level + 1,
-                        );
-
-                    if log_enabled!(log::Level::Trace) {
-                        trace!(
-                        "Empty node found, creating new node {} len{} lvl{}",
-                        id,
-                        id.get_id().1,
-                        level + 1
-                    );
-                        trace!("Next level {}", next_level);
-                        trace!(
-                            "Creating space for {} nodes",
-                            if next_level >= this_level {
-                                1 << (next_level - this_level)
-                            } else {
-                                1
-                            }
-                        );
-                    }
-
-                    trace!("multi uniq id {}", multi_uniq_id);
-
-                    let node_set = NodeSet::init(next_level - this_level);
-
-                    let ptrbitarr = next_node.ptrbitarr.load();
-                    let pfxbitarr = next_node.pfxbitarr.load();
-
-                    let (stored_node, its_us) =
-                        nodes.read().get_or_init(index, || StoredNode {
-                            node_id: id,
-                            node: next_node,
-                            node_set,
-                        });
-
-                    if stored_node.node_id == id {
-                        stored_node
-                            .node_set
-                            .update_rbm_index(multi_uniq_id)?;
-
-                        if !its_us && ptrbitarr != 0 {
-                            retry_count += 1;
-                            stored_node.node.ptrbitarr.merge_with(ptrbitarr);
-                        }
-
-                        if !its_us && pfxbitarr != 0 {
-                            retry_count += 1;
-                            stored_node.node.pfxbitarr.merge_with(pfxbitarr);
-                        }
-                    }
-
-                    return Ok((id, retry_count));
-                }
-                Some(stored_node) => {
-                    // A node exists, might be ours, might be
-                    // another one.
-
-                    if log_enabled!(log::Level::Trace) {
-                        trace!(
-                            "
-                        {} store: Node here exists {:?}",
-                            std::thread::current()
-                                .name()
-                                .unwrap_or("unnamed-thread"),
-                            stored_node.node_id
-                        );
-                        trace!("node_id {:?}", stored_node.node_id.get_id());
-                        trace!(
-                            "node_id {:032b}",
-                            stored_node.node_id.get_id().0
-                        );
-                        trace!("id {}", id);
-                        trace!("     id {:032b}", id.get_id().0);
-                    }
-
-                    // See if somebody beat us to creating our
-                    // node already, if so, we still need to do
-                    // work: we have to update the bitmap index
-                    // with the multi_uniq_id we've got from the
-                    // caller.
-                    if id == stored_node.node_id {
-                        stored_node
-                            .node_set
-                            .update_rbm_index(multi_uniq_id)?;
-
-                        if next_node.ptrbitarr.load() != 0 {
-                            stored_node
-                                .node
-                                .ptrbitarr
-                                .merge_with(next_node.ptrbitarr.load());
-                        }
-                        if next_node.pfxbitarr.load() != 0 {
-                            stored_node
-                                .node
-                                .pfxbitarr
-                                .merge_with(next_node.pfxbitarr.load());
-                        }
-
-                        return Ok((id, retry_count));
-                    } else {
-                        // it's not "our" node, make a (recursive)
-                        // call to create it.
-                        level += 1;
-                        trace!(
-                            "Collision with node_id {}, move to next level:
-                            {} len{} next_lvl{} index {}",
-                            stored_node.node_id,
-                            id,
-                            id.get_id().1,
-                            level,
-                            index
-                        );
-
-                        level += 1;
-                        match <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            id.get_id().1,
-                            level,
-                        ) {
-                            // on to the next level!
-                            next_bit_shift if next_bit_shift > 0 => {
-                                nodes = &stored_node.node_set;
-                                // (search_level.f)(
-                                //     search_level,
-                                //     &stored_node.node_set,
-                                //     new_node,
-                                //     multi_uniq_id,
-                                //     level,
-                                //     retry_count,
-                                // )
-                            }
-                            // There's no next level!
-                            _ => panic!(
-                                "out of storage levels, current level is {}",
-                                level
-                            ),
-                        };
-                    }
-                }
-            }
-        }
-    }
-
     fn store_node(
         &self,
         id: StrideNodeId<AF>,
@@ -897,7 +502,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         }
         self.counters.inc_nodes_count();
 
-        let mut nodes = self.node_buckets.get_store4(id);
+        let mut nodes = self.node_buckets.get_store(id);
         let new_node = next_node;
         let mut level = 0;
         let mut retry_count = 0;
@@ -962,10 +567,12 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                             .update_rbm_index(multi_uniq_id)?;
 
                         if !its_us && ptrbitarr != 0 {
+                            retry_count += 1;
                             stored_node.node.ptrbitarr.merge_with(ptrbitarr);
                         }
 
                         if !its_us && pfxbitarr != 0 {
+                            retry_count += 1;
                             stored_node.node.pfxbitarr.merge_with(pfxbitarr);
                         }
                     }
@@ -1061,123 +668,123 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
     //
     // Returns: a tuple with the node_id of the created node and the number of
     // retry_count
-    #[allow(clippy::type_complexity)]
-    fn _old_old_store_node(
-        &self,
-        id: StrideNodeId<AF>,
-        multi_uniq_id: u32,
-        next_node: TreeBitMapNode<AF, Stride4>,
-    ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn Fn(
-                &SearchLevel<AF, S>,
-                &NodeSet<AF, S>,
-                TreeBitMapNode<AF, S>,
-                u32, // multi_uniq_id
-                u8,  // the store level
-                u32, // retry_count
-            ) -> Result<
-                (StrideNodeId<AF>, u32),
-                PrefixStoreError,
-            >,
-        }
+    // #[allow(clippy::type_complexity)]
+    // fn _old_old_store_node(
+    //     &self,
+    //     id: StrideNodeId<AF>,
+    //     multi_uniq_id: u32,
+    //     next_node: TreeBitMapNode<AF, Stride4>,
+    // ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
+    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+    //         f: &'s dyn Fn(
+    //             &SearchLevel<AF, S>,
+    //             &NodeSet<AF, S>,
+    //             TreeBitMapNode<AF, S>,
+    //             u32, // multi_uniq_id
+    //             u8,  // the store level
+    //             u32, // retry_count
+    //         ) -> Result<
+    //             (StrideNodeId<AF>, u32),
+    //             PrefixStoreError,
+    //         >,
+    //     }
 
-        let search_level_3 =
-            store_node_closure![Stride3; id; guard; back_off;];
-        let search_level_4 =
-            store_node_closure![Stride4; id; guard; back_off;];
-        let search_level_5 =
-            store_node_closure![Stride5; id; guard; back_off;];
+    //     let search_level_3 =
+    //         store_node_closure![Stride3; id; guard; back_off;];
+    //     let search_level_4 =
+    //         store_node_closure![Stride4; id; guard; back_off;];
+    //     let search_level_5 =
+    //         store_node_closure![Stride5; id; guard; back_off;];
 
-        if log_enabled!(log::Level::Trace) {
-            debug!(
-                "{} store: Store node {}: {:?} mui {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                next_node,
-                multi_uniq_id
-            );
-        }
-        self.counters.inc_nodes_count();
+    //     if log_enabled!(log::Level::Trace) {
+    //         debug!(
+    //             "{} store: Store node {}: {:?} mui {}",
+    //             std::thread::current().name().unwrap_or("unnamed-thread"),
+    //             id,
+    //             next_node,
+    //             multi_uniq_id
+    //         );
+    //     }
+    //     self.counters.inc_nodes_count();
+
+    //     // match next_node {
+    //     // SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
+    //     //     &search_level_3,
+    //     //     self.node_buckets.get_store3(id),
+    //     //     new_node,
+    //     //     multi_uniq_id,
+    //     //     0,
+    //     //     0,
+    //     // ),
+    //     (search_level_4.f)(
+    //         &search_level_4,
+    //         self.node_buckets.get_store4(id),
+    //         next_node,
+    //         multi_uniq_id,
+    //         0,
+    //         0,
+    //     )
+    //     // SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
+    //     //     &search_level_5,
+    //     //     self.node_buckets.get_store5(id),
+    //     //     new_node,
+    //     //     multi_uniq_id,
+    //     //     0,
+    //     //     0,
+    //     // ),
+    //     // }
+    // }
 
-        // match next_node {
-        // SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
-        //     &search_level_3,
-        //     self.node_buckets.get_store3(id),
-        //     new_node,
-        //     multi_uniq_id,
-        //     0,
-        //     0,
-        // ),
-        (search_level_4.f)(
-            &search_level_4,
-            self.node_buckets.get_store4(id),
-            next_node,
-            multi_uniq_id,
-            0,
-            0,
-        )
-        // SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
-        //     &search_level_5,
-        //     self.node_buckets.get_store5(id),
-        //     new_node,
-        //     multi_uniq_id,
-        //     0,
-        //     0,
-        // ),
-        // }
-    }
+    // #[allow(clippy::type_complexity)]
+    // pub(crate) fn _retrieve_node_mut(
+    //     &self,
+    //     id: StrideNodeId<AF>,
+    //     multi_uniq_id: u32,
+    // ) -> Option<SizedStrideRef<'_, AF>> {
+    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
+    //         f: &'s dyn for<'a> Fn(
+    //             &SearchLevel<AF, S>,
+    //             &'a NodeSet<AF, S>,
+    //             u8,
+    //         )
+    //             -> Option<SizedStrideRef<'a, AF>>,
+    //     }
 
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn _retrieve_node_mut(
-        &self,
-        id: StrideNodeId<AF>,
-        multi_uniq_id: u32,
-    ) -> Option<SizedStrideRef<'_, AF>> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, S>,
-                &'a NodeSet<AF, S>,
-                u8,
-            )
-                -> Option<SizedStrideRef<'a, AF>>,
-        }
+    //     let search_level_3 =
+    //         retrieve_node_mut_closure![Stride3; id; multi_uniq_id;];
+    //     let search_level_4 =
+    //         retrieve_node_mut_closure![Stride4; id; multi_uniq_id;];
+    //     let search_level_5 =
+    //         retrieve_node_mut_closure![Stride5; id; multi_uniq_id;];
 
-        let search_level_3 =
-            retrieve_node_mut_closure![Stride3; id; multi_uniq_id;];
-        let search_level_4 =
-            retrieve_node_mut_closure![Stride4; id; multi_uniq_id;];
-        let search_level_5 =
-            retrieve_node_mut_closure![Stride5; id; multi_uniq_id;];
+    //     if log_enabled!(log::Level::Trace) {
+    //         trace!(
+    //             "{} store: Retrieve node mut {} from l{}",
+    //             std::thread::current().name().unwrap_or("unnamed-thread"),
+    //             id,
+    //             id.get_id().1
+    //         );
+    //     }
 
-        if log_enabled!(log::Level::Trace) {
-            trace!(
-                "{} store: Retrieve node mut {} from l{}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                id.get_id().1
-            );
-        }
+    //     match self.node_buckets.get_stride_for_id(id) {
+    //         3 => (search_level_3.f)(
+    //             &search_level_3,
+    //             self.node_buckets.get_store3(id),
+    //             0,
+    //         ),
 
-        match self.node_buckets.get_stride_for_id(id) {
-            3 => (search_level_3.f)(
-                &search_level_3,
-                self.node_buckets.get_store3(id),
-                0,
-            ),
-
-            4 => (search_level_4.f)(
-                &search_level_4,
-                self.node_buckets.get_store4(id),
-                0,
-            ),
-            _ => (search_level_5.f)(
-                &search_level_5,
-                self.node_buckets.get_store5(id),
-                0,
-            ),
-        }
-    }
+    //         4 => (search_level_4.f)(
+    //             &search_level_4,
+    //             self.node_buckets.get_store4(id),
+    //             0,
+    //         ),
+    //         _ => (search_level_5.f)(
+    //             &search_level_5,
+    //             self.node_buckets.get_store5(id),
+    //             0,
+    //         ),
+    //     }
+    // }
 
     pub fn retrieve_node_mut(
         &self,
@@ -1187,7 +794,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
-        let mut nodes = self.node_buckets.get_store4(id);
+        let mut nodes = self.node_buckets.get_store(id);
 
         loop {
             let index = Self::hash_node_id(id, level);
@@ -1277,7 +884,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
-        let mut nodes = self.node_buckets.get_store4(id);
+        let mut nodes = self.node_buckets.get_store(id);
 
         loop {
             let index = Self::hash_node_id(id, level);
@@ -1365,7 +972,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
-        let mut nodes = self.node_buckets.get_store4(id);
+        let mut nodes = self.node_buckets.get_store(id);
 
         loop {
             let index = Self::hash_node_id(id, level);
@@ -1456,107 +1063,6 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         }
     }
 
-    // #[allow(clippy::type_complexity)]
-    // pub(crate) fn retrieve_node(
-    //     &self,
-    //     id: StrideNodeId<AF>,
-    // ) -> Option<SizedStrideRef<'_, AF>> {
-    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-    //         f: &'s dyn for<'a> Fn(
-    //             &SearchLevel<AF, S>,
-    //             &'a NodeSet<AF, S>,
-    //             u8,
-    //         )
-    //             -> Option<SizedStrideRef<'a, AF>>,
-    //     }
-
-    //     let search_level_3 = impl_search_level![Stride3; id;];
-    //     let search_level_4 = impl_search_level![Stride4; id;];
-    //     let search_level_5 = impl_search_level![Stride5; id;];
-
-    //     if log_enabled!(log::Level::Trace) {
-    //         trace!(
-    //             "{} store: Retrieve node {} from l{}",
-    //             std::thread::current().name().unwrap_or("unnamed-thread"),
-    //             id,
-    //             id.get_id().1
-    //         );
-    //     }
-
-    //     match self.get_stride_for_id(id) {
-    //         3 => (search_level_3.f)(
-    //             &search_level_3,
-    //             self.node_buckets.get_store3(id),
-    //             0,
-    //         ),
-    //         4 => (search_level_4.f)(
-    //             &search_level_4,
-    //             self.node_buckets.get_store4(id),
-    //             0,
-    //         ),
-    //         _ => (search_level_5.f)(
-    //             &search_level_5,
-    //             self.node_buckets.get_store5(id),
-    //             0,
-    //         ),
-    //     }
-    // }
-
-    // retrieve a node, but only if its bitmap index contains the specified
-    // mui. Used for iterators per mui.
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn _old_retrieve_node_for_mui(
-        &self,
-        id: StrideNodeId<AF>,
-        // The mui that is tested to be present in the nodes bitmap index
-        mui: u32,
-    ) -> Option<SizedStrideRef<'_, AF>> {
-        struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, S>,
-                &'a NodeSet<AF, S>,
-                u8,
-            )
-                -> Option<SizedStrideRef<'a, AF>>,
-        }
-
-        let search_level_3 = impl_search_level_for_mui![Stride3; id; mui;];
-        let search_level_4 = impl_search_level_for_mui![Stride4; id; mui;];
-        let search_level_5 = impl_search_level_for_mui![Stride5; id; mui;];
-
-        if log_enabled!(log::Level::Trace) {
-            trace!(
-                "{} store: Retrieve node {} from l{} for mui {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                id,
-                id.get_id().1,
-                mui
-            );
-        }
-
-        match self.get_stride_for_id(id) {
-            3 => (search_level_3.f)(
-                &search_level_3,
-                self.node_buckets.get_store3(id),
-                0,
-            ),
-            4 => (search_level_4.f)(
-                &search_level_4,
-                self.node_buckets.get_store4(id),
-                0,
-            ),
-            _ => (search_level_5.f)(
-                &search_level_5,
-                self.node_buckets.get_store5(id),
-                0,
-            ),
-        }
-    }
-
-    pub(crate) fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8 {
-        self.node_buckets.get_stride_for_id(id)
-    }
-
     pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
         StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0)
     }
@@ -1579,10 +1085,6 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         self.node_buckets.get_stride_sizes()
     }
 
-    pub(crate) fn get_first_stride_size() -> u8 {
-        NB::get_first_stride_size()
-    }
-
     // Calculates the id of the node that COULD host a prefix in its
     // ptrbitarr.
     pub(crate) fn get_node_id_for_prefix(
@@ -1595,10 +1097,11 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             prefix.get_len()
         );
         let mut acc = 0;
-        for i in self.get_stride_sizes() {
-            acc += *i;
+        // for i in self.get_stride_sizes() {
+        loop {
+            acc += STRIDE_SIZE;
             if acc >= prefix.get_len() {
-                let node_len = acc - i;
+                let node_len = acc - STRIDE_SIZE;
                 return (
                     StrideNodeId::new_with_cleaned_id(
                         prefix.get_net(),
@@ -1621,7 +1124,6 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                 );
             }
         }
-        panic!("prefix length for {:?} is too long", prefix);
     }
 
     // ------- THE HASHING FUNCTION -----------------------------------------
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
index 511af05c..b63ddf15 100644
--- a/src/local_array/prefix_cht/iterators.rs
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -4,28 +4,24 @@ use roaring::RoaringBitmap;
 use crate::{
     local_array::{
         bit_span::BitSpan,
-        in_memory::{node::SizedStrideRef, tree::TreeBitMap},
+        in_memory::{
+            node::{NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter},
+            tree::{Stride4, TreeBitMap},
+        },
     },
     prelude::multi::{NodeBuckets, PrefixId},
-    rib::iterators::{SizedNodeMoreSpecificIter, SizedPrefixIter},
     AddressFamily,
 };
 
-type Type<AF: AddressFamily> = SizedNodeMoreSpecificIter<AF>;
-
 pub(crate) struct MoreSpecificPrefixIter<
     'a,
     AF: AddressFamily,
-    // M: Meta,
     NB: NodeBuckets<AF>,
-    // PB: PrefixBuckets<AF, M>,
 > {
     store: &'a TreeBitMap<AF, NB>,
-    cur_ptr_iter: SizedNodeMoreSpecificIter<AF>,
-    cur_pfx_iter: SizedPrefixIter<AF>,
-    // start_bit_span: BitSpan,
-    // skip_self: bool,
-    parent_and_position: Vec<Type<AF>>,
+    cur_ptr_iter: NodeMoreSpecificChildIter<AF, Stride4>,
+    cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF, Stride4>,
+    parent_and_position: Vec<NodeMoreSpecificChildIter<AF, Stride4>>,
     // If specified, we're only iterating over records for this mui.
     mui: Option<u32>,
     // This is the tree-wide index of withdrawn muis, used to rewrite the
@@ -79,81 +75,31 @@ impl<
             if let Some(next_ptr) = next_ptr {
                 let node = if self.mui.is_none() {
                     trace!("let's retriev node {}", next_ptr);
-                    self.store
-                        .retrieve_node(next_ptr)
-                        .map(|n| SizedStrideRef::Stride4(n))
+                    self.store.retrieve_node(next_ptr)
                 } else {
                     self.store
                         .retrieve_node_for_mui(next_ptr, self.mui.unwrap())
-                        .map(|n| SizedStrideRef::Stride4(n))
                 };
 
                 match node {
-                    Some(SizedStrideRef::Stride3(next_node)) => {
-                        // copy the current iterator into the parent vec and create
-                        // a new ptr iterator for this node
-                        self.parent_and_position.push(self.cur_ptr_iter);
-                        let ptr_iter = next_node.more_specific_ptr_iter(
-                            next_ptr,
-                            BitSpan { bits: 0, len: 0 },
-                        );
-                        self.cur_ptr_iter = ptr_iter.wrap();
-
-                        // trace!(
-                        //     "next stride new iterator stride 3 {:?} start \
-                        // bit_span {:?}",
-                        //     self.cur_ptr_iter,
-                        //     self.start_bit_span
-                        // );
-                        self.cur_pfx_iter = next_node
-                            .more_specific_pfx_iter(
-                                next_ptr,
-                                BitSpan::new(0, 0),
-                            )
-                            .wrap();
-                    }
-                    Some(SizedStrideRef::Stride4(next_node)) => {
+                    Some(next_node) => {
                         // create new ptr iterator for this node.
                         self.parent_and_position.push(self.cur_ptr_iter);
                         let ptr_iter = next_node.more_specific_ptr_iter(
                             next_ptr,
                             BitSpan { bits: 0, len: 0 },
                         );
-                        self.cur_ptr_iter = ptr_iter.wrap();
+                        self.cur_ptr_iter = ptr_iter;
 
                         trace!(
                             "next stride new iterator stride 4 {:?} start \
                         bit_span 0 0",
                             self.cur_ptr_iter,
                         );
-                        self.cur_pfx_iter = next_node
-                            .more_specific_pfx_iter(
-                                next_ptr,
-                                BitSpan::new(0, 0),
-                            )
-                            .wrap();
-                    }
-                    Some(SizedStrideRef::Stride5(next_node)) => {
-                        // create new ptr iterator for this node.
-                        self.parent_and_position.push(self.cur_ptr_iter);
-                        let ptr_iter = next_node.more_specific_ptr_iter(
+                        self.cur_pfx_iter = next_node.more_specific_pfx_iter(
                             next_ptr,
-                            BitSpan { bits: 0, len: 0 },
-                        );
-                        self.cur_ptr_iter = ptr_iter.wrap();
-
-                        // trace!(
-                        //     "next stride new iterator stride 5 {:?} start \
-                        // bit_span {:?}",
-                        //     self.cur_ptr_iter,
-                        //     self.start_bit_span
-                        // );
-                        self.cur_pfx_iter = next_node
-                            .more_specific_pfx_iter(
-                                next_ptr,
-                                BitSpan::new(0, 0),
-                            )
-                            .wrap();
+                            BitSpan::new(0, 0),
+                        )
                     }
                     None => {
                         println!("no node here.");
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index 2d0dec2f..f8c6fc14 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -4,6 +4,8 @@ use crate::prelude::multi::*;
 use crate::prelude::*;
 use rand::prelude::*;
 
+pub const STRIDE_SIZE: u8 = 4;
+
 // The default stride sizes for IPv4, IPv6, resp.
 #[create_store((
     ([4, 4, 4, 4, 4, 4, 4, 4], 5, 18, LongKey),

From 9656b262a4a0affd2cb7083536961b10b62bc8da Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 27 Feb 2025 09:49:27 +0100
Subject: [PATCH 088/147] remove S from NodeSet

---
 proc_macros/src/lib.rs                    |  12 +-
 src/local_array/in_memory/atomic_types.rs |  21 +-
 src/local_array/in_memory/node.rs         |   3 +-
 src/local_array/rib/macros.rs             | 676 +++++++++++-----------
 4 files changed, 356 insertions(+), 356 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 70d0963d..58c4ef19 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -200,12 +200,12 @@ pub fn stride_sizes(
         pub(crate) struct #buckets_name<AF: AddressFamily> {
             // created fields for each sub-prefix (StrideNodeId) length,
             // with hard-coded field-names, like this:
-            // l0: NodeSet<AF, Stride5>,
-            // l5: NodeSet<AF, Stride5>,
-            // l10: NodeSet<AF, Stride4>,
+            // l0: NodeSet<AF>,
+            // l4: NodeSet<AF>,
+            // l8: NodeSet<AF>,
             // ...
-            // l29: NodeSet<AF, Stride3>
-            # ( #strides_all_len_level: NodeSet<#ip_af, #strides>, )*
+            // l28: NodeSet<AF, Stride4>
+            # ( #strides_all_len_level: NodeSet<#ip_af>, )*
             _af: PhantomData<AF>,
             stride_sizes: [u8; 42],
             strides_len: u8
@@ -271,7 +271,7 @@ pub fn stride_sizes(
                 }
             }
 
-            fn get_store(&self, id: StrideNodeId<#ip_af>) -> &NodeSet<#ip_af, Stride4> {
+            fn get_store(&self, id: StrideNodeId<#ip_af>) -> &NodeSet<#ip_af> {
                 match id.get_id().1 as usize {
                     #( #strides_len4 => &self.#strides_len4_l, )*
                     // ex.:
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 8916ca77..43a254e1 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -28,28 +28,27 @@ use super::tree::{Stride, Stride4};
 // ----------- Node related structs -----------------------------------------
 
 #[derive(Debug)]
-pub(crate) struct StoredNode<AF, S>
+pub(crate) struct StoredNode<AF>
 where
     Self: Sized,
-    S: Stride,
     AF: AddressFamily,
 {
     pub(crate) node_id: StrideNodeId<AF>,
     // The ptrbitarr and pfxbitarr for this node
-    pub(crate) node: TreeBitMapNode<AF, S>,
+    pub(crate) node: TreeBitMapNode<AF, Stride4>,
     // Child nodes linked from this node
-    pub(crate) node_set: NodeSet<AF, S>,
+    pub(crate) node_set: NodeSet<AF>,
 }
 
 #[derive(Debug)]
-pub(crate) struct NodeSet<AF: AddressFamily, S: Stride>(
-    OnceBoxSlice<StoredNode<AF, S>>,
+pub struct NodeSet<AF: AddressFamily>(
+    OnceBoxSlice<StoredNode<AF>>,
     // A Bitmap index that keeps track of the `multi_uniq_id`s (mui) that are
     // present in value collections in the meta-data tree in the child nodes
     RwLock<RoaringBitmap>,
 );
 
-impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
+impl<AF: AddressFamily> NodeSet<AF> {
     pub(crate) fn init(p2_size: u8) -> Self {
         if log_enabled!(log::Level::Debug) {
             debug!(
@@ -71,7 +70,7 @@ impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
         multi_uniq_id: u32,
     ) -> Result<(u32, bool), crate::prelude::multi::PrefixStoreError>
     where
-        S: atomic_stride::Stride,
+        // S: atomic_stride::Stride,
         AF: crate::AddressFamily,
     {
         let try_count = 0;
@@ -87,7 +86,7 @@ impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
         _guard: &crate::epoch::Guard,
     ) -> Result<u32, crate::prelude::multi::PrefixStoreError>
     where
-        S: atomic_stride::Stride,
+        // S: atomic_stride::Stride,
         AF: crate::AddressFamily,
     {
         let try_count = 0;
@@ -98,7 +97,7 @@ impl<AF: AddressFamily, S: Stride> NodeSet<AF, S> {
         Ok(try_count)
     }
 
-    pub(crate) fn read(&self) -> &OnceBoxSlice<StoredNode<AF, S>> {
+    pub(crate) fn read(&self) -> &OnceBoxSlice<StoredNode<AF>> {
         &self.0
     }
 }
@@ -580,7 +579,7 @@ pub trait NodeBuckets<AF: AddressFamily> {
     fn get_stride_sizes(&self) -> &[u8];
     fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8;
     // fn get_store3(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride3>;
-    fn get_store(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride4>;
+    fn get_store(&self, id: StrideNodeId<AF>) -> &NodeSet<AF>;
     // fn get_store5(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride5>;
     fn get_strides_len() -> u8;
     // fn get_first_stride_size() -> u8;
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 558bbd64..7751a49f 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -12,6 +12,7 @@ use super::tree::{
 
 use crate::af::AddressFamily;
 use crate::af::Zero;
+use crate::local_array::rib::default_store::STRIDE_SIZE;
 use crate::local_array::types::PrefixId;
 
 //------------ TreeBitMap Node ----------------------------------------------
@@ -260,7 +261,7 @@ where
         // base_prefix (which is always the start length of the stride).
         (
             NewNodeOrIndex::ExistingNode(
-                base_prefix.add_to_len(4).truncate_to_len(),
+                base_prefix.add_to_len(STRIDE_SIZE).truncate_to_len(),
             ),
             retry_count,
         )
diff --git a/src/local_array/rib/macros.rs b/src/local_array/rib/macros.rs
index 5448b30e..c8bfb201 100644
--- a/src/local_array/rib/macros.rs
+++ b/src/local_array/rib/macros.rs
@@ -1,367 +1,367 @@
-#[macro_export]
-#[doc(hidden)]
-macro_rules! impl_search_level {
-    (
-        $(
-            $stride: ident;
-            $id: ident;
-        ),
-    * ) => {
-        $(
-        SearchLevel {
-            f: &|search_level: &SearchLevel<AF, $stride>,
-                nodes,
-                mut level: u8,
-            | {
-                    // HASHING FUNCTION
-                    let index = Self::hash_node_id($id, level);
+// #[macro_export]
+// #[doc(hidden)]
+// macro_rules! impl_search_level {
+//     (
+//         $(
+//             $stride: ident;
+//             $id: ident;
+//         ),
+//     * ) => {
+//         $(
+//         SearchLevel {
+//             f: &|search_level: &SearchLevel<AF, $stride>,
+//                 nodes,
+//                 mut level: u8,
+//             | {
+//                     // HASHING FUNCTION
+//                     let index = Self::hash_node_id($id, level);
 
-                    match nodes.read().get(index) {
-                        None => None,
-                        Some(stored_node) => {
-                            let StoredNode {
-                                node_id, node, node_set, .. } = stored_node;
-                            if $id == *node_id {
-                                // YES, It's the one we're looking for!
-                                return Some(SizedStrideRef::$stride(&node));
-                            };
-                            // Meh, it's not, but we can a go to the next
-                            // level and see if it lives there.
-                            level += 1;
-                            match <NB as NodeBuckets<AF>>::len_to_store_bits(
-                                $id.get_id().1, level
-                            ) {
-                                // on to the next level!
-                                next_bit_shift if next_bit_shift > 0 => {
-                                    (search_level.f)(
-                                        search_level,
-                                        &node_set,
-                                        level,
-                                    )
-                                }
-                                // There's no next level, we found nothing.
-                                _ => None,
-                            }
-                        }
-                    }
-                }
-        }
-        )*
-    };
-}
+//                     match nodes.read().get(index) {
+//                         None => None,
+//                         Some(stored_node) => {
+//                             let StoredNode {
+//                                 node_id, node, node_set, .. } = stored_node;
+//                             if $id == *node_id {
+//                                 // YES, It's the one we're looking for!
+//                                 return Some(SizedStrideRef::$stride(&node));
+//                             };
+//                             // Meh, it's not, but we can a go to the next
+//                             // level and see if it lives there.
+//                             level += 1;
+//                             match <NB as NodeBuckets<AF>>::len_to_store_bits(
+//                                 $id.get_id().1, level
+//                             ) {
+//                                 // on to the next level!
+//                                 next_bit_shift if next_bit_shift > 0 => {
+//                                     (search_level.f)(
+//                                         search_level,
+//                                         &node_set,
+//                                         level,
+//                                     )
+//                                 }
+//                                 // There's no next level, we found nothing.
+//                                 _ => None,
+//                             }
+//                         }
+//                     }
+//                 }
+//         }
+//         )*
+//     };
+// }
 
-#[macro_export]
-#[doc(hidden)]
-macro_rules! impl_search_level_for_mui {
-    (
-        $(
-            $stride: ident;
-            $id: ident;
-            $mui: ident;
-        ),
-    * ) => {
-        $(
-        SearchLevel {
-            f: &|search_level: &SearchLevel<AF, $stride>,
-                nodes,
-                mut level: u8| {
-                    // HASHING FUNCTION
-                    let index = Self::hash_node_id($id, level);
+// #[macro_export]
+// #[doc(hidden)]
+// macro_rules! impl_search_level_for_mui {
+//     (
+//         $(
+//             $stride: ident;
+//             $id: ident;
+//             $mui: ident;
+//         ),
+//     * ) => {
+//         $(
+//         SearchLevel {
+//             f: &|search_level: &SearchLevel<AF, $stride>,
+//                 nodes,
+//                 mut level: u8| {
+//                     // HASHING FUNCTION
+//                     let index = Self::hash_node_id($id, level);
 
-                    match nodes.read().get(index) {
-                        None => None,
-                        Some(this_node) => {
-                            let StoredNode {
-                                node_id, node, node_set, .. } = this_node;
+//                     match nodes.read().get(index) {
+//                         None => None,
+//                         Some(this_node) => {
+//                             let StoredNode {
+//                                 node_id, node, node_set, .. } = this_node;
 
-                            // early return if the mui is not in the index
-                            // stored in this node, meaning the mui does not
-                            // appear anywhere in the sub-tree formed from
-                            // this node.
-                            let bmin = node_set.rbm().read().unwrap();
-                            if !bmin.contains($mui) {
-                                return None;
-                            }
+//                             // early return if the mui is not in the index
+//                             // stored in this node, meaning the mui does not
+//                             // appear anywhere in the sub-tree formed from
+//                             // this node.
+//                             let bmin = node_set.rbm().read().unwrap();
+//                             if !bmin.contains($mui) {
+//                                 return None;
+//                             }
 
-                            if $id == *node_id {
-                                // YES, It's the one we're looking for!
-                                return Some(SizedStrideRef::$stride(&node));
-                            };
-                            // Meh, it's not, but we can a go to the next
-                            // level and see if it lives there.
-                            level += 1;
-                            match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
-                                // on to the next level!
-                                next_bit_shift if next_bit_shift > 0 => {
-                                    (search_level.f)(
-                                        search_level,
-                                        &node_set,
-                                        level,
-                                    )
-                                }
-                                // There's no next level, we found nothing.
-                                _ => None,
-                            }
-                        }
-                    }
-                }
-        }
-        )*
-    };
-}
+//                             if $id == *node_id {
+//                                 // YES, It's the one we're looking for!
+//                                 return Some(SizedStrideRef::$stride(&node));
+//                             };
+//                             // Meh, it's not, but we can a go to the next
+//                             // level and see if it lives there.
+//                             level += 1;
+//                             match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
+//                                 // on to the next level!
+//                                 next_bit_shift if next_bit_shift > 0 => {
+//                                     (search_level.f)(
+//                                         search_level,
+//                                         &node_set,
+//                                         level,
+//                                     )
+//                                 }
+//                                 // There's no next level, we found nothing.
+//                                 _ => None,
+//                             }
+//                         }
+//                     }
+//                 }
+//         }
+//         )*
+//     };
+// }
 
-// This macro creates a closure that is used in turn in the macro
-// 'eBox', that is used in the public `insert` method on a TreeBitMap.
-//
-// It retrieves the node specified by $id recursively, creates it if it does
-// not exist. It is responsible for setting/updating the RBMIN, but is does
-// *not* set/update the pfxbitarr or ptrbitarr of the TreeBitMapNode. The
-// `insert_match` takes care of the latter.
-//
-// This closure should not be called repeatedly to create the same node, if it
-// returns `None` that is basically a data race in the store and therefore an
-// error. Also the caller should make sure to stay within the limit of the
-// defined number of levels, although the closure will return at the end of
-// the maximum depth.
-#[macro_export]
-#[doc(hidden)]
-macro_rules! retrieve_node_mut_closure {
-    (
-        $(
-            $stride: ident;
-            $id: ident;
-            $multi_uniq_id: ident;
-        ),
-    * ) => {$(
-        SearchLevel {
-            f: &|
-                search_level: &SearchLevel<AF, $stride>,
-                nodes,
-                mut level: u8,
-            | {
-                // HASHING FUNCTION
-                let index = Self::hash_node_id($id, level);
-                let node;
+// // This macro creates a closure that is used in turn in the macro
+// // 'eBox', that is used in the public `insert` method on a TreeBitMap.
+// //
+// // It retrieves the node specified by $id recursively, creates it if it does
+// // not exist. It is responsible for setting/updating the RBMIN, but is does
+// // *not* set/update the pfxbitarr or ptrbitarr of the TreeBitMapNode. The
+// // `insert_match` takes care of the latter.
+// //
+// // This closure should not be called repeatedly to create the same node, if it
+// // returns `None` that is basically a data race in the store and therefore an
+// // error. Also the caller should make sure to stay within the limit of the
+// // defined number of levels, although the closure will return at the end of
+// // the maximum depth.
+// #[macro_export]
+// #[doc(hidden)]
+// macro_rules! retrieve_node_mut_closure {
+//     (
+//         $(
+//             $stride: ident;
+//             $id: ident;
+//             $multi_uniq_id: ident;
+//         ),
+//     * ) => {$(
+//         SearchLevel {
+//             f: &|
+//                 search_level: &SearchLevel<AF, $stride>,
+//                 nodes,
+//                 mut level: u8,
+//             | {
+//                 // HASHING FUNCTION
+//                 let index = Self::hash_node_id($id, level);
+//                 let node;
 
-                match nodes.read().get(index) {
-                    // This arm only ever gets called in multi-threaded code
-                    // where our thread (running this code *now*), andgot
-                    // ahead of another thread: After the other thread created
-                    // the TreeBitMapNode first, it was overtaken by our
-                    // thread running this method, so our thread enounters an
-                    // empty node in the store.
-                    None => {
-                        let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            $id.get_id().1, level
-                        );
-                        let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            $id.get_id().1, level + 1
-                        );
-                        let node_set = NodeSet::init(next_level - this_level);
+//                 match nodes.read().get(index) {
+//                     // This arm only ever gets called in multi-threaded code
+//                     // where our thread (running this code *now*), andgot
+//                     // ahead of another thread: After the other thread created
+//                     // the TreeBitMapNode first, it was overtaken by our
+//                     // thread running this method, so our thread enounters an
+//                     // empty node in the store.
+//                     None => {
+//                         let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
+//                             $id.get_id().1, level
+//                         );
+//                         let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
+//                             $id.get_id().1, level + 1
+//                         );
+//                         let node_set = NodeSet::init(next_level - this_level);
 
-                        // See if we can create the node
-                        (node, _) = nodes.read().get_or_init(index, || StoredNode {
-                            node_id: $id,
-                            node: TreeBitMapNode::new(),
-                            node_set
-                        });
+//                         // See if we can create the node
+//                         (node, _) = nodes.read().get_or_init(index, || StoredNode {
+//                             node_id: $id,
+//                             node: TreeBitMapNode::new(),
+//                             node_set
+//                         });
 
-                        // We may have lost, and a different node than we
-                        // intended could live here, if so go a level deeper
-                        if $id == node.node_id {
-                            // Nope, its ours or at least the node we need.
-                            let _retry_count = node.node_set.update_rbm_index(
-                                $multi_uniq_id
-                            ).ok();
+//                         // We may have lost, and a different node than we
+//                         // intended could live here, if so go a level deeper
+//                         if $id == node.node_id {
+//                             // Nope, its ours or at least the node we need.
+//                             let _retry_count = node.node_set.update_rbm_index(
+//                                 $multi_uniq_id
+//                             ).ok();
 
-                            return Some(SizedStrideRef::$stride(&node.node));
-                        };
-                    },
-                    Some(this_node) => {
-                        node = this_node;
-                        if $id == this_node.node_id {
-                            // YES, It's the one we're looking for!
+//                             return Some(SizedStrideRef::$stride(&node.node));
+//                         };
+//                     },
+//                     Some(this_node) => {
+//                         node = this_node;
+//                         if $id == this_node.node_id {
+//                             // YES, It's the one we're looking for!
 
-                            // Update the rbm_index in this node with the
-                            // multi_uniq_id that the caller specified. This
-                            // is the only atomic operation we need to do
-                            // here. The NodeSet that the index is attached
-                            // to, does not need to be written to, it's part
-                            // of a trie, so it just needs to "exist" (and it
-                            // already does).
-                            let retry_count = this_node.node_set.update_rbm_index(
-                                $multi_uniq_id
-                            ).ok();
+//                             // Update the rbm_index in this node with the
+//                             // multi_uniq_id that the caller specified. This
+//                             // is the only atomic operation we need to do
+//                             // here. The NodeSet that the index is attached
+//                             // to, does not need to be written to, it's part
+//                             // of a trie, so it just needs to "exist" (and it
+//                             // already does).
+//                             let retry_count = this_node.node_set.update_rbm_index(
+//                                 $multi_uniq_id
+//                             ).ok();
 
-                            trace!("Retry_count rbm index {:?}", retry_count);
-                            trace!("add multi uniq id to bitmap index {} for node {}",
-                                $multi_uniq_id, this_node.node
-                            );
-                            return Some(SizedStrideRef::$stride(&this_node.node));
-                        };
-                    }
-                }
-                // It isn't ours. Move one level deeper.
-                level += 1;
-                match <NB as NodeBuckets<AF>>::len_to_store_bits(
-                    $id.get_id().1, level
-                ) {
-                    // on to the next level!
-                    next_bit_shift if next_bit_shift > 0 => {
-                        (search_level.f)(
-                            search_level,
-                            &node.node_set,
-                            level,
-                        )
-                    }
-                    // There's no next level, we found nothing.
-                    _ => None,
-                }
-            }
-        }
-    )*};
-}
+//                             trace!("Retry_count rbm index {:?}", retry_count);
+//                             trace!("add multi uniq id to bitmap index {} for node {}",
+//                                 $multi_uniq_id, this_node.node
+//                             );
+//                             return Some(SizedStrideRef::$stride(&this_node.node));
+//                         };
+//                     }
+//                 }
+//                 // It isn't ours. Move one level deeper.
+//                 level += 1;
+//                 match <NB as NodeBuckets<AF>>::len_to_store_bits(
+//                     $id.get_id().1, level
+//                 ) {
+//                     // on to the next level!
+//                     next_bit_shift if next_bit_shift > 0 => {
+//                         (search_level.f)(
+//                             search_level,
+//                             &node.node_set,
+//                             level,
+//                         )
+//                     }
+//                     // There's no next level, we found nothing.
+//                     _ => None,
+//                 }
+//             }
+//         }
+//     )*};
+// }
 
-#[macro_export]
-#[doc(hidden)]
-macro_rules! store_node_closure {
-    (
-        $(
-            $stride: ident;
-            $id: ident;
-            // $multi_uniq_id: ident;
-            $guard: ident;
-            $back_off: ident;
-        ),
-    *) => {
-        $(
-        SearchLevel {
-            f: &|
-            search_level: &SearchLevel<AF, $stride>,
-            nodes,
-            new_node: TreeBitMapNode<AF, $stride>,
-            multi_uniq_id: u32,
-            mut level: u8,
-            retry_count: u32| {
-                let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level);
-                trace!("{:032b}", $id.get_id().0);
-                trace!("id {:?}", $id.get_id());
-                trace!("multi_uniq_id {}", multi_uniq_id);
+// #[macro_export]
+// #[doc(hidden)]
+// macro_rules! store_node_closure {
+//     (
+//         $(
+//             $stride: ident;
+//             $id: ident;
+//             // $multi_uniq_id: ident;
+//             $guard: ident;
+//             $back_off: ident;
+//         ),
+//     *) => {
+//         $(
+//         SearchLevel {
+//             f: &|
+//             search_level: &SearchLevel<AF, $stride>,
+//             nodes,
+//             new_node: TreeBitMapNode<AF, $stride>,
+//             multi_uniq_id: u32,
+//             mut level: u8,
+//             retry_count: u32| {
+//                 let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level);
+//                 trace!("{:032b}", $id.get_id().0);
+//                 trace!("id {:?}", $id.get_id());
+//                 trace!("multi_uniq_id {}", multi_uniq_id);
 
-                // HASHING FUNCTION
-                let index = Self::hash_node_id($id, level);
+//                 // HASHING FUNCTION
+//                 let index = Self::hash_node_id($id, level);
 
-                match nodes.read().get(index) {
-                    None => {
-                        // No node exists, so we create one here.
-                        let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level + 1);
+//                 match nodes.read().get(index) {
+//                     None => {
+//                         // No node exists, so we create one here.
+//                         let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level + 1);
 
-                        if log_enabled!(log::Level::Trace) {
-                            trace!("Empty node found, creating new node {} len{} lvl{}",
-                                $id, $id.get_id().1, level + 1
-                            );
-                            trace!("Next level {}",
-                                next_level
-                            );
-                            trace!("Creating space for {} nodes",
-                                if next_level >= this_level { 1 << (next_level - this_level) } else { 1 }
-                            );
-                        }
+//                         if log_enabled!(log::Level::Trace) {
+//                             trace!("Empty node found, creating new node {} len{} lvl{}",
+//                                 $id, $id.get_id().1, level + 1
+//                             );
+//                             trace!("Next level {}",
+//                                 next_level
+//                             );
+//                             trace!("Creating space for {} nodes",
+//                                 if next_level >= this_level { 1 << (next_level - this_level) } else { 1 }
+//                             );
+//                         }
 
-                        trace!("multi uniq id {}", multi_uniq_id);
+//                         trace!("multi uniq id {}", multi_uniq_id);
 
-                        let node_set = NodeSet::init(next_level - this_level);
+//                         let node_set = NodeSet::init(next_level - this_level);
 
-                        let ptrbitarr = new_node.ptrbitarr.load();
-                        let pfxbitarr = new_node.pfxbitarr.load();
+//                         let ptrbitarr = new_node.ptrbitarr.load();
+//                         let pfxbitarr = new_node.pfxbitarr.load();
 
-                        let (stored_node, its_us) = nodes.read().get_or_init(
-                            index,
-                            || StoredNode {
-                                node_id: $id,
-                                node: new_node,
-                                node_set
-                            }
-                        );
+//                         let (stored_node, its_us) = nodes.read().get_or_init(
+//                             index,
+//                             || StoredNode {
+//                                 node_id: $id,
+//                                 node: new_node,
+//                                 node_set
+//                             }
+//                         );
 
-                        if stored_node.node_id == $id {
-                            stored_node.node_set.update_rbm_index(
-                                multi_uniq_id
-                            )?;
+//                         if stored_node.node_id == $id {
+//                             stored_node.node_set.update_rbm_index(
+//                                 multi_uniq_id
+//                             )?;
 
-                            if !its_us && ptrbitarr != 0 {
-                                stored_node.node.ptrbitarr.merge_with(ptrbitarr);
-                            }
+//                             if !its_us && ptrbitarr != 0 {
+//                                 stored_node.node.ptrbitarr.merge_with(ptrbitarr);
+//                             }
 
-                            if !its_us && pfxbitarr != 0 {
-                                stored_node.node.pfxbitarr.merge_with(pfxbitarr);
-                            }
-                        }
+//                             if !its_us && pfxbitarr != 0 {
+//                                 stored_node.node.pfxbitarr.merge_with(pfxbitarr);
+//                             }
+//                         }
 
-                        return Ok(($id, retry_count));
-                    }
-                    Some(stored_node) => {
-                        // A node exists, might be ours, might be
-                        // another one.
+//                         return Ok(($id, retry_count));
+//                     }
+//                     Some(stored_node) => {
+//                         // A node exists, might be ours, might be
+//                         // another one.
 
-                        if log_enabled!(log::Level::Trace) {
-                            trace!("
-                                {} store: Node here exists {:?}",
-                                    std::thread::current().name().unwrap_or("unnamed-thread"),
-                                    stored_node.node_id
-                            );
-                            trace!("node_id {:?}", stored_node.node_id.get_id());
-                            trace!("node_id {:032b}", stored_node.node_id.get_id().0);
-                            trace!("id {}", $id);
-                            trace!("     id {:032b}", $id.get_id().0);
-                        }
+//                         if log_enabled!(log::Level::Trace) {
+//                             trace!("
+//                                 {} store: Node here exists {:?}",
+//                                     std::thread::current().name().unwrap_or("unnamed-thread"),
+//                                     stored_node.node_id
+//                             );
+//                             trace!("node_id {:?}", stored_node.node_id.get_id());
+//                             trace!("node_id {:032b}", stored_node.node_id.get_id().0);
+//                             trace!("id {}", $id);
+//                             trace!("     id {:032b}", $id.get_id().0);
+//                         }
 
-                        // See if somebody beat us to creating our
-                        // node already, if so, we still need to do
-                        // work: we have to update the bitmap index
-                        // with the multi_uniq_id we've got from the
-                        // caller.
-                        if $id == stored_node.node_id {
-                            stored_node.node_set.update_rbm_index(
-                                multi_uniq_id
-                            )?;
+//                         // See if somebody beat us to creating our
+//                         // node already, if so, we still need to do
+//                         // work: we have to update the bitmap index
+//                         // with the multi_uniq_id we've got from the
+//                         // caller.
+//                         if $id == stored_node.node_id {
+//                             stored_node.node_set.update_rbm_index(
+//                                 multi_uniq_id
+//                             )?;
 
-                            if new_node.ptrbitarr.load() != 0 {
-                                stored_node.node.ptrbitarr.merge_with(new_node.ptrbitarr.load());
-                            }
-                            if new_node.pfxbitarr.load() != 0 {
-                                stored_node.node.pfxbitarr.merge_with(new_node.pfxbitarr.load());
-                            }
+//                             if new_node.ptrbitarr.load() != 0 {
+//                                 stored_node.node.ptrbitarr.merge_with(new_node.ptrbitarr.load());
+//                             }
+//                             if new_node.pfxbitarr.load() != 0 {
+//                                 stored_node.node.pfxbitarr.merge_with(new_node.pfxbitarr.load());
+//                             }
 
-                            return Ok(($id, retry_count));
-                        } else {
-                            // it's not "our" node, make a (recursive)
-                            // call to create it.
-                            level += 1;
-                            trace!("Collision with node_id {}, move to next level: {} len{} next_lvl{} index {}",
-                                stored_node.node_id, $id, $id.get_id().1, level, index
-                            );
+//                             return Ok(($id, retry_count));
+//                         } else {
+//                             // it's not "our" node, make a (recursive)
+//                             // call to create it.
+//                             level += 1;
+//                             trace!("Collision with node_id {}, move to next level: {} len{} next_lvl{} index {}",
+//                                 stored_node.node_id, $id, $id.get_id().1, level, index
+//                             );
 
-                            return match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
-                                // on to the next level!
-                                next_bit_shift if next_bit_shift > 0 => {
-                                    (search_level.f)(
-                                        search_level,
-                                        &stored_node.node_set,
-                                        new_node,
-                                        multi_uniq_id,
-                                        level,
-                                        retry_count
-                                    )
-                                }
-                                // There's no next level!
-                                _ => panic!("out of storage levels, current level is {}", level),
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    )*
-    };
-}
+//                             return match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
+//                                 // on to the next level!
+//                                 next_bit_shift if next_bit_shift > 0 => {
+//                                     (search_level.f)(
+//                                         search_level,
+//                                         &stored_node.node_set,
+//                                         new_node,
+//                                         multi_uniq_id,
+//                                         level,
+//                                         retry_count
+//                                     )
+//                                 }
+//                                 // There's no next level!
+//                                 _ => panic!("out of storage levels, current level is {}", level),
+//                             }
+//                         }
+//                     }
+//                 }
+//             }
+//         }
+//     )*
+//     };
+// }

From 66c86acf17e060cc10d161ca51aa2bb4e8edcd77 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 27 Feb 2025 17:06:55 +0100
Subject: [PATCH 089/147] eliminate Stride #1

---
 src/local_array/in_memory/atomic_stride.rs |  42 ++++----
 src/local_array/in_memory/iterators.rs     |  10 +-
 src/local_array/in_memory/macros.rs        |  30 +++---
 src/local_array/in_memory/node.rs          | 112 ++++++++++++---------
 src/local_array/in_memory/tree.rs          |   1 +
 src/local_array/prefix_cht/iterators.rs    |   6 +-
 6 files changed, 109 insertions(+), 92 deletions(-)

diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index ff8ce2c3..82800384 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -415,37 +415,37 @@ where
     // length, this follows from the fact that the `nibble` value represents
     // *both* the bitmap part, we're considering here *and* the position
     // relative to the nibble length offset in the bitmap.
-    fn get_bit_pos(
-        bit_pos: BitSpan,
-    ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
+    fn get_bit_pos(bit_pos: BitSpan) -> u32;
+    // ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
 
     fn get_bit_pos_as_u8(nibble: u32, len: u8) -> u8;
 
-    fn bit_pos_from_index(
-        i: u8,
-    ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
+    fn bit_pos_from_index(i: u8) -> u32;
+    // ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
 
-    fn ptr_bit_pos_from_index(
-        i: u8,
-    ) -> <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
+    fn ptr_bit_pos_from_index(i: u8) -> u16;
+    // ) -> <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
 
     fn cursor_from_bit_span(bs: BitSpan) -> u8;
 
     // fn ptr_cursor_from_bit_span(bs: BitSpan) -> u8;
 
     fn ptr_range(
-        ptrbitarr: <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::
-            InnerType,
+        // ptrbitarr: <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::
+        //     InnerType,
+        ptrbitarr: u16,
         range: BitSpan,
     ) -> (
-        <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
+        // <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
+        u16,
         u8,
     );
 
     fn ms_pfx_mask(
-        pfxbitarr: <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
+        // pfxbitarr: <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
+        pfxbitarr: u32,
         range: BitSpan,
-    ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
+    ) -> u32;
 
     // Clear the bitmap to the right of the pointer and count the number of
     // ones. This number represents the index to the corresponding prefix in
@@ -504,8 +504,9 @@ where
     // one bit to the left.
     #[allow(clippy::wrong_self_convention)]
     fn into_stride_size(
-        bitmap: <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
-    ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
+        bitmap: u16, // bitmap: <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
+                     // ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
+    ) -> u32;
 
     // Convert a pfxbitarr sized bitmap into a ptrbitarr sized
     // Note that bitwise operators align bits of unsigned types with
@@ -514,15 +515,16 @@ where
     // accommodate the unused pfxbitarr's last bit.
     #[allow(clippy::wrong_self_convention)]
     fn into_ptrbitarr_size(
-        bitmap: <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
-    ) -> <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
-
+        // bitmap: <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
+        bitmap: u32,
+    ) -> u16;
+    // ) -> <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
     fn leading_zeros(self) -> u32;
 }
 
 impl_primitive_atomic_stride![3; 16; u16; AtomicStride3; u8; AtomicStride2];
 impl_primitive_atomic_stride![4; 32; u32; AtomicStride4; u16; AtomicStride3];
-impl_primitive_atomic_stride![5; 64; u64; AtomicStride5; u32; AtomicStride4];
+// impl_primitive_atomic_stride![5; 64; u64; AtomicStride5; u32; AtomicStride4];
 // impl_primitive_stride![6; 128; u128; u64];
 
 // impl Stride for Stride7 {
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index d019074c..faafe247 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -60,9 +60,9 @@ pub(crate) struct MoreSpecificPrefixIter<
     NB: NodeBuckets<AF>,
 > {
     tree: &'a TreeBitMap<AF, NB>,
-    cur_ptr_iter: NodeMoreSpecificChildIter<AF, Stride4>,
-    cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF, Stride4>,
-    parent_and_position: Vec<NodeMoreSpecificChildIter<AF, Stride4>>,
+    cur_ptr_iter: NodeMoreSpecificChildIter<AF>,
+    cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF>,
+    parent_and_position: Vec<NodeMoreSpecificChildIter<AF>>,
 }
 
 impl<'a, AF: AddressFamily + 'a, NB: NodeBuckets<AF>> Iterator
@@ -304,8 +304,8 @@ impl<
                 start_bs.len
             );
 
-            let cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF, Stride4>;
-            let cur_ptr_iter: NodeMoreSpecificChildIter<AF, Stride4>;
+            let cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF>;
+            let cur_ptr_iter: NodeMoreSpecificChildIter<AF>;
             let node = self.retrieve_node(start_node_id);
 
             if let Some(node) = node {
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index 162b212f..a2f5f257 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -178,7 +178,7 @@ macro_rules! impl_primitive_atomic_stride {
                 const BITS: u8 = $bits;
                 const STRIDE_LEN: u8 = $len;
 
-                fn get_bit_pos(bs: BitSpan) -> $pfxsize {
+                fn get_bit_pos(bs: BitSpan) -> u32 {
                     // trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
                     1 << (
                             <Self as Stride>::BITS - ((1 << bs.len) - 1) as u8
@@ -186,14 +186,14 @@ macro_rules! impl_primitive_atomic_stride {
                     )
                 }
 
-                fn bit_pos_from_index(i: u8) -> $pfxsize {
-                    <$pfxsize>::try_from(1).unwrap().rotate_right(1) >> i
+                fn bit_pos_from_index(i: u8) -> u32 {
+                    <u32>::try_from(1).unwrap().rotate_right(1) >> i
                 }
 
-                fn ptr_bit_pos_from_index(i: u8) -> $ptrsize {
+                fn ptr_bit_pos_from_index(i: u8) -> u16 {
                     // trace!("pfx {} ptr {} strlen {}",
                     // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
-                    <$ptrsize>::try_from(1).unwrap().rotate_right(1)
+                    <u16>::try_from(1).unwrap().rotate_right(1)
                         >> (i + 1)
                 }
 
@@ -203,12 +203,12 @@ macro_rules! impl_primitive_atomic_stride {
                 }
 
                 fn ptr_range(
-                    ptrbitarr: $ptrsize,
+                    ptrbitarr: u16,
                     bs: BitSpan
-                ) -> ($ptrsize, u8) {
+                ) -> (u16, u8) {
                     let start: u8 = (bs.bits << (4 - bs.len)) as u8;
                     let stop: u8 = start + (1 << (4 - bs.len));
-                    let mask: $ptrsize = (
+                    let mask: u16 = (
                         (((1_u32 << (stop as u32 - start as u32)) - 1)
                              as u32
                     )
@@ -225,10 +225,10 @@ macro_rules! impl_primitive_atomic_stride {
                 }
 
                 fn ms_pfx_mask(
-                    pfxbitarr: $pfxsize,
+                    pfxbitarr: u32,
                     bs: BitSpan
-                ) -> $pfxsize {
-                    <$pfxsize>::try_from(
+                ) -> u32 {
+                    <u32>::try_from(
                         $crate::local_array::in_memory::node::
                         ms_prefix_mask_arr(bs) & pfxbitarr as u32
                     ).unwrap()
@@ -270,12 +270,12 @@ macro_rules! impl_primitive_atomic_stride {
                     id
                 }
 
-                fn into_stride_size(bitmap: $ptrsize) -> $pfxsize {
-                    bitmap as $pfxsize << 1
+                fn into_stride_size(bitmap: u16) -> u32 {
+                    (bitmap as u32) << 1
                 }
 
-                fn into_ptrbitarr_size(bitmap: $pfxsize) -> $ptrsize {
-                    (bitmap >> 1) as $ptrsize
+                fn into_ptrbitarr_size(bitmap: u32) -> u16 {
+                    (bitmap >> 1) as u16
                 }
 
                 #[inline]
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 7751a49f..d3ec23db 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -1,4 +1,5 @@
 use num_traits::PrimInt;
+use serde::de::value::U16Deserializer;
 use std::sync::atomic::{AtomicU16, AtomicU32};
 use std::{fmt::Debug, marker::PhantomData};
 
@@ -36,9 +37,12 @@ where
     S: Stride,
     AF: AddressFamily,
 {
-    pub ptrbitarr: <S as Stride>::AtomicPtrSize,
-    pub pfxbitarr: <S as Stride>::AtomicPfxSize,
+    // pub ptrbitarr: <S as Stride>::AtomicPtrSize,
+    pub ptrbitarr: AtomicStride3,
+    // pub pfxbitarr: <S as Stride>::AtomicPfxSize,
+    pub pfxbitarr: AtomicStride4,
     pub _af: PhantomData<AF>,
+    pub _s: PhantomData<S>,
 }
 
 impl<AF, S> Debug for TreeBitMapNode<AF, S>
@@ -76,9 +80,12 @@ where
 {
     pub(crate) fn new() -> Self {
         TreeBitMapNode {
-            ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
-            pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
+            // ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
+            ptrbitarr: AtomicStride3::new(),
+            pfxbitarr: AtomicStride4::new(),
+            // pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
             _af: PhantomData,
+            _s: PhantomData,
         }
     }
 
@@ -89,12 +96,12 @@ where
         &self,
         base_prefix: StrideNodeId<AF>,
         start_bs: BitSpan,
-    ) -> NodeMoreSpecificsPrefixIter<AF, S> {
+    ) -> NodeMoreSpecificsPrefixIter<AF> {
         debug_assert!(start_bs.check());
         let mut pfxbitarr = self.pfxbitarr.load();
         pfxbitarr = S::ms_pfx_mask(pfxbitarr, start_bs);
 
-        NodeMoreSpecificsPrefixIter::<AF, S> {
+        NodeMoreSpecificsPrefixIter::<AF> {
             pfxbitarr,
             base_prefix,
         }
@@ -106,12 +113,13 @@ where
         &self,
         base_prefix: StrideNodeId<AF>,
         start_bs: BitSpan,
-    ) -> NodeMoreSpecificChildIter<AF, S> {
+    ) -> NodeMoreSpecificChildIter<AF> {
         debug_assert!(start_bs.check());
         let ptrbitarr = self.ptrbitarr.load();
-        let (bitrange, start_cursor) = S::ptr_range(ptrbitarr, start_bs);
+        let (bitrange, start_cursor) =
+            Stride4::ptr_range(ptrbitarr, start_bs);
 
-        NodeMoreSpecificChildIter::<AF, S> {
+        NodeMoreSpecificChildIter::<AF> {
             bitrange,
             base_prefix,
             start_bs,
@@ -169,14 +177,15 @@ where
         if !is_last_stride {
             // We are not at the last stride
             // Check it the ptr bit is already set in this position
-            if (S::into_stride_size(ptrbitarr) & bit_pos) ==
-                <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::
-                    InnerType>::zero() {
+            if (S::into_stride_size(ptrbitarr) & bit_pos) == 0 {
+                // <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::
+                //     InnerType>::zero() {
                 // Nope, set it and create a child node
                 new_node = TreeBitMapNode {
                     ptrbitarr: AtomicStride3(AtomicU16::new(0)),
                     pfxbitarr: AtomicStride4(AtomicU32::new(0)),
                     _af: PhantomData,
+                    _s: PhantomData,
                 };
 
                 // THE CRITICAL SECTION
@@ -188,8 +197,9 @@ where
                 let mut a_ptrbitarr = self.ptrbitarr.compare_exchange(
                     ptrbitarr,
                     S::into_ptrbitarr_size(
-                    bit_pos | S::into_stride_size(ptrbitarr),
-                ));
+                        bit_pos | S::into_stride_size(ptrbitarr),
+                    ),
+                );
                 let mut spinwait = SpinWait::new();
                 loop {
                     match a_ptrbitarr {
@@ -203,24 +213,24 @@ where
                             a_ptrbitarr = self.ptrbitarr.compare_exchange(
                                 newer_array,
                                 S::into_ptrbitarr_size(
-                                bit_pos | S::into_stride_size(newer_array),
-                            ));
+                                    bit_pos
+                                        | S::into_stride_size(newer_array),
+                                ),
+                            );
                         }
                     };
                     spinwait.spin_no_yield();
                 }
 
-                return (NewNodeOrIndex::NewNode(
-                    new_node
-                ), retry_count);
+                return (NewNodeOrIndex::NewNode(new_node), retry_count);
             }
         } else {
             // only at the last stride do we create the bit in the prefix
             // bitmap, and only if it doesn't exist already
-            if pfxbitarr & bit_pos
-                == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::
-                    InnerType as std::ops::BitAnd>::Output::zero()
-            {
+            if pfxbitarr & bit_pos == 0 {
+                // == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::
+                //     InnerType as std::ops::BitAnd>::Output::zero()
+                // {
 
                 // THE CRITICAL SECTION
                 //
@@ -228,10 +238,9 @@ where
                 //
                 // preventing using an old pfxbitarr and overwrite bits set
                 // in the meantime elsewhere in the bitarray.
-                let mut a_pfxbitarr =
-                self.pfxbitarr.compare_exchange(
-                    pfxbitarr, bit_pos | pfxbitarr
-                );
+                let mut a_pfxbitarr = self
+                    .pfxbitarr
+                    .compare_exchange(pfxbitarr, bit_pos | pfxbitarr);
                 let mut spinwait = SpinWait::new();
 
                 loop {
@@ -244,7 +253,8 @@ where
                             // newer array.
                             retry_count += 1;
                             a_pfxbitarr = self.pfxbitarr.compare_exchange(
-                                newer_array, bit_pos | newer_array
+                                newer_array,
+                                bit_pos | newer_array,
                             );
                         }
                     };
@@ -349,19 +359,19 @@ type PfxBitArr<S> = <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
 // done now.
 
 #[derive(Debug, Copy, Clone)]
-pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily, S: Stride> {
+pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily> {
     base_prefix: StrideNodeId<AF>,
-    bitrange: PtrBitArr<S>,
+    bitrange: PtrBitArr<Stride4>,
     start_bs: BitSpan,
     start_cursor: u8,
 }
 
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator
-    for NodeMoreSpecificChildIter<AF, S>
+impl<AF: AddressFamily> std::iter::Iterator
+    for NodeMoreSpecificChildIter<AF>
 {
     type Item = StrideNodeId<AF>;
     fn next(&mut self) -> Option<Self::Item> {
-        if self.bitrange == PtrBitArr::<S>::zero() {
+        if self.bitrange == PtrBitArr::<Stride4>::zero() {
             trace!("empty ptrbitarr. This iterator is done.");
             return None;
         }
@@ -381,22 +391,23 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
             self.start_bs,
             self.start_cursor,
             <u8>::min(
-                (1 << (S::STRIDE_LEN - self.start_bs.len))
+                (1 << (Stride4::STRIDE_LEN - self.start_bs.len))
                     + self.start_cursor,
-                S::BITS - 2
+                Stride4::BITS as u8 - 2
             )
         );
 
         trace!("bitrange  {:032b}", self.bitrange);
 
-        self.bitrange = self.bitrange ^ S::ptr_bit_pos_from_index(cursor);
+        self.bitrange =
+            self.bitrange ^ Stride4::ptr_bit_pos_from_index(cursor);
 
-        trace!("mask      {:032b}", S::ptr_bit_pos_from_index(cursor));
+        trace!("mask      {:032b}", Stride4::ptr_bit_pos_from_index(cursor));
         trace!("next br   {:032b}", self.bitrange);
 
         let bs = BitSpan::from_bit_pos_index(cursor);
         if log_enabled!(log::Level::Trace) {
-            let bit_pos = S::ptr_bit_pos_from_index(cursor);
+            let bit_pos = Stride4::ptr_bit_pos_from_index(cursor);
             trace!(
                 "{:02}: {:05b} {:032b} bit_span: {:04b} ({:02}) (len: {})",
                 cursor,
@@ -417,7 +428,7 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
 
         let pfx = self.base_prefix.add_bit_span(BitSpan {
             bits: bs.bits,
-            len: S::STRIDE_LEN,
+            len: Stride4::STRIDE_LEN,
         });
         Some(pfx)
     }
@@ -546,19 +557,19 @@ pub const fn ms_prefix_mask_arr(bs: BitSpan) -> u32 {
 // the bit_span { bits: 2, len: 3 }, a.k.a. 0010 << 1. But now we will have
 // to go over a different amount of 1 << (5 - 4) = 2 iterations to reap the
 // next bit_spans of 0010 0 and 0010 1.
-pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily, S: Stride> {
+pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily> {
     base_prefix: StrideNodeId<AF>,
-    pfxbitarr: PfxBitArr<S>,
+    pfxbitarr: u32,
 }
 
-impl<AF: AddressFamily, S: Stride> std::iter::Iterator
-    for NodeMoreSpecificsPrefixIter<AF, S>
+impl<AF: AddressFamily> std::iter::Iterator
+    for NodeMoreSpecificsPrefixIter<AF>
 {
     type Item = PrefixId<AF>;
 
     fn next(&mut self) -> Option<Self::Item> {
         // Empty bitmap
-        if self.pfxbitarr == PfxBitArr::<S>::zero() {
+        if self.pfxbitarr == PfxBitArr::<Stride4>::zero() {
             trace!("empty pfxbitarr. This iterator is done.");
             return None;
         }
@@ -568,17 +579,17 @@ impl<AF: AddressFamily, S: Stride> std::iter::Iterator
         trace!(
             "ms prefix iterator start_bs {:?} start cursor {}",
             bs,
-            S::cursor_from_bit_span(bs)
+            Stride4::cursor_from_bit_span(bs)
         );
         trace!("pfx {:032b}", self.pfxbitarr);
-        let bit_pos = S::get_bit_pos(bs);
+        let bit_pos = Stride4::get_bit_pos(bs);
         let prefix_id: PrefixId<AF> = self
             .base_prefix
             .add_bit_span(BitSpan::from_bit_pos_index(
                 bit_pos.leading_zeros() as u8,
             ))
             .into();
-        self.pfxbitarr = self.pfxbitarr ^ S::bit_pos_from_index(cursor);
+        self.pfxbitarr = self.pfxbitarr ^ Stride4::bit_pos_from_index(cursor);
         Some(prefix_id)
     }
 }
@@ -590,8 +601,11 @@ where
 {
     fn default() -> Self {
         Self {
-            ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
-            pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
+            // ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
+            // pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
+            ptrbitarr: AtomicStride3::new(),
+            pfxbitarr: AtomicStride4::new(),
+            _s: PhantomData,
             _af: PhantomData,
         }
     }
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index d4df7d7d..e25312b1 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -235,6 +235,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                 ptrbitarr: AtomicStride3(AtomicU16::new(0)),
                 pfxbitarr: AtomicStride4(AtomicU32::new(0)),
                 _af: PhantomData,
+                _s: PhantomData,
             },
         )?;
 
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
index b63ddf15..515d49e3 100644
--- a/src/local_array/prefix_cht/iterators.rs
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -19,9 +19,9 @@ pub(crate) struct MoreSpecificPrefixIter<
     NB: NodeBuckets<AF>,
 > {
     store: &'a TreeBitMap<AF, NB>,
-    cur_ptr_iter: NodeMoreSpecificChildIter<AF, Stride4>,
-    cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF, Stride4>,
-    parent_and_position: Vec<NodeMoreSpecificChildIter<AF, Stride4>>,
+    cur_ptr_iter: NodeMoreSpecificChildIter<AF>,
+    cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF>,
+    parent_and_position: Vec<NodeMoreSpecificChildIter<AF>>,
     // If specified, we're only iterating over records for this mui.
     mui: Option<u32>,
     // This is the tree-wide index of withdrawn muis, used to rewrite the

From 4a215d048f28d79df7f17dadce62497b7680aaa0 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 27 Feb 2025 18:09:24 +0100
Subject: [PATCH 090/147] eliminate Stride #2

---
 src/local_array/in_memory/atomic_stride.rs | 382 ++++++---------------
 src/local_array/in_memory/atomic_types.rs  |   2 +-
 src/local_array/in_memory/iterators.rs     |   1 -
 src/local_array/in_memory/macros.rs        |  13 +-
 src/local_array/in_memory/node.rs          |  82 ++---
 src/local_array/in_memory/tree.rs          |   9 +-
 src/local_array/rib/default_store.rs       |   1 +
 7 files changed, 140 insertions(+), 350 deletions(-)

diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 82800384..81092d88 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -1,26 +1,17 @@
 use log::trace;
 use parking_lot_core::SpinWait;
 use std::fmt::{Binary, Debug};
-use std::sync::atomic::{
-    fence, AtomicU16, AtomicU32, AtomicU64, AtomicU8, Ordering,
-};
+use std::sync::atomic::{fence, AtomicU16, AtomicU32, Ordering};
 
 use crate::af::Zero;
+use crate::local_array;
 use crate::local_array::bit_span::BitSpan;
-use crate::synth_int::AtomicU128;
-use crate::{impl_primitive_atomic_stride, AddressFamily};
+use crate::local_array::rib::default_store::STRIDE_BITS;
 
-use super::node::StrideNodeId;
+pub type Stride4 = AtomicStride4;
 
-pub type Stride3 = u16;
-pub type Stride4 = u32;
-pub type Stride5 = u64;
-
-pub struct AtomicStride2(pub AtomicU8);
 pub struct AtomicStride3(pub AtomicU16);
 pub struct AtomicStride4(pub AtomicU32);
-pub struct AtomicStride5(pub AtomicU64);
-pub struct AtomicStride6(pub AtomicU128);
 
 pub struct CasResult<InnerType>(pub Result<InnerType, InnerType>);
 
@@ -72,46 +63,46 @@ where
     }
 }
 
-impl AtomicBitmap for AtomicStride2 {
-    type InnerType = u8;
+// impl AtomicBitmap for AtomicStride2 {
+//     type InnerType = u8;
 
-    fn new() -> Self {
-        AtomicStride2(AtomicU8::new(0))
-    }
+//     fn new() -> Self {
+//         AtomicStride2(AtomicU8::new(0))
+//     }
 
-    fn compare_exchange(
-        &self,
-        current: Self::InnerType,
-        new: Self::InnerType,
-    ) -> CasResult<Self::InnerType> {
-        CasResult(self.0.compare_exchange(
-            current,
-            new,
-            Ordering::Acquire,
-            Ordering::Relaxed,
-        ))
-    }
+//     fn compare_exchange(
+//         &self,
+//         current: Self::InnerType,
+//         new: Self::InnerType,
+//     ) -> CasResult<Self::InnerType> {
+//         CasResult(self.0.compare_exchange(
+//             current,
+//             new,
+//             Ordering::Acquire,
+//             Ordering::Relaxed,
+//         ))
+//     }
 
-    fn load(&self) -> Self::InnerType {
-        self.0.load(Ordering::SeqCst)
-    }
-}
+//     fn load(&self) -> Self::InnerType {
+//         self.0.load(Ordering::SeqCst)
+//     }
+// }
 
-impl Zero for AtomicStride2 {
-    fn zero() -> Self {
-        AtomicStride2(AtomicU8::new(0))
-    }
+// impl Zero for AtomicStride2 {
+//     fn zero() -> Self {
+//         AtomicStride2(AtomicU8::new(0))
+//     }
 
-    fn is_zero(&self) -> bool {
-        self.0.load(Ordering::SeqCst) == 0
-    }
-}
+//     fn is_zero(&self) -> bool {
+//         self.0.load(Ordering::SeqCst) == 0
+//     }
+// }
 
-impl From<u8> for AtomicStride2 {
-    fn from(value: u8) -> Self {
-        Self(AtomicU8::new(value))
-    }
-}
+// impl From<u8> for AtomicStride2 {
+//     fn from(value: u8) -> Self {
+//         Self(AtomicU8::new(value))
+//     }
+// }
 
 impl AtomicBitmap for AtomicStride3 {
     type InnerType = u16;
@@ -191,211 +182,8 @@ impl Zero for AtomicStride4 {
     }
 }
 
-impl AtomicBitmap for AtomicStride5 {
-    type InnerType = u64;
-
-    fn new() -> Self {
-        AtomicStride5(AtomicU64::new(0))
-    }
-    fn compare_exchange(
-        &self,
-        current: Self::InnerType,
-        new: Self::InnerType,
-    ) -> CasResult<Self::InnerType> {
-        CasResult(self.0.compare_exchange(
-            current,
-            new,
-            Ordering::SeqCst,
-            Ordering::SeqCst,
-        ))
-    }
-    fn load(&self) -> Self::InnerType {
-        self.0.load(Ordering::SeqCst)
-    }
-}
-
-impl From<u64> for AtomicStride5 {
-    fn from(value: u64) -> Self {
-        Self(AtomicU64::new(value))
-    }
-}
-
-impl Zero for AtomicStride5 {
-    fn zero() -> Self {
-        AtomicStride5(AtomicU64::new(0))
-    }
-
-    fn is_zero(&self) -> bool {
-        self.0.load(Ordering::SeqCst) == 0
-    }
-}
-
-impl AtomicBitmap for AtomicStride6 {
-    type InnerType = u128;
-
-    fn new() -> Self {
-        AtomicStride6(AtomicU128::new(0))
-    }
-    fn compare_exchange(
-        &self,
-        current: Self::InnerType,
-        new: Self::InnerType,
-    ) -> CasResult<Self::InnerType> {
-        // TODO TODO
-        // This is not actually thread-safe, it actually
-        // needs a memory fence, since we're writing
-        // to two different memory locations.
-        (
-            self.0 .0.compare_exchange(
-                ((current << 64) >> 64) as u64,
-                ((new >> 64) << 64) as u64,
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ),
-            self.0 .1.compare_exchange(
-                ((current << 64) >> 64) as u64,
-                ((new >> 64) << 64) as u64,
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ),
-        )
-            .into()
-    }
-    fn load(&self) -> Self::InnerType {
-        let hi = self.0 .0.load(Ordering::SeqCst).to_be_bytes();
-        let lo = self.0 .1.load(Ordering::SeqCst).to_be_bytes();
-        u128::from_be_bytes([
-            hi[0], hi[1], hi[2], hi[3], hi[4], hi[5], hi[6], hi[7], lo[0],
-            lo[1], lo[2], lo[3], lo[4], lo[5], lo[6], lo[7],
-        ])
-    }
-
-    fn merge_with(&self, _node: Self::InnerType) {
-        todo!()
-    }
-}
-
-impl From<u128> for AtomicStride6 {
-    fn from(value: u128) -> Self {
-        Self(AtomicU128::new(value))
-    }
-}
-
-impl Zero for AtomicStride6 {
-    fn zero() -> Self {
-        AtomicStride6(AtomicU128::new(0))
-    }
-
-    fn is_zero(&self) -> bool {
-        self.0 .0.load(Ordering::SeqCst) == 0
-            && self.0 .1.load(Ordering::SeqCst) == 0
-    }
-}
-
-impl From<(Result<u64, u64>, Result<u64, u64>)> for CasResult<u128> {
-    fn from(r: (Result<u64, u64>, Result<u64, u64>)) -> Self {
-        match r {
-            (Ok(hi), Ok(lo)) => CasResult::new(u128::from_be_bytes([
-                hi.to_be_bytes()[0],
-                hi.to_be_bytes()[1],
-                hi.to_be_bytes()[2],
-                hi.to_be_bytes()[3],
-                hi.to_be_bytes()[4],
-                hi.to_be_bytes()[5],
-                hi.to_be_bytes()[6],
-                hi.to_be_bytes()[7],
-                lo.to_be_bytes()[0],
-                lo.to_be_bytes()[1],
-                lo.to_be_bytes()[2],
-                lo.to_be_bytes()[3],
-                lo.to_be_bytes()[4],
-                lo.to_be_bytes()[5],
-                lo.to_be_bytes()[6],
-                lo.to_be_bytes()[7],
-            ])),
-            (Err(hi), Ok(lo)) => CasResult(Err(u128::from_be_bytes([
-                hi.to_be_bytes()[0],
-                hi.to_be_bytes()[1],
-                hi.to_be_bytes()[2],
-                hi.to_be_bytes()[3],
-                hi.to_be_bytes()[4],
-                hi.to_be_bytes()[5],
-                hi.to_be_bytes()[6],
-                hi.to_be_bytes()[7],
-                lo.to_be_bytes()[0],
-                lo.to_be_bytes()[1],
-                lo.to_be_bytes()[2],
-                lo.to_be_bytes()[3],
-                lo.to_be_bytes()[4],
-                lo.to_be_bytes()[5],
-                lo.to_be_bytes()[6],
-                lo.to_be_bytes()[7],
-            ]))),
-            (Ok(hi), Err(lo)) => CasResult(Err(u128::from_be_bytes([
-                hi.to_be_bytes()[0],
-                hi.to_be_bytes()[1],
-                hi.to_be_bytes()[2],
-                hi.to_be_bytes()[3],
-                hi.to_be_bytes()[4],
-                hi.to_be_bytes()[5],
-                hi.to_be_bytes()[6],
-                hi.to_be_bytes()[7],
-                lo.to_be_bytes()[0],
-                lo.to_be_bytes()[1],
-                lo.to_be_bytes()[2],
-                lo.to_be_bytes()[3],
-                lo.to_be_bytes()[4],
-                lo.to_be_bytes()[5],
-                lo.to_be_bytes()[6],
-                lo.to_be_bytes()[7],
-            ]))),
-            (Err(hi), Err(lo)) => CasResult(Err(u128::from_be_bytes([
-                hi.to_be_bytes()[0],
-                hi.to_be_bytes()[1],
-                hi.to_be_bytes()[2],
-                hi.to_be_bytes()[3],
-                hi.to_be_bytes()[4],
-                hi.to_be_bytes()[5],
-                hi.to_be_bytes()[6],
-                hi.to_be_bytes()[7],
-                lo.to_be_bytes()[0],
-                lo.to_be_bytes()[1],
-                lo.to_be_bytes()[2],
-                lo.to_be_bytes()[3],
-                lo.to_be_bytes()[4],
-                lo.to_be_bytes()[5],
-                lo.to_be_bytes()[6],
-                lo.to_be_bytes()[7],
-            ]))),
-        }
-    }
-}
-pub(crate) trait Stride:
-    Sized
-    + Debug
-    + Eq
-    + Binary
-    + PartialOrd
-    + PartialEq
-    + Zero
-    + std::ops::BitAnd<Output = Self>
-    + std::ops::BitOr<Output = Self>
-where
-    Self::AtomicPtrSize: AtomicBitmap,
-    Self::AtomicPfxSize: AtomicBitmap,
-    Self::PtrSize: Zero
-        + Binary
-        + Copy
-        + Debug
-        + std::ops::BitAnd<Output = Self::PtrSize>
-        + PartialOrd
-        + Zero,
-{
-    type AtomicPfxSize;
-    type AtomicPtrSize;
-    type PtrSize;
-    const BITS: u8;
-    const STRIDE_LEN: u8;
+pub(crate) trait Stride {
+    // const STRIDE_LEN: u8;
 
     // Get the bit position of the start of the given nibble.
     // The nibble is defined as a `len` number of bits set from the right.
@@ -418,28 +206,13 @@ where
     fn get_bit_pos(bit_pos: BitSpan) -> u32;
     // ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
 
-    fn get_bit_pos_as_u8(nibble: u32, len: u8) -> u8;
-
     fn bit_pos_from_index(i: u8) -> u32;
-    // ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
 
     fn ptr_bit_pos_from_index(i: u8) -> u16;
-    // ) -> <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
 
     fn cursor_from_bit_span(bs: BitSpan) -> u8;
 
-    // fn ptr_cursor_from_bit_span(bs: BitSpan) -> u8;
-
-    fn ptr_range(
-        // ptrbitarr: <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::
-        //     InnerType,
-        ptrbitarr: u16,
-        range: BitSpan,
-    ) -> (
-        // <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
-        u16,
-        u8,
-    );
+    fn ptr_range(ptrbitarr: u16, range: BitSpan) -> (u16, u8);
 
     fn ms_pfx_mask(
         // pfxbitarr: <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
@@ -464,7 +237,6 @@ where
 
     // get_pfx_index only needs nibble and len for fixed-layout bitarrays,
     // since the index can be deducted from them.
-    fn get_pfx_index(bit_span: BitSpan) -> usize;
 
     // Clear the bitmap to the right of the pointer and count the number of
     // ones. This number represents the index to the corresponding child node
@@ -485,16 +257,6 @@ where
     // The bit position relative to the offset for the nibble length, this
     // index is only used at the last (relevant) stride, so the offset is
     // always 0.
-    fn get_ptr_index(
-        bitmap: <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
-        nibble: u32,
-    ) -> usize;
-
-    #[allow(clippy::wrong_self_convention)]
-    fn into_node_id<AF: AddressFamily>(
-        addr_bits: AF,
-        len: u8,
-    ) -> StrideNodeId<AF>;
 
     // Convert a ptrbitarr into a pfxbitarr sized bitmap,
     // so we can do bitwise operations with a pfxbitarr sized
@@ -502,7 +264,6 @@ where
     // Since the last bit in the pfxbitarr isn't used, but the
     // full ptrbitarr *is* used, the prtbitarr should be shifted
     // one bit to the left.
-    #[allow(clippy::wrong_self_convention)]
     fn into_stride_size(
         bitmap: u16, // bitmap: <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
                      // ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
@@ -513,17 +274,15 @@ where
     // different sizes to the right, so we don't have to do anything to pad
     // the smaller sized type. We do have to shift one bit to the left, to
     // accommodate the unused pfxbitarr's last bit.
-    #[allow(clippy::wrong_self_convention)]
     fn into_ptrbitarr_size(
         // bitmap: <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
         bitmap: u32,
     ) -> u16;
     // ) -> <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
-    fn leading_zeros(self) -> u32;
 }
 
-impl_primitive_atomic_stride![3; 16; u16; AtomicStride3; u8; AtomicStride2];
-impl_primitive_atomic_stride![4; 32; u32; AtomicStride4; u16; AtomicStride3];
+// impl_primitive_atomic_stride![3; 16; u16; AtomicStride3; u8; AtomicStride2];
+// impl_primitive_atomic_stride![4; 32; u32; AtomicStride4; u16; AtomicStride3];
 // impl_primitive_atomic_stride![5; 64; u64; AtomicStride5; u32; AtomicStride4];
 // impl_primitive_stride![6; 128; u128; u64];
 
@@ -687,3 +446,58 @@ impl_primitive_atomic_stride![4; 32; u32; AtomicStride4; u16; AtomicStride3];
 //         lz
 //     }
 // }
+impl Stride for AtomicStride4 {
+    // type AtomicPfxSize = $atomicpfxsize;
+    // type AtomicPtrSize = $atomicptrsize;
+    // type PtrSize = $ptrsize;
+    // const BITS: u8 = $bits;
+    // const STRIDE_LEN: u8 = 4;
+
+    fn get_bit_pos(bs: BitSpan) -> u32 {
+        // trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
+        1 << (STRIDE_BITS - ((1 << bs.len) - 1) as u8 - bs.bits as u8 - 1)
+    }
+
+    fn bit_pos_from_index(i: u8) -> u32 {
+        <u32>::try_from(1).unwrap().rotate_right(1) >> i
+    }
+
+    fn ptr_bit_pos_from_index(i: u8) -> u16 {
+        // trace!("pfx {} ptr {} strlen {}",
+        // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
+        <u16>::try_from(1).unwrap().rotate_right(1) >> (i + 1)
+    }
+
+    fn cursor_from_bit_span(bs: BitSpan) -> u8 {
+        Self::get_bit_pos(bs).leading_zeros() as u8
+    }
+
+    fn ptr_range(ptrbitarr: u16, bs: BitSpan) -> (u16, u8) {
+        let start: u8 = (bs.bits << (4 - bs.len)) as u8;
+        let stop: u8 = start + (1 << (4 - bs.len));
+        let mask: u16 = (((1_u32 << (stop as u32 - start as u32)) - 1)
+            .rotate_right(stop as u32)
+            >> 16)
+            .try_into()
+            .unwrap();
+        trace!("- mask      {:032b}", mask);
+        trace!("- ptrbitarr {:032b}", ptrbitarr);
+        trace!("- shl bitar {:032b}", ptrbitarr & mask);
+
+        // if ptrbitarr & mask == <$ptrsize>::zero() { panic!("stop"); }
+
+        (ptrbitarr & mask, start)
+    }
+
+    fn ms_pfx_mask(pfxbitarr: u32, bs: BitSpan) -> u32 {
+        local_array::in_memory::node::ms_prefix_mask_arr(bs) & pfxbitarr
+    }
+
+    fn into_stride_size(bitmap: u16) -> u32 {
+        (bitmap as u32) << 1
+    }
+
+    fn into_ptrbitarr_size(bitmap: u32) -> u16 {
+        (bitmap >> 1) as u16
+    }
+}
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 43a254e1..a59db3eb 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -35,7 +35,7 @@ where
 {
     pub(crate) node_id: StrideNodeId<AF>,
     // The ptrbitarr and pfxbitarr for this node
-    pub(crate) node: TreeBitMapNode<AF, Stride4>,
+    pub(crate) node: TreeBitMapNode<AF>,
     // Child nodes linked from this node
     pub(crate) node_set: NodeSet<AF>,
 }
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index faafe247..3acec37b 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -20,7 +20,6 @@
 // in the TreeBitMap.
 
 use crate::local_array::in_memory::atomic_types::NodeBuckets;
-use crate::local_array::in_memory::tree::Stride4;
 use crate::local_array::in_memory::tree::TreeBitMap;
 use crate::{
     af::AddressFamily,
diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
index a2f5f257..b01ef872 100644
--- a/src/local_array/in_memory/macros.rs
+++ b/src/local_array/in_memory/macros.rs
@@ -234,20 +234,9 @@ macro_rules! impl_primitive_atomic_stride {
                     ).unwrap()
                 }
 
-                // Ptrbitarr searches are only done in the last half of
-                // the bitarray, in the len = S::STRIDE_LEN part. We need a
-                // complete BitSpan still to figure when to stop.
-                // fn ptr_cursor_from_bit_span(bs: BitSpan) -> u8 {
-                //     let p = Self::get_bit_pos(bs.bits << (4 - bs.len), 4)
-                //         .leading_zeros() as u8;
-                //     trace!("bs in {:?}", bs);
-                //     trace!("pos {}", p);
-                //     p
-                // }
-
                 fn get_bit_pos_as_u8(nibble: u32, len: u8) -> u8 {
                     1 << (
-                            <Self as Stride>::BITS - ((1 << len) - 1) as u8
+                            STRIDE_BITS - ((1 << len) - 1) as u8
                             - nibble as u8 - 1
                     )
                 }
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index d3ec23db..351ab2ab 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -13,7 +13,7 @@ use super::tree::{
 
 use crate::af::AddressFamily;
 use crate::af::Zero;
-use crate::local_array::rib::default_store::STRIDE_SIZE;
+use crate::local_array::rib::default_store::{STRIDE_BITS, STRIDE_SIZE};
 use crate::local_array::types::PrefixId;
 
 //------------ TreeBitMap Node ----------------------------------------------
@@ -31,24 +31,19 @@ use crate::local_array::types::PrefixId;
 // in a treebitmap node is enabled by the storage backend for the
 // multi-threaded store, since holds its entries keyed on the [node|prefix]
 // id. (in contrast with arrays or `vec`s, that have
-pub struct TreeBitMapNode<AF, S>
+pub struct TreeBitMapNode<AF>
 where
     Self: Sized,
-    S: Stride,
     AF: AddressFamily,
 {
-    // pub ptrbitarr: <S as Stride>::AtomicPtrSize,
     pub ptrbitarr: AtomicStride3,
-    // pub pfxbitarr: <S as Stride>::AtomicPfxSize,
     pub pfxbitarr: AtomicStride4,
     pub _af: PhantomData<AF>,
-    pub _s: PhantomData<S>,
 }
 
-impl<AF, S> Debug for TreeBitMapNode<AF, S>
+impl<AF> Debug for TreeBitMapNode<AF>
 where
     AF: AddressFamily,
-    S: Stride,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("TreeBitMapNode")
@@ -58,10 +53,9 @@ where
     }
 }
 
-impl<AF, S> std::fmt::Display for TreeBitMapNode<AF, S>
+impl<AF> std::fmt::Display for TreeBitMapNode<AF>
 where
     AF: AddressFamily,
-    S: Stride,
 {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
@@ -73,19 +67,15 @@ where
     }
 }
 
-impl<AF, S> TreeBitMapNode<AF, S>
+impl<AF> TreeBitMapNode<AF>
 where
     AF: AddressFamily,
-    S: Stride,
 {
     pub(crate) fn new() -> Self {
         TreeBitMapNode {
-            // ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
             ptrbitarr: AtomicStride3::new(),
             pfxbitarr: AtomicStride4::new(),
-            // pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
             _af: PhantomData,
-            _s: PhantomData,
         }
     }
 
@@ -99,7 +89,7 @@ where
     ) -> NodeMoreSpecificsPrefixIter<AF> {
         debug_assert!(start_bs.check());
         let mut pfxbitarr = self.pfxbitarr.load();
-        pfxbitarr = S::ms_pfx_mask(pfxbitarr, start_bs);
+        pfxbitarr = AtomicStride4::ms_pfx_mask(pfxbitarr, start_bs);
 
         NodeMoreSpecificsPrefixIter::<AF> {
             pfxbitarr,
@@ -117,7 +107,7 @@ where
         debug_assert!(start_bs.check());
         let ptrbitarr = self.ptrbitarr.load();
         let (bitrange, start_cursor) =
-            Stride4::ptr_range(ptrbitarr, start_bs);
+            AtomicStride4::ptr_range(ptrbitarr, start_bs);
 
         NodeMoreSpecificChildIter::<AF> {
             bitrange,
@@ -163,8 +153,8 @@ where
         let mut retry_count = 0;
         let ptrbitarr = self.ptrbitarr.load();
         let pfxbitarr = self.pfxbitarr.load();
-        let bit_pos = S::get_bit_pos(bit_span);
-        let new_node: TreeBitMapNode<AF, Stride4>;
+        let bit_pos = AtomicStride4::get_bit_pos(bit_span);
+        let new_node: TreeBitMapNode<AF>;
 
         // Check that we're not at the last stride (pfx.len <= stride_end),
         // Note that next_stride may have a value, but we still don't want to
@@ -177,7 +167,7 @@ where
         if !is_last_stride {
             // We are not at the last stride
             // Check it the ptr bit is already set in this position
-            if (S::into_stride_size(ptrbitarr) & bit_pos) == 0 {
+            if (AtomicStride4::into_stride_size(ptrbitarr) & bit_pos) == 0 {
                 // <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::
                 //     InnerType>::zero() {
                 // Nope, set it and create a child node
@@ -185,7 +175,6 @@ where
                     ptrbitarr: AtomicStride3(AtomicU16::new(0)),
                     pfxbitarr: AtomicStride4(AtomicU32::new(0)),
                     _af: PhantomData,
-                    _s: PhantomData,
                 };
 
                 // THE CRITICAL SECTION
@@ -196,8 +185,8 @@ where
                 // in the meantime elsewhere in the bitarray.
                 let mut a_ptrbitarr = self.ptrbitarr.compare_exchange(
                     ptrbitarr,
-                    S::into_ptrbitarr_size(
-                        bit_pos | S::into_stride_size(ptrbitarr),
+                    AtomicStride4::into_ptrbitarr_size(
+                        bit_pos | AtomicStride4::into_stride_size(ptrbitarr),
                     ),
                 );
                 let mut spinwait = SpinWait::new();
@@ -212,9 +201,11 @@ where
                             retry_count += 1;
                             a_ptrbitarr = self.ptrbitarr.compare_exchange(
                                 newer_array,
-                                S::into_ptrbitarr_size(
+                                AtomicStride4::into_ptrbitarr_size(
                                     bit_pos
-                                        | S::into_stride_size(newer_array),
+                                        | AtomicStride4::into_stride_size(
+                                            newer_array,
+                                        ),
                                 ),
                             );
                         }
@@ -308,8 +299,8 @@ where
 // child pfxs len      /27-29 /30-32
 // child Nodes len     /29    /32
 
-type PtrBitArr<S> = <<S as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
-type PfxBitArr<S> = <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
+type PtrBitArr = u16;
+type PfxBitArr = u32;
 
 // ----------- NodeMoreSpecificChildIter ------------------------------------
 
@@ -361,7 +352,7 @@ type PfxBitArr<S> = <<S as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
 #[derive(Debug, Copy, Clone)]
 pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily> {
     base_prefix: StrideNodeId<AF>,
-    bitrange: PtrBitArr<Stride4>,
+    bitrange: PtrBitArr,
     start_bs: BitSpan,
     start_cursor: u8,
 }
@@ -371,7 +362,7 @@ impl<AF: AddressFamily> std::iter::Iterator
 {
     type Item = StrideNodeId<AF>;
     fn next(&mut self) -> Option<Self::Item> {
-        if self.bitrange == PtrBitArr::<Stride4>::zero() {
+        if self.bitrange == 0 {
             trace!("empty ptrbitarr. This iterator is done.");
             return None;
         }
@@ -391,23 +382,24 @@ impl<AF: AddressFamily> std::iter::Iterator
             self.start_bs,
             self.start_cursor,
             <u8>::min(
-                (1 << (Stride4::STRIDE_LEN - self.start_bs.len))
-                    + self.start_cursor,
-                Stride4::BITS as u8 - 2
+                (1 << (STRIDE_SIZE - self.start_bs.len)) + self.start_cursor,
+                STRIDE_BITS - 2
             )
         );
 
         trace!("bitrange  {:032b}", self.bitrange);
 
-        self.bitrange =
-            self.bitrange ^ Stride4::ptr_bit_pos_from_index(cursor);
+        self.bitrange ^= AtomicStride4::ptr_bit_pos_from_index(cursor);
 
-        trace!("mask      {:032b}", Stride4::ptr_bit_pos_from_index(cursor));
+        trace!(
+            "mask      {:032b}",
+            AtomicStride4::ptr_bit_pos_from_index(cursor)
+        );
         trace!("next br   {:032b}", self.bitrange);
 
         let bs = BitSpan::from_bit_pos_index(cursor);
         if log_enabled!(log::Level::Trace) {
-            let bit_pos = Stride4::ptr_bit_pos_from_index(cursor);
+            let bit_pos = AtomicStride4::ptr_bit_pos_from_index(cursor);
             trace!(
                 "{:02}: {:05b} {:032b} bit_span: {:04b} ({:02}) (len: {})",
                 cursor,
@@ -428,7 +420,7 @@ impl<AF: AddressFamily> std::iter::Iterator
 
         let pfx = self.base_prefix.add_bit_span(BitSpan {
             bits: bs.bits,
-            len: Stride4::STRIDE_LEN,
+            len: STRIDE_SIZE,
         });
         Some(pfx)
     }
@@ -569,7 +561,7 @@ impl<AF: AddressFamily> std::iter::Iterator
 
     fn next(&mut self) -> Option<Self::Item> {
         // Empty bitmap
-        if self.pfxbitarr == PfxBitArr::<Stride4>::zero() {
+        if self.pfxbitarr == 0 {
             trace!("empty pfxbitarr. This iterator is done.");
             return None;
         }
@@ -579,40 +571,36 @@ impl<AF: AddressFamily> std::iter::Iterator
         trace!(
             "ms prefix iterator start_bs {:?} start cursor {}",
             bs,
-            Stride4::cursor_from_bit_span(bs)
+            AtomicStride4::cursor_from_bit_span(bs)
         );
         trace!("pfx {:032b}", self.pfxbitarr);
-        let bit_pos = Stride4::get_bit_pos(bs);
+        let bit_pos = AtomicStride4::get_bit_pos(bs);
         let prefix_id: PrefixId<AF> = self
             .base_prefix
             .add_bit_span(BitSpan::from_bit_pos_index(
                 bit_pos.leading_zeros() as u8,
             ))
             .into();
-        self.pfxbitarr = self.pfxbitarr ^ Stride4::bit_pos_from_index(cursor);
+        self.pfxbitarr ^= AtomicStride4::bit_pos_from_index(cursor);
         Some(prefix_id)
     }
 }
 
-impl<AF, S> Default for TreeBitMapNode<AF, S>
+impl<AF> Default for TreeBitMapNode<AF>
 where
     AF: AddressFamily,
-    S: Stride,
 {
     fn default() -> Self {
         Self {
-            // ptrbitarr: <<S as Stride>::AtomicPtrSize as AtomicBitmap>::new(),
-            // pfxbitarr: <<S as Stride>::AtomicPfxSize as AtomicBitmap>::new(),
             ptrbitarr: AtomicStride3::new(),
             pfxbitarr: AtomicStride4::new(),
-            _s: PhantomData,
             _af: PhantomData,
         }
     }
 }
 
 pub(crate) enum NewNodeOrIndex<AF: AddressFamily> {
-    NewNode(TreeBitMapNode<AF, Stride4>),
+    NewNode(TreeBitMapNode<AF>),
     ExistingNode(StrideNodeId<AF>),
     NewPrefix,
     ExistingPrefix,
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index e25312b1..06293af5 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -235,7 +235,6 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                 ptrbitarr: AtomicStride3(AtomicU16::new(0)),
                 pfxbitarr: AtomicStride4(AtomicU32::new(0)),
                 _af: PhantomData,
-                _s: PhantomData,
             },
         )?;
 
@@ -490,7 +489,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         &self,
         id: StrideNodeId<AF>,
         multi_uniq_id: u32,
-        next_node: TreeBitMapNode<AF, Stride4>,
+        next_node: TreeBitMapNode<AF>,
     ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
         if log_enabled!(log::Level::Trace) {
             debug!(
@@ -791,7 +790,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         &self,
         id: StrideNodeId<AF>,
         mui: u32,
-    ) -> Option<&TreeBitMapNode<AF, Stride4>> {
+    ) -> Option<&TreeBitMapNode<AF>> {
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
@@ -881,7 +880,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
     pub fn retrieve_node(
         &self,
         id: StrideNodeId<AF>,
-    ) -> Option<&TreeBitMapNode<AF, Stride4>> {
+    ) -> Option<&TreeBitMapNode<AF>> {
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
@@ -969,7 +968,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         &self,
         id: StrideNodeId<AF>,
         mui: u32,
-    ) -> Option<&TreeBitMapNode<AF, Stride4>> {
+    ) -> Option<&TreeBitMapNode<AF>> {
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index f8c6fc14..ab4c2291 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -5,6 +5,7 @@ use crate::prelude::*;
 use rand::prelude::*;
 
 pub const STRIDE_SIZE: u8 = 4;
+pub const STRIDE_BITS: u8 = 32;
 
 // The default stride sizes for IPv4, IPv6, resp.
 #[create_store((

From 677d5265d8714bda410da180d7964d8f02fd225c Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 28 Feb 2025 13:26:43 +0100
Subject: [PATCH 091/147] eliminate Stride #3

---
 src/local_array/bit_span.rs                |  10 +
 src/local_array/in_memory/atomic_stride.rs | 555 +++++++--------------
 src/local_array/in_memory/atomic_types.rs  |   2 -
 src/local_array/in_memory/node.rs          |  72 +--
 src/local_array/in_memory/tree.rs          |   8 +-
 src/local_array/prefix_cht/iterators.rs    |   2 +-
 6 files changed, 217 insertions(+), 432 deletions(-)

diff --git a/src/local_array/bit_span.rs b/src/local_array/bit_span.rs
index be1e03f0..c8038f9f 100644
--- a/src/local_array/bit_span.rs
+++ b/src/local_array/bit_span.rs
@@ -1,3 +1,5 @@
+use super::rib::default_store::STRIDE_BITS;
+
 #[derive(Copy, Clone, Debug)]
 pub struct BitSpan {
     pub bits: u32,
@@ -31,6 +33,14 @@ impl BitSpan {
             && self.bits < 16
             && (self.bits << (32 - self.len)) >> (32 - self.len) == self.bits
     }
+
+    pub(crate) fn into_bit_pos(self) -> u32 {
+        1 << (STRIDE_BITS - ((1 << self.len) - 1) as u8 - self.bits as u8 - 1)
+    }
+
+    pub(crate) fn cursor_from_bit_span(self) -> u8 {
+        self.into_bit_pos().leading_zeros() as u8
+    }
 }
 
 impl std::fmt::Binary for BitSpan {
diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 81092d88..0c39148c 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -1,4 +1,4 @@
-use log::trace;
+use log::{log_enabled, trace};
 use parking_lot_core::SpinWait;
 use std::fmt::{Binary, Debug};
 use std::sync::atomic::{fence, AtomicU16, AtomicU32, Ordering};
@@ -6,21 +6,12 @@ use std::sync::atomic::{fence, AtomicU16, AtomicU32, Ordering};
 use crate::af::Zero;
 use crate::local_array;
 use crate::local_array::bit_span::BitSpan;
-use crate::local_array::rib::default_store::STRIDE_BITS;
 
-pub type Stride4 = AtomicStride4;
-
-pub struct AtomicStride3(pub AtomicU16);
-pub struct AtomicStride4(pub AtomicU32);
+pub struct AtomicPtrBitArr(pub AtomicU16);
+pub struct AtomicPfxBitArr(pub AtomicU32);
 
 pub struct CasResult<InnerType>(pub Result<InnerType, InnerType>);
 
-impl<InnerType> CasResult<InnerType> {
-    fn new(value: InnerType) -> Self {
-        CasResult(Ok(value))
-    }
-}
-
 pub(crate) trait AtomicBitmap
 where
     Self: From<Self::InnerType>,
@@ -63,52 +54,35 @@ where
     }
 }
 
-// impl AtomicBitmap for AtomicStride2 {
-//     type InnerType = u8;
-
-//     fn new() -> Self {
-//         AtomicStride2(AtomicU8::new(0))
-//     }
-
-//     fn compare_exchange(
-//         &self,
-//         current: Self::InnerType,
-//         new: Self::InnerType,
-//     ) -> CasResult<Self::InnerType> {
-//         CasResult(self.0.compare_exchange(
-//             current,
-//             new,
-//             Ordering::Acquire,
-//             Ordering::Relaxed,
-//         ))
-//     }
-
-//     fn load(&self) -> Self::InnerType {
-//         self.0.load(Ordering::SeqCst)
-//     }
-// }
-
-// impl Zero for AtomicStride2 {
-//     fn zero() -> Self {
-//         AtomicStride2(AtomicU8::new(0))
-//     }
+impl AtomicPtrBitArr {
+    pub(crate) fn ptr_range(&self, bs: BitSpan) -> (u16, u8) {
+        let ptrbitarr = self.load();
+        let start: u8 = (bs.bits << (4 - bs.len)) as u8;
+        let stop: u8 = start + (1 << (4 - bs.len));
+        let mask: u16 = (((1_u32 << (stop as u32 - start as u32)) - 1)
+            .rotate_right(stop as u32)
+            >> 16)
+            .try_into()
+            .unwrap();
+        if log_enabled!(log::Level::Trace) {
+            trace!("- mask      {:032b}", mask);
+            trace!("- ptrbitarr {:032b}", ptrbitarr);
+            trace!("- shl bitar {:032b}", ptrbitarr & mask);
+        }
 
-//     fn is_zero(&self) -> bool {
-//         self.0.load(Ordering::SeqCst) == 0
-//     }
-// }
+        (ptrbitarr & mask, start)
+    }
 
-// impl From<u8> for AtomicStride2 {
-//     fn from(value: u8) -> Self {
-//         Self(AtomicU8::new(value))
-//     }
-// }
+    pub(crate) fn as_stride_size(&self) -> u32 {
+        (self.load() as u32) << 1
+    }
+}
 
-impl AtomicBitmap for AtomicStride3 {
+impl AtomicBitmap for AtomicPtrBitArr {
     type InnerType = u16;
 
     fn new() -> Self {
-        AtomicStride3(AtomicU16::new(0))
+        AtomicPtrBitArr(AtomicU16::new(0))
     }
     fn compare_exchange(
         &self,
@@ -128,27 +102,42 @@ impl AtomicBitmap for AtomicStride3 {
     }
 }
 
-impl From<u16> for AtomicStride3 {
+impl From<u16> for AtomicPtrBitArr {
     fn from(value: u16) -> Self {
         Self(AtomicU16::new(value))
     }
 }
 
-impl Zero for AtomicStride3 {
-    fn zero() -> Self {
-        AtomicStride3(AtomicU16::new(0))
+impl AtomicPfxBitArr {
+    pub(crate) fn ms_pfx_mask(&self, bs: BitSpan) -> u32 {
+        let pfxbitarr = self.load();
+        local_array::in_memory::node::ms_prefix_mask_arr(bs) & pfxbitarr
     }
+}
 
-    fn is_zero(&self) -> bool {
-        self.0.load(Ordering::SeqCst) == 0
-    }
+pub(crate) fn into_ptrbitarr(bitmap: u32) -> u16 {
+    (bitmap >> 1) as u16
+}
+
+pub(crate) fn into_pfxbitarr(bitmap: u16) -> u32 {
+    (bitmap as u32) << 1
+}
+
+pub(crate) fn bit_pos_from_index(i: u8) -> u32 {
+    <u32>::try_from(1).unwrap().rotate_right(1) >> i
+}
+
+pub(crate) fn ptr_bit_pos_from_index(i: u8) -> u16 {
+    // trace!("pfx {} ptr {} strlen {}",
+    // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
+    <u16>::try_from(1).unwrap().rotate_right(1) >> (i + 1)
 }
 
-impl AtomicBitmap for AtomicStride4 {
+impl AtomicBitmap for AtomicPfxBitArr {
     type InnerType = u32;
 
     fn new() -> Self {
-        AtomicStride4(AtomicU32::new(0))
+        AtomicPfxBitArr(AtomicU32::new(0))
     }
     fn compare_exchange(
         &self,
@@ -167,337 +156,143 @@ impl AtomicBitmap for AtomicStride4 {
     }
 }
 
-impl From<u32> for AtomicStride4 {
+impl From<u32> for AtomicPfxBitArr {
     fn from(value: u32) -> Self {
         Self(AtomicU32::new(value))
     }
 }
-impl Zero for AtomicStride4 {
-    fn zero() -> Self {
-        AtomicStride4(AtomicU32::new(0))
-    }
 
-    fn is_zero(&self) -> bool {
-        self.0.load(Ordering::SeqCst) == 0
-    }
-}
-
-pub(crate) trait Stride {
-    // const STRIDE_LEN: u8;
-
-    // Get the bit position of the start of the given nibble.
-    // The nibble is defined as a `len` number of bits set from the right.
-    // bit_pos always has only one bit set in the complete array.
-    // e.g.:
-    // len: 4
-    // nibble: u16  =  0b0000 0000 0000 0111
-    // bit_pos: u16 =  0b0000 0000 0000 1000
-
-    // `<Self as Stride>::BITS`
-    // is the whole length of the bitmap, since we are shifting to the left,
-    // we have to start at the end of the bitmap.
-    // `((1 << len) - 1)`
-    // is the offset for this nibble length in the bitmap.
-    // `nibble`
-    // shifts to the right position withing the bit range for this nibble
-    // length, this follows from the fact that the `nibble` value represents
-    // *both* the bitmap part, we're considering here *and* the position
-    // relative to the nibble length offset in the bitmap.
-    fn get_bit_pos(bit_pos: BitSpan) -> u32;
-    // ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
-
-    fn bit_pos_from_index(i: u8) -> u32;
-
-    fn ptr_bit_pos_from_index(i: u8) -> u16;
-
-    fn cursor_from_bit_span(bs: BitSpan) -> u8;
-
-    fn ptr_range(ptrbitarr: u16, range: BitSpan) -> (u16, u8);
-
-    fn ms_pfx_mask(
-        // pfxbitarr: <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
-        pfxbitarr: u32,
-        range: BitSpan,
-    ) -> u32;
-
-    // Clear the bitmap to the right of the pointer and count the number of
-    // ones. This number represents the index to the corresponding prefix in
-    // the pfx_vec.
-
-    // Clearing is performed by shifting to the right until we have the
-    // nibble all the way at the right.
-
-    // `(<Self as Stride>::BITS >> 1)`
-    // The end of the bitmap (this bitmap is half the size of the pfx bitmap)
-
-    // `nibble`
-    // The bit position relative to the offset for the nibble length, this
-    // index is only used at the last (relevant) stride, so the offset is
-    // always 0.
-
-    // get_pfx_index only needs nibble and len for fixed-layout bitarrays,
-    // since the index can be deducted from them.
-
-    // Clear the bitmap to the right of the pointer and count the number of
-    // ones. This number represents the index to the corresponding child node
-    // in the ptr_vec.
-
-    // Clearing is performed by shifting to the right until we have the
-    // nibble all the way at the right.
-
-    // For ptrbitarr the only index we want is the one for a full-length
-    // nibble (stride length) at the last stride, so we don't need the length
-    //  of the nibble.
-
-    // `(<Self as Stride>::BITS >> 1)`
-    // The end of the bitmap (this bitmap is half the size of the pfx bitmap)
-    // AF::BITS is the size of the pfx bitmap.
-
-    // `nibble`
-    // The bit position relative to the offset for the nibble length, this
-    // index is only used at the last (relevant) stride, so the offset is
-    // always 0.
-
-    // Convert a ptrbitarr into a pfxbitarr sized bitmap,
-    // so we can do bitwise operations with a pfxbitarr sized
-    // bitmap on them.
-    // Since the last bit in the pfxbitarr isn't used, but the
-    // full ptrbitarr *is* used, the prtbitarr should be shifted
-    // one bit to the left.
-    fn into_stride_size(
-        bitmap: u16, // bitmap: <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType,
-                     // ) -> <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType;
-    ) -> u32;
-
-    // Convert a pfxbitarr sized bitmap into a ptrbitarr sized
-    // Note that bitwise operators align bits of unsigned types with
-    // different sizes to the right, so we don't have to do anything to pad
-    // the smaller sized type. We do have to shift one bit to the left, to
-    // accommodate the unused pfxbitarr's last bit.
-    fn into_ptrbitarr_size(
-        // bitmap: <<Self as Stride>::AtomicPfxSize as AtomicBitmap>::InnerType,
-        bitmap: u32,
-    ) -> u16;
-    // ) -> <<Self as Stride>::AtomicPtrSize as AtomicBitmap>::InnerType;
-}
-
-// impl_primitive_atomic_stride![3; 16; u16; AtomicStride3; u8; AtomicStride2];
-// impl_primitive_atomic_stride![4; 32; u32; AtomicStride4; u16; AtomicStride3];
-// impl_primitive_atomic_stride![5; 64; u64; AtomicStride5; u32; AtomicStride4];
-// impl_primitive_stride![6; 128; u128; u64];
-
-// impl Stride for Stride7 {
-//     type PtrSize = u128;
-//     const BITS: u8 = 255;
-//     const STRIDE_LEN: u8 = 7;
-
-//     fn get_bit_pos(nibble: u32, len: u8) -> Self {
-//         match 256 - ((1 << len) - 1) as u16 - nibble as u16 - 1 {
-//             n if n < 128 => U256(0, 1 << n),
-//             n => U256(1 << (n as u16 - 128), 0),
-//         }
-//     }
-
-//     fn get_pfx_index(bitmap: Self, nibble: u32, len: u8) -> usize {
-//         let n = 256 - ((1 << len) - 1) as u16 - nibble as u16 - 1;
-//         match n {
-//             // if we move less than 128 bits to the right,
-//             // all of bitmap.0 and a part of bitmap.1 will be used for counting zeros
-//             // ex.
-//             // ...1011_1010... >> 2 => ...0010_111010...
-//             //    ____ ====                 -- --====
-//             n if n < 128 => {
-//                 bitmap.0.count_ones() as usize + (bitmap.1 >> n).count_ones() as usize - 1
-//             }
-//             // if we move more than 128 bits to the right,
-//             // all of bitmap.1 wil be shifted out of sight,
-//             // so we only have to count bitmap.0 zeroes than (after) shifting of course).
-//             n => (bitmap.0 >> (n - 128)).count_ones() as usize - 1,
-//         }
-//     }
-
-//     fn get_ptr_index(bitmap: Self::PtrSize, nibble: u32) -> usize {
-//         (bitmap >> ((256 >> 1) - nibble as u16 - 1) as usize).count_ones() as usize - 1
-//     }
-
-//     fn into_stride_size(bitmap: Self::PtrSize) -> Self {
-//         // One bit needs to move into the self.0 u128,
-//         // since the last bit of the *whole* bitmap isn't used.
-//         U256(bitmap >> 127, bitmap << 1)
-//     }
-
-//     fn into_ptrbitarr_size(bitmap: Self) -> Self::PtrSize {
-//         // TODO expand:
-//         // self.ptrbitarr =
-//         // S::into_ptrbitarr_size(bit_pos | S::into_stride_size(self.ptrbitarr));
-//         (bitmap.0 << 127 | bitmap.1 >> 1) as u128
-//     }
-
-//     #[inline]
-//     fn leading_zeros(self) -> u32 {
-//         let lz = self.0.leading_zeros();
-//         if lz == 128 {
-//             lz + self.1.leading_zeros()
-//         } else {
-//             lz
-//         }
-//     }
+// pub(crate) trait Stride {
+// Get the bit position of the start of the given nibble.
+// The nibble is defined as a `len` number of bits set from the right.
+// bit_pos always has only one bit set in the complete array.
+// e.g.:
+// len: 4
+// nibble: u16  =  0b0000 0000 0000 0111
+// bit_pos: u16 =  0b0000 0000 0000 1000
+
+// `<Self as Stride>::BITS`
+// is the whole length of the bitmap, since we are shifting to the left,
+// we have to start at the end of the bitmap.
+// `((1 << len) - 1)`
+// is the offset for this nibble length in the bitmap.
+// `nibble`
+// shifts to the right position withing the bit range for this nibble
+// length, this follows from the fact that the `nibble` value represents
+// *both* the bitmap part, we're considering here *and* the position
+// relative to the nibble length offset in the bitmap.
+// fn get_bit_pos(bit_span: BitSpan) -> u32;
+
+// fn bit_pos_from_index(i: u8) -> u32;
+
+// fn ptr_bit_pos_from_index(i: u8) -> u16;
+
+// fn cursor_from_bit_span(bs: BitSpan) -> u8;
+
+// fn ptr_range(ptrbitarr: u16, range: BitSpan) -> (u16, u8);
+
+// fn ms_pfx_mask(pfxbitarr: u32, range: BitSpan) -> u32;
+
+// Clear the bitmap to the right of the pointer and count the number of
+// ones. This number represents the index to the corresponding prefix in
+// the pfx_vec.
+
+// Clearing is performed by shifting to the right until we have the
+// nibble all the way at the right.
+
+// `(<Self as Stride>::BITS >> 1)`
+// The end of the bitmap (this bitmap is half the size of the pfx bitmap)
+
+// `nibble`
+// The bit position relative to the offset for the nibble length, this
+// index is only used at the last (relevant) stride, so the offset is
+// always 0.
+
+// get_pfx_index only needs nibble and len for fixed-layout bitarrays,
+// since the index can be deducted from them.
+
+// Clear the bitmap to the right of the pointer and count the number of
+// ones. This number represents the index to the corresponding child node
+// in the ptr_vec.
+
+// Clearing is performed by shifting to the right until we have the
+// nibble all the way at the right.
+
+// For ptrbitarr the only index we want is the one for a full-length
+// nibble (stride length) at the last stride, so we don't need the length
+//  of the nibble.
+
+// `(<Self as Stride>::BITS >> 1)`
+// The end of the bitmap (this bitmap is half the size of the pfx bitmap)
+// AF::BITS is the size of the pfx bitmap.
+
+// `nibble`
+// The bit position relative to the offset for the nibble length, this
+// index is only used at the last (relevant) stride, so the offset is
+// always 0.
+
+// Convert a ptrbitarr into a pfxbitarr sized bitmap,
+// so we can do bitwise operations with a pfxbitarr sized
+// bitmap on them.
+// Since the last bit in the pfxbitarr isn't used, but the
+// full ptrbitarr *is* used, the prtbitarr should be shifted
+// one bit to the left.
+// fn into_stride_size(bitmap: u16) -> u32;
+
+// Convert a pfxbitarr sized bitmap into a ptrbitarr sized
+// Note that bitwise operators align bits of unsigned types with
+// different sizes to the right, so we don't have to do anything to pad
+// the smaller sized type. We do have to shift one bit to the left, to
+// accommodate the unused pfxbitarr's last bit.
+// fn into_ptrbitarr_size(bitmap: u32) -> u16;
 // }
 
-// impl Stride for Stride8 {
-//     type PtrSize = U256;
-//     const BITS: u8 = 255; // bogus
-//     const STRIDE_LEN: u8 = 8;
-
-//     fn get_bit_pos(nibble: u32, len: u8) -> Self {
-//         match 512 - ((1 << len) - 1) as u16 - nibble as u16 - 1 {
-//             n if n < 128 => U512(0, 0, 0, 1 << n),
-//             n if n < 256 => U512(0, 0, 1 << (n as u16 - 128), 0),
-//             n if n < 384 => U512(0, 1 << (n as u16 - 256), 0, 0),
-//             n => U512(1 << (n as u16 - 384), 0, 0, 0),
-//         }
-//     }
-
-//     fn get_pfx_index(bitmap: Self, nibble: u32, len: u8) -> usize {
-//         let n = 512 - ((1 << len) - 1) as u16 - nibble as u16 - 1;
-//         match n {
-//             // if we move less than 128 bits to the right, all of bitmap.2
-//             // and a part of bitmap.3 will be used for counting zeros.
-//             // ex.
-//             // ...1011_1010... >> 2 => ...0010_111010...
-//             //    ____ ====                 -- --====
-//             n if n < 128 => {
-//                 bitmap.0.count_ones() as usize
-//                     + bitmap.1.count_ones() as usize
-//                     + bitmap.2.count_ones() as usize
-//                     + (bitmap.3 >> n).count_ones() as usize
-//                     - 1
-//             }
-
-//             n if n < 256 => {
-//                 bitmap.0.count_ones() as usize
-//                     + bitmap.1.count_ones() as usize
-//                     + (bitmap.2 >> (n - 128)).count_ones() as usize
-//                     - 1
-//             }
-
-//             n if n < 384 => {
-//                 bitmap.0.count_ones() as usize + (bitmap.1 >> (n - 256)).count_ones() as usize - 1
-//             }
-
-//             // if we move more than 384 bits to the right, all of bitmap.
-//             // [1,2,3] will be shifted out of sight, so we only have to count
-//             // bitmap.0 zeroes then (after shifting of course).
-//             n => (bitmap.0 >> (n - 384)).count_ones() as usize - 1,
-//         }
-//     }
-
-//     fn get_ptr_index(bitmap: Self::PtrSize, nibble: u32) -> usize {
-//         let n = (512 >> 1) - nibble as u16 - 1;
-//         match n {
-//             // if we move less than 256 bits to the right, all of bitmap.0
-//             // and a part of bitmap.1 will be used for counting zeros
-//             // ex.
-//             // ...1011_1010... >> 2 => ...0010_111010...
-//             //    ____ ====                 -- --====
-//             n if n < 128 => {
-//                 bitmap.0.count_ones() as usize + (bitmap.1 >> n).count_ones() as usize - 1
-//             }
-//             // if we move more than 256 bits to the right, all of bitmap.1
-//             // wil be shifted out of sight, so we only have to count bitmap.0
-//             // zeroes than (after) shifting of course).
-//             n => (bitmap.0 >> (n - 128)).count_ones() as usize - 1,
-//         }
-//     }
-
-//     fn into_stride_size(bitmap: Self::PtrSize) -> Self {
-//         // One bit needs to move into the self.0 u128,
-//         // since the last bit of the *whole* bitmap isn't used.
-//         U512(
-//             0,
-//             bitmap.0 >> 127,
-//             (bitmap.0 << 1) | (bitmap.1 >> 127),
-//             bitmap.1 << 1,
-//         )
-//     }
-
-//     fn into_ptrbitarr_size(bitmap: Self) -> Self::PtrSize {
-//         // TODO expand:
-//         // self.ptrbitarr =
-//         // S::into_ptrbitarr_size(bit_pos | S::into_stride_size(self.ptrbitarr));
-//         U256(
-//             (bitmap.1 << 127 | bitmap.2 >> 1) as u128,
-//             (bitmap.2 << 127 | bitmap.3 >> 1) as u128,
-//         )
-//     }
-
-//     #[inline]
-//     fn leading_zeros(self) -> u32 {
-//         let mut lz = self.0.leading_zeros();
-//         if lz == 128 {
-//             lz += self.1.leading_zeros();
-//             if lz == 256 {
-//                 lz += self.2.leading_zeros();
-//                 if lz == 384 {
-//                     lz += self.3.leading_zeros();
-//                 }
-//             }
-//         }
-//         lz
-//     }
+// impl Stride for AtomicPfxBitArr {
+// fn get_bit_pos(bs: BitSpan) -> u32 {
+//     // trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
+//     1 << (STRIDE_BITS - ((1 << bs.len) - 1) as u8 - bs.bits as u8 - 1)
 // }
-impl Stride for AtomicStride4 {
-    // type AtomicPfxSize = $atomicpfxsize;
-    // type AtomicPtrSize = $atomicptrsize;
-    // type PtrSize = $ptrsize;
-    // const BITS: u8 = $bits;
-    // const STRIDE_LEN: u8 = 4;
-
-    fn get_bit_pos(bs: BitSpan) -> u32 {
-        // trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
-        1 << (STRIDE_BITS - ((1 << bs.len) - 1) as u8 - bs.bits as u8 - 1)
-    }
-
-    fn bit_pos_from_index(i: u8) -> u32 {
-        <u32>::try_from(1).unwrap().rotate_right(1) >> i
-    }
-
-    fn ptr_bit_pos_from_index(i: u8) -> u16 {
-        // trace!("pfx {} ptr {} strlen {}",
-        // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
-        <u16>::try_from(1).unwrap().rotate_right(1) >> (i + 1)
-    }
 
-    fn cursor_from_bit_span(bs: BitSpan) -> u8 {
-        Self::get_bit_pos(bs).leading_zeros() as u8
-    }
+// fn bit_pos_from_index(i: u8) -> u32 {
+//     <u32>::try_from(1).unwrap().rotate_right(1) >> i
+// }
 
-    fn ptr_range(ptrbitarr: u16, bs: BitSpan) -> (u16, u8) {
-        let start: u8 = (bs.bits << (4 - bs.len)) as u8;
-        let stop: u8 = start + (1 << (4 - bs.len));
-        let mask: u16 = (((1_u32 << (stop as u32 - start as u32)) - 1)
-            .rotate_right(stop as u32)
-            >> 16)
-            .try_into()
-            .unwrap();
-        trace!("- mask      {:032b}", mask);
-        trace!("- ptrbitarr {:032b}", ptrbitarr);
-        trace!("- shl bitar {:032b}", ptrbitarr & mask);
+// fn ptr_bit_pos_from_index(i: u8) -> u16 {
+//     // trace!("pfx {} ptr {} strlen {}",
+//     // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
+//     <u16>::try_from(1).unwrap().rotate_right(1) >> (i + 1)
+// }
 
-        // if ptrbitarr & mask == <$ptrsize>::zero() { panic!("stop"); }
+// fn cursor_from_bit_span(bs: BitSpan) -> u8 {
+//     Self::get_bit_pos(bs).leading_zeros() as u8
+// }
 
-        (ptrbitarr & mask, start)
-    }
+// fn ptr_range(ptrbitarr: u16, bs: BitSpan) -> (u16, u8) {
+//     let start: u8 = (bs.bits << (4 - bs.len)) as u8;
+//     let stop: u8 = start + (1 << (4 - bs.len));
+//     let mask: u16 = (((1_u32 << (stop as u32 - start as u32)) - 1)
+//         .rotate_right(stop as u32)
+//         >> 16)
+//         .try_into()
+//         .unwrap();
+//     trace!("- mask      {:032b}", mask);
+//     trace!("- ptrbitarr {:032b}", ptrbitarr);
+//     trace!("- shl bitar {:032b}", ptrbitarr & mask);
+
+//     // if ptrbitarr & mask == <$ptrsize>::zero() { panic!("stop"); }
+
+//     (ptrbitarr & mask, start)
+// }
 
-    fn ms_pfx_mask(pfxbitarr: u32, bs: BitSpan) -> u32 {
-        local_array::in_memory::node::ms_prefix_mask_arr(bs) & pfxbitarr
-    }
+// fn ms_pfx_mask(pfxbitarr: u32, bs: BitSpan) -> u32 {
+//     local_array::in_memory::node::ms_prefix_mask_arr(bs) & pfxbitarr
+// }
 
-    fn into_stride_size(bitmap: u16) -> u32 {
-        (bitmap as u32) << 1
-    }
+// fn into_stride_size(bitmap: u16) -> u32 {
+//     (bitmap as u32) << 1
+// }
 
-    fn into_ptrbitarr_size(bitmap: u32) -> u16 {
-        (bitmap >> 1) as u16
-    }
-}
+// fn into_ptrbitarr_size(bitmap: u32) -> u16 {
+//     (bitmap >> 1) as u16
+// }
+// }
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index a59db3eb..065d263c 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -20,10 +20,8 @@ use crate::prelude::Meta;
 use crate::AddressFamily;
 
 use super::super::errors::PrefixStoreError;
-use super::atomic_stride;
 use super::node::{StrideNodeId, TreeBitMapNode};
 use super::oncebox::OnceBoxSlice;
-use super::tree::{Stride, Stride4};
 
 // ----------- Node related structs -----------------------------------------
 
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 351ab2ab..c58dc450 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -1,5 +1,3 @@
-use num_traits::PrimInt;
-use serde::de::value::U16Deserializer;
 use std::sync::atomic::{AtomicU16, AtomicU32};
 use std::{fmt::Debug, marker::PhantomData};
 
@@ -8,11 +6,14 @@ use parking_lot_core::SpinWait;
 
 use super::super::bit_span::BitSpan;
 use super::tree::{
-    AtomicBitmap, AtomicStride3, AtomicStride4, CasResult, Stride, Stride4,
+    into_pfxbitarr, into_ptrbitarr, AtomicBitmap, AtomicPfxBitArr,
+    AtomicPtrBitArr, CasResult,
 };
 
 use crate::af::AddressFamily;
-use crate::af::Zero;
+use crate::local_array::in_memory::tree::{
+    bit_pos_from_index, ptr_bit_pos_from_index,
+};
 use crate::local_array::rib::default_store::{STRIDE_BITS, STRIDE_SIZE};
 use crate::local_array::types::PrefixId;
 
@@ -36,8 +37,8 @@ where
     Self: Sized,
     AF: AddressFamily,
 {
-    pub ptrbitarr: AtomicStride3,
-    pub pfxbitarr: AtomicStride4,
+    pub ptrbitarr: AtomicPtrBitArr,
+    pub pfxbitarr: AtomicPfxBitArr,
     pub _af: PhantomData<AF>,
 }
 
@@ -73,8 +74,8 @@ where
 {
     pub(crate) fn new() -> Self {
         TreeBitMapNode {
-            ptrbitarr: AtomicStride3::new(),
-            pfxbitarr: AtomicStride4::new(),
+            ptrbitarr: AtomicPtrBitArr::new(),
+            pfxbitarr: AtomicPfxBitArr::new(),
             _af: PhantomData,
         }
     }
@@ -88,11 +89,8 @@ where
         start_bs: BitSpan,
     ) -> NodeMoreSpecificsPrefixIter<AF> {
         debug_assert!(start_bs.check());
-        let mut pfxbitarr = self.pfxbitarr.load();
-        pfxbitarr = AtomicStride4::ms_pfx_mask(pfxbitarr, start_bs);
-
         NodeMoreSpecificsPrefixIter::<AF> {
-            pfxbitarr,
+            pfxbitarr: self.pfxbitarr.ms_pfx_mask(start_bs),
             base_prefix,
         }
     }
@@ -105,9 +103,8 @@ where
         start_bs: BitSpan,
     ) -> NodeMoreSpecificChildIter<AF> {
         debug_assert!(start_bs.check());
-        let ptrbitarr = self.ptrbitarr.load();
-        let (bitrange, start_cursor) =
-            AtomicStride4::ptr_range(ptrbitarr, start_bs);
+        // let ptrbitarr = self.ptrbitarr.load();
+        let (bitrange, start_cursor) = self.ptrbitarr.ptr_range(start_bs);
 
         NodeMoreSpecificChildIter::<AF> {
             bitrange,
@@ -153,7 +150,7 @@ where
         let mut retry_count = 0;
         let ptrbitarr = self.ptrbitarr.load();
         let pfxbitarr = self.pfxbitarr.load();
-        let bit_pos = AtomicStride4::get_bit_pos(bit_span);
+        let bit_pos = bit_span.into_bit_pos();
         let new_node: TreeBitMapNode<AF>;
 
         // Check that we're not at the last stride (pfx.len <= stride_end),
@@ -167,13 +164,11 @@ where
         if !is_last_stride {
             // We are not at the last stride
             // Check it the ptr bit is already set in this position
-            if (AtomicStride4::into_stride_size(ptrbitarr) & bit_pos) == 0 {
-                // <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::
-                //     InnerType>::zero() {
+            if (self.ptrbitarr.as_stride_size() & bit_pos) == 0 {
                 // Nope, set it and create a child node
                 new_node = TreeBitMapNode {
-                    ptrbitarr: AtomicStride3(AtomicU16::new(0)),
-                    pfxbitarr: AtomicStride4(AtomicU32::new(0)),
+                    ptrbitarr: AtomicPtrBitArr(AtomicU16::new(0)),
+                    pfxbitarr: AtomicPfxBitArr(AtomicU32::new(0)),
                     _af: PhantomData,
                 };
 
@@ -185,9 +180,7 @@ where
                 // in the meantime elsewhere in the bitarray.
                 let mut a_ptrbitarr = self.ptrbitarr.compare_exchange(
                     ptrbitarr,
-                    AtomicStride4::into_ptrbitarr_size(
-                        bit_pos | AtomicStride4::into_stride_size(ptrbitarr),
-                    ),
+                    into_ptrbitarr(bit_pos | into_pfxbitarr(ptrbitarr)),
                 );
                 let mut spinwait = SpinWait::new();
                 loop {
@@ -201,11 +194,8 @@ where
                             retry_count += 1;
                             a_ptrbitarr = self.ptrbitarr.compare_exchange(
                                 newer_array,
-                                AtomicStride4::into_ptrbitarr_size(
-                                    bit_pos
-                                        | AtomicStride4::into_stride_size(
-                                            newer_array,
-                                        ),
+                                into_ptrbitarr(
+                                    bit_pos | into_pfxbitarr(newer_array),
                                 ),
                             );
                         }
@@ -219,10 +209,6 @@ where
             // only at the last stride do we create the bit in the prefix
             // bitmap, and only if it doesn't exist already
             if pfxbitarr & bit_pos == 0 {
-                // == <<<S as Stride>::AtomicPfxSize as AtomicBitmap>::
-                //     InnerType as std::ops::BitAnd>::Output::zero()
-                // {
-
                 // THE CRITICAL SECTION
                 //
                 // UPDATING pfxbitarr
@@ -300,7 +286,6 @@ where
 // child Nodes len     /29    /32
 
 type PtrBitArr = u16;
-type PfxBitArr = u32;
 
 // ----------- NodeMoreSpecificChildIter ------------------------------------
 
@@ -389,17 +374,14 @@ impl<AF: AddressFamily> std::iter::Iterator
 
         trace!("bitrange  {:032b}", self.bitrange);
 
-        self.bitrange ^= AtomicStride4::ptr_bit_pos_from_index(cursor);
+        self.bitrange ^= ptr_bit_pos_from_index(cursor);
 
-        trace!(
-            "mask      {:032b}",
-            AtomicStride4::ptr_bit_pos_from_index(cursor)
-        );
+        trace!("mask      {:032b}", ptr_bit_pos_from_index(cursor));
         trace!("next br   {:032b}", self.bitrange);
 
         let bs = BitSpan::from_bit_pos_index(cursor);
         if log_enabled!(log::Level::Trace) {
-            let bit_pos = AtomicStride4::ptr_bit_pos_from_index(cursor);
+            let bit_pos = ptr_bit_pos_from_index(cursor);
             trace!(
                 "{:02}: {:05b} {:032b} bit_span: {:04b} ({:02}) (len: {})",
                 cursor,
@@ -571,17 +553,17 @@ impl<AF: AddressFamily> std::iter::Iterator
         trace!(
             "ms prefix iterator start_bs {:?} start cursor {}",
             bs,
-            AtomicStride4::cursor_from_bit_span(bs)
+            bs.cursor_from_bit_span()
         );
         trace!("pfx {:032b}", self.pfxbitarr);
-        let bit_pos = AtomicStride4::get_bit_pos(bs);
+        let bit_pos = bs.into_bit_pos();
         let prefix_id: PrefixId<AF> = self
             .base_prefix
             .add_bit_span(BitSpan::from_bit_pos_index(
                 bit_pos.leading_zeros() as u8,
             ))
             .into();
-        self.pfxbitarr ^= AtomicStride4::bit_pos_from_index(cursor);
+        self.pfxbitarr ^= bit_pos_from_index(cursor);
         Some(prefix_id)
     }
 }
@@ -592,8 +574,8 @@ where
 {
     fn default() -> Self {
         Self {
-            ptrbitarr: AtomicStride3::new(),
-            pfxbitarr: AtomicStride4::new(),
+            ptrbitarr: AtomicPtrBitArr::new(),
+            pfxbitarr: AtomicPfxBitArr::new(),
             _af: PhantomData,
         }
     }
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 06293af5..f50aef3a 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -232,8 +232,8 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0),
             0_u32,
             TreeBitMapNode {
-                ptrbitarr: AtomicStride3(AtomicU16::new(0)),
-                pfxbitarr: AtomicStride4(AtomicU32::new(0)),
+                ptrbitarr: AtomicPtrBitArr(AtomicU16::new(0)),
+                pfxbitarr: AtomicPfxBitArr(AtomicU32::new(0)),
                 _af: PhantomData,
             },
         )?;
@@ -412,7 +412,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         match self.retrieve_node(node_id) {
             Some(n) => {
                 let pfxbitarr = n.pfxbitarr.load();
-                pfxbitarr & Stride4::get_bit_pos(bs) > 0
+                pfxbitarr & bs.into_bit_pos() > 0
             }
             None => false,
         }
@@ -429,7 +429,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         match self.retrieve_node_for_mui(node_id, mui) {
             Some(n) => {
                 let pfxbitarr = n.pfxbitarr.load();
-                pfxbitarr & Stride4::get_bit_pos(bs) > 0
+                pfxbitarr & bs.into_bit_pos() > 0
             }
             None => false,
         }
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
index 515d49e3..00ecd65a 100644
--- a/src/local_array/prefix_cht/iterators.rs
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -6,7 +6,7 @@ use crate::{
         bit_span::BitSpan,
         in_memory::{
             node::{NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter},
-            tree::{Stride4, TreeBitMap},
+            tree::TreeBitMap,
         },
     },
     prelude::multi::{NodeBuckets, PrefixId},

From 52699264b6365067a4ec5cc7e02964929b181acc Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 28 Feb 2025 14:03:13 +0100
Subject: [PATCH 092/147] remove singlethreaded store

---
 src/af.rs                                    |  57 --
 src/lib.rs                                   |   4 -
 src/local_array/in_memory/atomic_stride.rs   |   2 -
 src/local_array/in_memory/atomic_types.rs    |   4 +-
 src/local_array/in_memory/mod.rs             |   4 +-
 src/local_array/persist/lsm_tree.rs          |   2 +-
 src/local_array/prefix_cht/iterators.rs      |   9 +-
 src/local_vec/macros.rs                      | 158 ----
 src/local_vec/mod.rs                         |  12 -
 src/local_vec/node.rs                        | 490 ------------
 src/local_vec/query.rs                       | 793 -------------------
 src/local_vec/storage_backend.rs             | 260 ------
 src/local_vec/store.rs                       | 148 ----
 src/local_vec/tests/full_table_single.rs     | 164 ----
 src/local_vec/tests/mod.rs                   |   2 -
 src/local_vec/tests/more_specifics_single.rs | 163 ----
 src/local_vec/tree.rs                        | 568 -------------
 src/macros.rs                                | 121 +--
 src/node_id.rs                               |  62 --
 src/prelude/mod.rs                           |   1 -
 src/rotonda_store.rs                         |   6 +-
 src/stats.rs                                 | 105 +--
 src/stride.rs                                | 263 ------
 src/synth_int.rs                             | 444 -----------
 24 files changed, 43 insertions(+), 3799 deletions(-)
 delete mode 100644 src/local_vec/macros.rs
 delete mode 100644 src/local_vec/mod.rs
 delete mode 100644 src/local_vec/node.rs
 delete mode 100644 src/local_vec/query.rs
 delete mode 100644 src/local_vec/storage_backend.rs
 delete mode 100644 src/local_vec/store.rs
 delete mode 100644 src/local_vec/tests/full_table_single.rs
 delete mode 100644 src/local_vec/tests/mod.rs
 delete mode 100644 src/local_vec/tests/more_specifics_single.rs
 delete mode 100644 src/local_vec/tree.rs
 delete mode 100644 src/node_id.rs
 delete mode 100644 src/stride.rs
 delete mode 100644 src/synth_int.rs

diff --git a/src/af.rs b/src/af.rs
index a6ffb7ab..3cefab94 100644
--- a/src/af.rs
+++ b/src/af.rs
@@ -389,60 +389,3 @@ impl IntoIpAddr for u128 {
         std::net::IpAddr::V6(std::net::Ipv6Addr::from(self))
     }
 }
-
-// ----------- Zero Trait ---------------------------------------------------
-
-pub trait Zero {
-    fn zero() -> Self;
-    fn is_zero(&self) -> bool;
-}
-
-impl Zero for u128 {
-    fn zero() -> Self {
-        0
-    }
-
-    fn is_zero(&self) -> bool {
-        *self == 0
-    }
-}
-
-impl Zero for u64 {
-    fn zero() -> Self {
-        0
-    }
-
-    fn is_zero(&self) -> bool {
-        *self == 0
-    }
-}
-
-impl Zero for u32 {
-    fn zero() -> Self {
-        0
-    }
-
-    fn is_zero(&self) -> bool {
-        *self == 0
-    }
-}
-
-impl Zero for u16 {
-    fn zero() -> Self {
-        0
-    }
-
-    fn is_zero(&self) -> bool {
-        *self == 0
-    }
-}
-
-impl Zero for u8 {
-    fn zero() -> Self {
-        0
-    }
-
-    fn is_zero(&self) -> bool {
-        *self == 0
-    }
-}
diff --git a/src/lib.rs b/src/lib.rs
index 740eadb0..06783840 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -12,11 +12,7 @@
 //! Read more about the data-structure in this [blog post](https://blog.nlnetlabs.nl/donkeys-mules-horses/).
 mod af;
 mod local_array;
-// mod local_vec;
-mod node_id;
 mod prefix_record;
-mod stride;
-mod synth_int;
 
 pub mod test_types;
 
diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 0c39148c..ca296f53 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -3,7 +3,6 @@ use parking_lot_core::SpinWait;
 use std::fmt::{Binary, Debug};
 use std::sync::atomic::{fence, AtomicU16, AtomicU32, Ordering};
 
-use crate::af::Zero;
 use crate::local_array;
 use crate::local_array::bit_span::BitSpan;
 
@@ -19,7 +18,6 @@ where
     type InnerType: Binary
         + Copy
         + Debug
-        + Zero
         + PartialOrd
         + std::ops::BitAnd<Output = Self::InnerType>
         + std::ops::BitOr<Output = Self::InnerType>
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 065d263c..5059d088 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -68,7 +68,6 @@ impl<AF: AddressFamily> NodeSet<AF> {
         multi_uniq_id: u32,
     ) -> Result<(u32, bool), crate::prelude::multi::PrefixStoreError>
     where
-        // S: atomic_stride::Stride,
         AF: crate::AddressFamily,
     {
         let try_count = 0;
@@ -78,13 +77,12 @@ impl<AF: AddressFamily> NodeSet<AF> {
         Ok((try_count, !absent))
     }
 
-    pub(crate) fn remove_from_rbm_index(
+    pub(crate) fn _remove_from_rbm_index(
         &self,
         multi_uniq_id: u32,
         _guard: &crate::epoch::Guard,
     ) -> Result<u32, crate::prelude::multi::PrefixStoreError>
     where
-        // S: atomic_stride::Stride,
         AF: crate::AddressFamily,
     {
         let try_count = 0;
diff --git a/src/local_array/in_memory/mod.rs b/src/local_array/in_memory/mod.rs
index ad345222..42284fb8 100644
--- a/src/local_array/in_memory/mod.rs
+++ b/src/local_array/in_memory/mod.rs
@@ -11,5 +11,5 @@ pub mod iterators;
 
 mod query;
 
-#[macro_use]
-mod macros;
+// #[macro_use]
+// mod macros;
diff --git a/src/local_array/persist/lsm_tree.rs b/src/local_array/persist/lsm_tree.rs
index 266ada9c..b0c1d0bf 100644
--- a/src/local_array/persist/lsm_tree.rs
+++ b/src/local_array/persist/lsm_tree.rs
@@ -177,7 +177,7 @@ impl<
         self.tree.insert::<&[u8], &[u8]>(key, value, 0)
     }
 
-    pub fn remove(&self, key: &[u8]) {
+    pub fn _remove(&self, key: &[u8]) {
         self.tree.remove_weak(key, 0);
         // the first byte of the prefix holds the length of the prefix.
         self.counters.dec_prefixes_count(key[0]);
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
index 00ecd65a..a11ed54a 100644
--- a/src/local_array/prefix_cht/iterators.rs
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -31,13 +31,8 @@ pub(crate) struct MoreSpecificPrefixIter<
     include_withdrawn: bool,
 }
 
-impl<
-        'a,
-        AF: AddressFamily + 'a,
-        // M: Meta,
-        NB: NodeBuckets<AF>,
-        // PB: PrefixBuckets<AF, M>,
-    > Iterator for MoreSpecificPrefixIter<'a, AF, NB>
+impl<'a, AF: AddressFamily + 'a, NB: NodeBuckets<AF>> Iterator
+    for MoreSpecificPrefixIter<'a, AF, NB>
 {
     type Item = PrefixId<AF>;
 
diff --git a/src/local_vec/macros.rs b/src/local_vec/macros.rs
deleted file mode 100644
index 3b25ed64..00000000
--- a/src/local_vec/macros.rs
+++ /dev/null
@@ -1,158 +0,0 @@
-#[macro_export]
-#[doc(hidden)]
-macro_rules! match_node_for_strides_with_local_vec {
-    // This macro expands into a match node {}
-    // with match arms for all SizedStrideNode::Stride[3-8]
-    // for use in insert()
-
-    (
-        $self: ident;
-        // $user_data: ident;
-        $nibble_len: expr;
-        $nibble: expr;
-        $is_last_stride: expr;
-        $pfx: ident;
-        $cur_i: expr;
-        $level: expr;
-        // $enum: ident;
-        // The strides to generate match arms for,
-        // $variant is the name of the enum varian (Stride[3..8]) and
-        // $len is the index of the stats level, so 0..5
-        $( $variant: ident; $stats_level: expr ), *
-    ) => {
-        match std::mem::take($self.retrieve_node_mut($cur_i)?) {
-            $( SizedStrideNode::$variant(mut current_node) =>
-            match current_node.eval_node_or_prefix_at(
-                $nibble,
-                $nibble_len,
-                $self.strides.get(($level + 1) as usize),
-                $is_last_stride,
-            ) {
-                NewNodeOrIndex::NewNode(n, bit_id) => {
-                    $self.stats[$stats_level].inc($level); // Stride3 logs to stats[0], Stride4 logs to stats[1], etc.
-                    // let new_id = Store::NodeType::new(&bit_id,&$cur_i.get_part());
-                    let new_id = $self.store.acquire_new_node_id(bit_id, $cur_i.get_part());
-                    current_node.ptr_vec.push(new_id);
-                    current_node.ptr_vec.sort();
-                    let i = $self.store_node(Some(new_id), n).unwrap();
-
-                    $self.store.update_node($cur_i,SizedStrideNode::$variant(current_node));
-
-                    // let _default_val = std::mem::replace(
-                    //     $self.retrieve_node_mut($cur_i).unwrap(),
-                    //     SizedStrideNode::$variant(current_node));
-                    Some(i)
-                }
-                NewNodeOrIndex::ExistingNode(i) => {
-                    $self.store.update_node($cur_i,SizedStrideNode::$variant(current_node));
-
-                    // let _default_val = std::mem::replace(
-                    //     $self.retrieve_node_mut($cur_i).unwrap(),
-                    //     SizedStrideNode::$variant(current_node));
-                    Some(i)
-                },
-                NewNodeOrIndex::NewPrefix => {
-
-                    // let pfx_len = $pfx.len.clone();
-                    // let pfx_net = $pfx.net.clone();
-                    // let i = $self.store_prefix($pfx)?;
-                    // Construct the SortKey by default from the nibble and
-                    // nibble_len, so that nibble_len determines the base
-                    // position (2^nibble_len) and then nibble is the offset
-                    // from the base position.
-                    let new_id = $self.store.acquire_new_prefix_id(&((1 << $nibble_len) + $nibble as u16).into(), &$pfx);
-                    $self.stats[$stats_level].inc_prefix_count($level);
-
-                    current_node
-                        .pfx_vec
-                        .push(new_id);
-                        current_node.pfx_vec.sort();
-
-                    $self.store_prefix($pfx)?;
-                    $self.store.update_node($cur_i,SizedStrideNode::$variant(current_node));
-
-                    // let _default_val = std::mem::replace(
-                    //     $self.retrieve_node_mut($cur_i).unwrap(),
-                    //     SizedStrideNode::$variant(current_node),
-                    // );
-                    return Ok(());
-                }
-                NewNodeOrIndex::ExistingPrefix(pfx_idx) => {
-                    // ExistingPrefix is guaranteed to only happen at the last stride,
-                    // so we can return from here.
-                    // If we don't then we cannot move pfx.meta into the update_prefix_meta function,
-                    // since the compiler can't figure out that it will happen only once.
-                    $self.update_prefix_meta(pfx_idx, $pfx.meta)?;
-                    $self.store.update_node($cur_i,SizedStrideNode::$variant(current_node));
-
-                    // let _default_val = std::mem::replace(
-                    //     $self.retrieve_node_mut($cur_i).unwrap(),
-                    //     // expands into SizedStrideNode::Stride[3-8](current_node)
-                    //     SizedStrideNode::$variant(current_node),
-                    // );
-                    return Ok(());
-                }
-            } )*,
-        }
-    };
-}
-
-//     example expansion for Stride4:
-
-//     SizedStrideNode::Stride4(mut current_node) => match current_node
-//         .eval_node_or_prefix_at(
-//             nibble,
-//             nibble_len,
-//             // No, next_stride.is_none does *not* mean that it's the last stride
-//             // There may very well be a Some(next_stride), next_stride goes all the
-//             // way to the end of the length of the network address space (like 32 bits for IPv4 etc),
-//             // whereas the last stride stops at the end of the prefix length.
-//             // `is_last_stride` is an indicator for the upsert function to write the prefix in the
-//             // node's vec.
-//             next_stride,
-//             pfx_len <= stride_end,
-//         ) {
-//         NewNodeOrIndex::NewNode(n, bit_id) => {
-//             self.stats[1].inc(level); // [1] here corresponds to stats for Stride4
-//             let i = self.store_node(n);
-//             current_node.ptr_vec.push(NodeId::new(bit_id, i));
-//             current_node.ptr_vec.sort();
-//              let _default_val = std::mem::replace(
-//                 self.retrieve_node_mut(cur_i).unwrap(),
-//                 SizedStrideNode::Stride4(current_node),
-//             );
-//             Some(i)
-//         }
-//         NewNodeOrIndex::ExistingNode(i) => {
-//              let _default_val = std::mem::replace(
-//                 self.retrieve_node_mut(cur_i).unwrap(),
-//                 SizedStrideNode::Stride4(current_node),
-//             );
-//             Some(i)
-//         }
-//         NewNodeOrIndex::NewPrefix => {
-//             let i = self.store_prefix(pfx);
-//             self.stats[1].inc_prefix_count(level);
-//             current_node
-//                 .pfx_vec
-//                 .push(((pfx_net >> (AF::BITS - pfx_len) as usize), i));
-//             current_node.pfx_vec.sort();
-//             let _default_val = std::mem::replace(
-//                 self.retrieve_node_mut(cur_i).unwrap(),
-//                 SizedStrideNode::Stride4(current_node),
-//             );
-//             return Ok(());
-//         }
-//         NewNodeOrIndex::ExistingPrefix(pfx_idx) => {
-//             // ExitingPrefix is guaranteed to only happen at the last stride,
-//             // so we can return from here.
-//             // If we don't then we cannot move pfx.meta into the update_prefix_meta function,
-//             // since the compiler can't figure out that it will happen only once.
-//             self.update_prefix_meta(pfx_idx, pfx.meta)?;
-//             let _default_val = std::mem::replace(
-//                 self.retrieve_node_mut(cur_i).unwrap(),
-//                 SizedStrideNode::Stride4(current_node),
-//             );
-//             return Ok(());
-//         }
-//     },
diff --git a/src/local_vec/mod.rs b/src/local_vec/mod.rs
deleted file mode 100644
index a319a7aa..00000000
--- a/src/local_vec/mod.rs
+++ /dev/null
@@ -1,12 +0,0 @@
-pub mod tree;
-pub mod query;
-pub mod node;
-pub mod storage_backend;
-pub mod store;
-
-pub(crate) use tree::TreeBitMap;
-
-#[macro_use]
-mod macros;
-
-mod tests;
\ No newline at end of file
diff --git a/src/local_vec/node.rs b/src/local_vec/node.rs
deleted file mode 100644
index 96a2323f..00000000
--- a/src/local_vec/node.rs
+++ /dev/null
@@ -1,490 +0,0 @@
-use std::{
-    fmt::{Binary, Debug},
-    marker::PhantomData,
-};
-
-use crate::af::Zero;
-use crate::node_id::SortableNodeId;
-pub use crate::stride::*;
-use crate::synth_int::{U256, U512};
-
-use crate::local_vec::tree::{NewNodeOrIndex, SizedStrideNode};
-
-use crate::af::AddressFamily;
-
-use super::query::PrefixId;
-
-pub struct TreeBitMapNode<AF: AddressFamily, S, NodeId>
-where
-    S: Stride,
-    <S as Stride>::PtrSize: Debug + Binary + Copy,
-    AF: AddressFamily,
-    NodeId: SortableNodeId + Copy,
-{
-    pub ptrbitarr: <S as Stride>::PtrSize,
-    pub pfxbitarr: S,
-    // The vec of prefixes hosted by this node,
-    // referenced by (bit_id, global prefix index)
-    // This is the exact same type as for the NodeIds,
-    // so we reuse that.
-    pub pfx_vec: Vec<NodeId>,
-    // The vec of child nodes hosted by this
-    // node, referenced by (ptrbitarr_index, global vec index)
-    // We need the u16 (ptrbitarr_index) to sort the
-    // vec that's stored in the node.
-    pub ptr_vec: Vec<NodeId>,
-    pub _af: PhantomData<AF>,
-}
-
-impl<AF, S, NodeId> Debug for TreeBitMapNode<AF, S, NodeId>
-where
-    AF: AddressFamily,
-    S: Stride,
-    <S as Stride>::PtrSize: Debug + Binary + Copy,
-    NodeId: SortableNodeId + Copy,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("TreeBitMapNode")
-            .field("ptrbitarr", &self.ptrbitarr)
-            .field("pfxbitarr", &self.pfxbitarr)
-            .field("ptr_vec", &self.ptr_vec)
-            .field("pfx_vec", &self.pfx_vec)
-            .finish()
-    }
-}
-
-impl<AF, S, NodeId> TreeBitMapNode<AF, S, NodeId>
-where
-    AF: AddressFamily,
-    S: Stride
-        + std::ops::BitAnd<Output = S>
-        + std::ops::BitOr<Output = S>
-        + Zero,
-    <S as Stride>::PtrSize: Debug
-        + Binary
-        + Copy
-        + std::ops::BitAnd<Output = S::PtrSize>
-        + PartialOrd
-        + Zero,
-    NodeId: SortableNodeId + Copy,
-{
-    // Inspects the stride (nibble, nibble_len) to see it there's
-    // already a child node (if not at the last stride) or a prefix
-    //  (if it's the last stride).
-    //
-    // Returns one of:
-    // - A newly created child node.
-    // - The index of the existing child node in the global `nodes` vec
-    // - A newly created Prefix
-    // - The index of the existing prefix in the global `prefixes` vec
-    pub(crate) fn eval_node_or_prefix_at(
-        &mut self,
-        nibble: u32,
-        nibble_len: u8,
-        next_stride: Option<&u8>,
-        is_last_stride: bool,
-    ) -> NewNodeOrIndex<AF, NodeId> {
-        let bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let new_node: SizedStrideNode<AF, NodeId>;
-
-        // Check that we're not at the last stride (pfx.len <= stride_end),
-        // Note that next_stride may have a value, but we still don't want to
-        // continue, because we've exceeded the length of the prefix to
-        // be inserted.
-        // Also note that a nibble_len < S::BITS (a smaller than full nibble)
-        // does indeed indicate the last stride has been reached, but the
-        // reverse is *not* true, i.e. a full nibble can also be the last
-        // stride. Hence the `is_last_stride` argument
-        if !is_last_stride {
-            // We are not at the last stride
-            // Check it the ptr bit is already set in this position
-            if (S::into_stride_size(self.ptrbitarr) & bit_pos) == S::zero() {
-                // Nope, set it and create a child node
-                self.ptrbitarr = S::into_ptrbitarr_size(
-                    bit_pos | S::into_stride_size(self.ptrbitarr),
-                );
-
-                match next_stride.unwrap() {
-                    3_u8 => {
-                        new_node = SizedStrideNode::Stride3(TreeBitMapNode {
-                            ptrbitarr: <Stride3 as Stride>::PtrSize::zero(),
-                            pfxbitarr: Stride3::zero(),
-                            pfx_vec: vec![],
-                            ptr_vec: vec![],
-                            _af: PhantomData,
-                        });
-                    }
-                    4_u8 => {
-                        new_node = SizedStrideNode::Stride4(TreeBitMapNode {
-                            ptrbitarr: <Stride4 as Stride>::PtrSize::zero(),
-                            pfxbitarr: Stride4::zero(),
-                            pfx_vec: vec![],
-                            ptr_vec: vec![],
-                            _af: PhantomData,
-                        });
-                    }
-                    5_u8 => {
-                        new_node = SizedStrideNode::Stride5(TreeBitMapNode {
-                            ptrbitarr: <Stride5 as Stride>::PtrSize::zero(),
-                            pfxbitarr: Stride5::zero(),
-                            pfx_vec: vec![],
-                            ptr_vec: vec![],
-                            _af: PhantomData,
-                        });
-                    }
-                    6_u8 => {
-                        new_node = SizedStrideNode::Stride6(TreeBitMapNode {
-                            ptrbitarr: <Stride6 as Stride>::PtrSize::zero(),
-                            pfxbitarr: Stride6::zero(),
-                            pfx_vec: vec![],
-                            ptr_vec: vec![],
-                            _af: PhantomData,
-                        });
-                    }
-                    7_u8 => {
-                        new_node = SizedStrideNode::Stride7(TreeBitMapNode {
-                            ptrbitarr: 0_u128,
-                            pfxbitarr: U256(0_u128, 0_u128),
-                            pfx_vec: vec![],
-                            ptr_vec: vec![],
-                            _af: PhantomData,
-                        });
-                    }
-                    8_u8 => {
-                        new_node = SizedStrideNode::Stride8(TreeBitMapNode {
-                            ptrbitarr: U256(0_u128, 0_u128),
-                            pfxbitarr: U512(0_u128, 0_u128, 0_u128, 0_u128),
-                            pfx_vec: vec![],
-                            ptr_vec: vec![],
-                            _af: PhantomData,
-                        });
-                    }
-                    _ => {
-                        panic!("can't happen");
-                    }
-                };
-
-                // we can return bit_pos.leading_zeros() since bit_pos is the bitmap that
-                // points to the current bit in ptrbitarr (it's **not** the prefix of the node!),
-                // so the number of zeros in front of it should always be unique and describes
-                // the index of this node in the ptrbitarr.
-                // ex.:
-                // In a stride3 (ptrbitarr lenght is 8):
-                // bit_pos 0001 0000
-                // so this is the fourth bit, so points to index = 3
-                return NewNodeOrIndex::NewNode(
-                    new_node,
-                    (bit_pos.leading_zeros() as u16).into(),
-                );
-            }
-        } else {
-            // only at the last stride do we create the bit in the prefix bitmap,
-            // and only if it doesn't exist already
-            if self.pfxbitarr & bit_pos
-                == <S as std::ops::BitAnd>::Output::zero()
-            {
-                self.pfxbitarr = bit_pos | self.pfxbitarr;
-                return NewNodeOrIndex::NewPrefix;
-            }
-            return NewNodeOrIndex::ExistingPrefix(
-                self.pfx_vec
-                    [S::get_pfx_index(self.pfxbitarr, nibble, nibble_len)]
-                .get_part(),
-            );
-        }
-
-        NewNodeOrIndex::ExistingNode(
-            self.ptr_vec[S::get_ptr_index(self.ptrbitarr, nibble)],
-        )
-    }
-
-    //-------- Search nibble functions --------------------------------------
-
-    // This function looks for the longest marching prefix in the provided nibble,
-    // by iterating over all the bits in it and comparing that with the appriopriate
-    // bytes from the requested prefix.
-    // It mutates the `less_specifics_vec` that was passed in to hold all the prefixes
-    // found along the way.
-    pub(crate) fn search_stride_for_longest_match_at(
-        &self,
-        search_pfx: PrefixId<AF>,
-        mut nibble: u32,
-        nibble_len: u8,
-        start_bit: u8,
-        less_specifics_vec: &mut Option<Vec<NodeId>>,
-    ) -> (Option<NodeId>, Option<NodeId>) {
-        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_pfx = None;
-
-        for n_l in 1..(nibble_len + 1) {
-            // Move the bit in the right position.
-            nibble = AddressFamily::get_nibble(
-                search_pfx.get_net(),
-                start_bit,
-                n_l,
-            );
-            bit_pos = S::get_bit_pos(nibble, n_l);
-
-            // Check if the prefix has been set, if so select the prefix. This is not
-            // necessarily the final prefix that will be returned.
-
-            // Check it there's a prefix matching in this bitmap for this nibble
-            if self.pfxbitarr & bit_pos > S::zero() {
-                let f_pfx = self.pfx_vec
-                    [S::get_pfx_index(self.pfxbitarr, nibble, n_l)];
-
-                // Receiving a less_specifics_vec means that the user wants to have
-                // all the last-specific prefixes returned, so add the found prefix.
-                if let Some(ls_vec) = less_specifics_vec {
-                    if !(search_pfx.get_len() <= start_bit + nibble_len
-                        || (S::into_stride_size(self.ptrbitarr)
-                            & S::get_bit_pos(nibble, nibble_len))
-                            == S::zero())
-                    {
-                        ls_vec.push(f_pfx);
-                    }
-                }
-
-                found_pfx = Some(f_pfx);
-            }
-        }
-
-        // Check if this the last stride, or if they're no more children to go to,
-        // if so return what we found up until now.
-        if search_pfx.get_len() <= start_bit + nibble_len
-            || (S::into_stride_size(self.ptrbitarr) & bit_pos) == S::zero()
-        // No children or at the end, return the definitive LMP we found.
-        {
-            return (
-                None,      /* no more children */
-                found_pfx, /* The definitive LMP if any */
-            );
-        }
-
-        // There's another child, return it together with the preliminary LMP we found.
-        (
-            Some(self.ptr_vec[S::get_ptr_index(self.ptrbitarr, nibble)]), /* The node that has children the next stride */
-            found_pfx,
-        )
-    }
-
-    // This function looks for the exactly matching prefix in the provided nibble.
-    // It doesn't needd to iterate over anything it just compares the complete nibble, with
-    // the appropriate bits in the requested prefix.
-    // Although this is rather efficient, there's no way to collect less-specific prefixes from
-    // the search prefix.
-    pub(crate) fn search_stride_for_exact_match_at(
-        &self,
-        search_pfx: PrefixId<AF>,
-        nibble: u32,
-        nibble_len: u8,
-        start_bit: u8,
-        _: &mut Option<Vec<NodeId>>,
-    ) -> (Option<NodeId>, Option<NodeId>) {
-        // This is an exact match, so we're only considering the position of the full nibble.
-        let bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_pfx = None;
-        let mut found_child = None;
-
-        // Is this the last nibble?
-        // Otherwise we're not looking for a prefix (exact matching only lives at last nibble)
-        match search_pfx.get_len() <= start_bit + nibble_len {
-            // We're at the last nibble.
-            true => {
-                // Check for an actual prefix at the right position, i.e. consider the complete nibble
-                if self.pfxbitarr & bit_pos > S::zero() {
-                    found_pfx = Some(
-                        self.pfx_vec[S::get_pfx_index(
-                            self.pfxbitarr,
-                            nibble,
-                            nibble_len,
-                        )],
-                    );
-                }
-            }
-            // We're not at the last nibble.
-            false => {
-                // Check for a child node at the right position, i.e. consider the complete nibble.
-                if (S::into_stride_size(self.ptrbitarr) & bit_pos) > S::zero()
-                {
-                    found_child = Some(
-                        self.ptr_vec
-                            [S::get_ptr_index(self.ptrbitarr, nibble)],
-                    );
-                }
-            }
-        }
-
-        (
-            found_child, /* The node that has children in the next stride, if any */
-            found_pfx,   /* The exactly matching prefix, if any */
-        )
-    }
-
-    // This function looks for the exactly matching prefix in the provided nibble,
-    // just like the one above, but this *does* iterate over all the bytes in the nibble to collect
-    // the less-specific prefixes of the the search prefix.
-    // This is of course slower, so it should only be used when the user explicitly requests less-specifics.
-    pub(crate) fn search_stride_for_exact_match_with_less_specifics_at(
-        &self,
-        search_pfx: PrefixId<AF>,
-        mut nibble: u32,
-        nibble_len: u8,
-        start_bit: u8,
-        less_specifics_vec: &mut Option<Vec<NodeId>>,
-    ) -> (Option<NodeId>, Option<NodeId>) {
-        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_pfx = None;
-
-        let ls_vec = less_specifics_vec
-            .as_mut()
-            .expect("You shouldn't call this function without a `less_specifics_vec` buffer. Supply one when calling this function or use `search_stride_for_exact_match_at`");
-
-        for n_l in 1..(nibble_len + 1) {
-            // Move the bit in the right position.
-            nibble = AddressFamily::get_nibble(
-                search_pfx.get_net(),
-                start_bit,
-                n_l,
-            );
-            bit_pos = S::get_bit_pos(nibble, n_l);
-
-            // Check if the prefix has been set, if so select the prefix. This is not
-            // necessarily the final prefix that will be returned.
-
-            // Check it there's a prefix matching in this bitmap for this nibble,
-
-            if self.pfxbitarr & bit_pos > S::zero() {
-                // since we want an exact match only, we will fill the prefix field only
-                // if we're exactly at the last bit of the nibble
-                if n_l == nibble_len {
-                    found_pfx = Some(
-                        self.pfx_vec
-                            [S::get_pfx_index(self.pfxbitarr, nibble, n_l)],
-                    );
-                }
-
-                // Receiving a less_specifics_vec means that the user wants to have
-                // all the last-specific prefixes returned, so add the found prefix.
-                ls_vec.push(
-                    self.pfx_vec
-                        [S::get_pfx_index(self.pfxbitarr, nibble, n_l)],
-                );
-            }
-        }
-
-        if found_pfx.is_none() {
-            // no prefix here, clear out all of the prefixes we found along the way,
-            // since it doesn't make sense to return less-specifics if we don't have a exact match.
-            ls_vec.clear();
-        }
-
-        // Check if this the last stride, or if they're no more children to go to,
-        // if so return what we found up until now.
-        match search_pfx.get_len() <= start_bit + nibble_len
-            || (S::into_stride_size(self.ptrbitarr) & bit_pos)
-                == <S as std::ops::BitAnd>::Output::zero()
-        {
-            // No children or at the end, return the definitive LMP we found.
-            true => (
-                None,      /* no more children */
-                found_pfx, /* The definitive LMP if any */
-            ),
-            // There's another child, we won't return the found_pfx, since we're not
-            // at the last nibble and we want an exact match only.
-            false => (
-                Some(self.ptr_vec[S::get_ptr_index(self.ptrbitarr, nibble)]), /* The node that has children the next stride */
-                None,
-            ),
-        }
-    }
-
-    // Search a stride for more-specific prefixes and child nodes containing
-    // more specifics for `search_prefix`.
-    pub fn add_more_specifics_at(
-        &self,
-        nibble: u32,
-        nibble_len: u8,
-    ) -> (
-        // Option<NodeId>, /* the node with children in the next stride  */
-        Vec<NodeId>, /* child nodes with more more-specifics in this stride */
-        Vec<NodeId>, /* more-specific prefixes in this stride */
-    ) {
-        let mut found_children_with_more_specifics = vec![];
-        let mut found_more_specifics_vec: Vec<NodeId> = vec![];
-
-        // This is an exact match, so we're only considering the position of the full nibble.
-        let mut bit_pos = S::get_bit_pos(nibble, nibble_len);
-        let mut found_child = None;
-
-        // Is there also a child node here?
-        // Note that even without a child node, there may be more specifics further up in this
-        // pfxbitarr or children in this ptrbitarr.
-        if (S::into_stride_size(self.ptrbitarr) & bit_pos) > S::zero() {
-            found_child =
-                Some(self.ptr_vec[S::get_ptr_index(self.ptrbitarr, nibble)]);
-        }
-
-        if let Some(child) = found_child {
-            found_children_with_more_specifics.push(child);
-        }
-
-        // println!("{}..{}", nibble_len + start_bit, S::STRIDE_LEN + start_bit);
-        // println!("start nibble: {:032b}", nibble);
-        // println!("extra bit: {}", (S::STRIDE_LEN - nibble_len));
-
-        // We're expanding the search for more-specifics bit-by-bit.
-        // `ms_nibble_len` is the number of bits including the original nibble we're considering,
-        // e.g. if our prefix has a length of 25 and we've all strides sized 4,
-        // We would end up with a last nibble_len of 1.
-        // `ms_nibble_len` will expand then from 2 up and till 4.
-        // ex.:
-        // nibble: 1 , (nibble_len: 1)
-        // Iteration:
-        // ms_nibble_len=1,n_l=0: 10, n_l=1: 11
-        // ms_nibble_len=2,n_l=0: 100, n_l=1: 101, n_l=2: 110, n_l=3: 111
-        // ms_nibble_len=3,n_l=0: 1000, n_l=1: 1001, n_l=2: 1010, ..., n_l=7: 1111
-
-        for ms_nibble_len in nibble_len + 1..S::STRIDE_LEN + 1 {
-            // iterate over all the possible values for this `ms_nibble_len`,
-            // e.g. two bits can have 4 different values.
-            for n_l in 0..(1 << (ms_nibble_len - nibble_len)) {
-                // move the nibble left with the amount of bits we're going to loop over.
-                // e.g. a stride of size 4 with a nibble 0000 0000 0000 0011 becomes 0000 0000 0000 1100
-                // then it will iterate over ...1100,...1101,...1110,...1111
-                let ms_nibble =
-                    (nibble << (ms_nibble_len - nibble_len)) + n_l as u32;
-                bit_pos = S::get_bit_pos(ms_nibble, ms_nibble_len);
-
-                // println!("nibble:    {:032b}", ms_nibble);
-                // println!("ptrbitarr: {:032b}", self.ptrbitarr);
-                // println!("bitpos:    {:032b}", bit_pos);
-
-                if (S::into_stride_size(self.ptrbitarr) & bit_pos) > S::zero()
-                {
-                    found_children_with_more_specifics.push(
-                        self.ptr_vec
-                            [S::get_ptr_index(self.ptrbitarr, ms_nibble)],
-                    );
-                }
-
-                if self.pfxbitarr & bit_pos > S::zero() {
-                    found_more_specifics_vec.push(
-                        self.pfx_vec[S::get_pfx_index(
-                            self.pfxbitarr,
-                            ms_nibble,
-                            ms_nibble_len,
-                        )],
-                    );
-                }
-            }
-        }
-
-        (
-            // We're done here, the caller should now go over all nodes in found_children_with_more_specifics vec and add
-            // ALL prefixes found in there.
-            found_children_with_more_specifics,
-            found_more_specifics_vec,
-        )
-    }
-}
diff --git a/src/local_vec/query.rs b/src/local_vec/query.rs
deleted file mode 100644
index 76a25ed5..00000000
--- a/src/local_vec/query.rs
+++ /dev/null
@@ -1,793 +0,0 @@
-use crate::local_vec::node::TreeBitMapNode;
-use crate::local_vec::storage_backend::*;
-use crate::local_vec::tree::{SizedStrideNode, TreeBitMap};
-use crate::node_id::SortableNodeId;
-use crate::prefix_record::{PublicRecord, RecordSet, RecordSingleSet};
-use crate::{MatchOptions, MatchType};
-
-use crate::af::AddressFamily;
-use inetnum::addr::Prefix;
-
-#[derive(Hash, Eq, PartialEq, Debug, Copy, Clone)]
-pub struct PrefixId<AF: AddressFamily>((AF, u8));
-
-impl<AF: AddressFamily> PrefixId<AF> {
-    pub fn new(net: AF, len: u8) -> Self {
-        PrefixId((net, len))
-    }
-
-    pub fn get_net(&self) -> AF {
-        self.0 .0
-    }
-
-    pub fn get_len(&self) -> u8 {
-        self.0 .1
-    }
-}
-
-//------------- QuerySingleResult --------------------------------------------
-
-#[derive(Clone, Debug)]
-pub struct QuerySingleResult<M: crate::prefix_record::Meta> {
-    pub match_type: MatchType,
-    pub prefix: Option<Prefix>,
-    pub prefix_meta: Option<M>,
-    pub less_specifics: Option<RecordSingleSet<M>>,
-    pub more_specifics: Option<RecordSingleSet<M>>,
-}
-
-impl<M: crate::prefix_record::Meta> std::fmt::Display
-    for QuerySingleResult<M>
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        let pfx_str = match self.prefix {
-            Some(pfx) => format!("{}", pfx),
-            None => "".to_string(),
-        };
-        let pfx_meta_str = match &self.prefix_meta {
-            Some(pfx_meta) => format!("{}", pfx_meta),
-            None => "".to_string(),
-        };
-        write!(
-            f,
-            "match_type: {}\nprefix: {}\nmetadata: {}\nless_specifics: {}\nmore_specifics: {}",
-            self.match_type,
-            pfx_str,
-            pfx_meta_str,
-            if let Some(ls) = self.less_specifics.as_ref() {
-                format!("{}", ls)
-            } else {
-                "".to_string()
-            },
-            if let Some(ms) = self.more_specifics.as_ref() {
-                format!("{}", ms)
-            } else {
-                "".to_string()
-            },
-        )
-    }
-}
-
-//------------- QueryResult --------------------------------------------
-
-#[derive(Clone, Debug)]
-pub struct QueryResult<M: crate::prefix_record::Meta> {
-    pub match_type: MatchType,
-    pub prefix: Option<Prefix>,
-    pub prefix_meta: PublicRecord<M>,
-    pub less_specifics: Option<RecordSet<M>>,
-    pub more_specifics: Option<RecordSet<M>>,
-}
-
-impl<M: crate::prefix_record::Meta> std::fmt::Display for QueryResult<M> {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        let pfx_str = match self.prefix {
-            Some(pfx) => format!("{}", pfx),
-            None => "".to_string(),
-        };
-        writeln!(
-            f,
-            "match_type: {}\nprefix: {}\nmetadata: {}\nless_specifics: {}\nmore_specifics: {}",
-            self.match_type,
-            pfx_str,
-            self.prefix_meta,
-            if let Some(ls) = self.less_specifics.as_ref() {
-                format!("{}", ls)
-            } else {
-                "".to_string()
-            },
-            if let Some(ms) = self.more_specifics.as_ref() {
-                format!("{}", ms)
-            } else {
-                "".to_string()
-            },
-        )
-    }
-}
-
-//------------ Longest Matching Prefix  -------------------------------------
-
-impl<Store> TreeBitMap<Store>
-where
-    Store: StorageBackend,
-{
-    // In a LMP search we have to go over all the nibble lengths in the stride up
-    // until the value of the actual nibble length were looking for (until we reach
-    // stride length for all strides that aren't the last) and see if the
-    // prefix bit in that posision is set.
-    // Note that this does not search for prefixes with length 0 (which would always
-    // match).
-    // So for matching a nibble 1010, we have to search for 1, 10, 101 and 1010 on
-    // resp. position 1, 5, 12 and 25:
-    //                       ↓          ↓                         ↓                                                              ↓
-    // pfx bit arr (u32)   0 1 2  3  4  5  6   7   8   9  10  11  12  13  14   15   16   17   18   19   20   21   22   23   24   25   26   27   28   29   30   31
-    // nibble              * 0 1 00 01 10 11 000 001 010 011 100 101 110 111 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 1100 1101 1110 1111    x
-    // nibble len offset   0 1    2            3                                4
-
-    pub(crate) fn match_prefix(
-        &self,
-        search_pfx: PrefixId<Store::AF>,
-        options: &MatchOptions,
-    ) -> QuerySingleResult<Store::Meta> {
-        let mut stride_end = 0;
-
-        let mut node = self.retrieve_node(self.get_root_node_id()).unwrap();
-        let mut nibble;
-        let mut nibble_len;
-
-        //---- result values ------------------------------------------------
-
-        // These result values are kept in mutable variables, and assembled
-        // at the end into a QueryResult struct. This proved to result in the
-        // most efficient code, where we don't have to match on
-        // SizedStrideNode over and over. The `match_type` field in the
-        // QueryResult is computed at the end.
-
-        // The final prefix
-        let mut match_prefix_idx: Option<
-            <<Store as StorageBackend>::NodeType as SortableNodeId>::Part,
-        > = None;
-
-        // The indexes of the less-specifics
-        let mut less_specifics_vec = if options.include_less_specifics {
-            Some(Vec::<Store::NodeType>::new())
-        } else {
-            None
-        };
-
-        // The indexes of the more-specifics.
-        let mut more_specifics_vec = if options.include_more_specifics {
-            Some(Vec::<Store::NodeType>::new())
-        } else {
-            None
-        };
-
-        //-------------- Stride Processing ----------------------------------
-
-        // We're going to iterate over all the strides in the treebitmap (so
-        // up to the last bit in the max prefix lentgth for that tree). When
-        // a final prefix is found or we get to the end of the strides,
-        // depending on the options.match_type (the type requested by the user).
-        // we ALWAYS break out of the loop. WE ALWAYS BREAK OUT OF THE LOOP
-        // Just before breaking some processing is done inside the loop before
-        // the break (looking up more-specifics mainly), which looks at bit
-        // repetitious, but again it's been done like that to avoid having to
-        // match over a SizedStrideNode again in the `post-processing`
-        // section.
-
-        for stride in self.strides.iter() {
-            stride_end += stride;
-            let last_stride = search_pfx.get_len() < stride_end;
-
-            nibble_len = if last_stride {
-                stride + search_pfx.get_len() - stride_end
-            } else {
-                *stride
-            };
-
-            // Shift left and right to set the bits to zero that are not in
-            // the nibble we're handling here.
-            nibble = AddressFamily::get_nibble(
-                search_pfx.get_net(),
-                stride_end - stride,
-                nibble_len,
-            );
-
-            match node {
-                SizedStrideNode::Stride3(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-
-                    // This whole match assumes that:
-                    // - if the first value in the return tuple of
-                    //   `search_fn` holds a value, then we need to continue
-                    //   searching by following the node contained in the
-                    //    value.
-                    // - The second value in the tuple holds the prefix that
-                    //   was found.
-                    // The less_specifics_vec is mutated by `search_fn` to
-                    // hold the prefixes found along the way, in the cases
-                    // where `include_less_specifics` was requested by the user.
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        // This and the next match will handle all
-                        // intermediary nodes, but they might also handle
-                        // exit nodes.
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        // This handles exact and longest matches: there are
-                        // no more children, but there is a prefix on this node.
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            break;
-                        }
-                        // This handles cases where there's no prefix (and no
-                        // child) for exact match or longest match, the empty
-                        // match - which doesn't care about actually finding
-                        // a prefix - just continues in search of
-                        // more-specifics.
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    // To make sure we don't process this
-                                    // match arm more then once, we return
-                                    // early here.
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                SizedStrideNode::Stride4(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    // To make sure we don't process this match arm more then once, we
-                                    // return early here.
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                SizedStrideNode::Stride5(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                SizedStrideNode::Stride6(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                SizedStrideNode::Stride7(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                SizedStrideNode::Stride8(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    .get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx.get_part());
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    more_specifics_vec = self
-                                        .get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-            }
-        }
-
-        //------------------ post-processing --------------------------------
-
-        // If the above loop finishes (so not hitting a break) we have processed all strides and have found a child node and maybe a prefix.
-        // Now we will look up more-specifics for longest-matching prefixes that were found in the last stride only,
-        // Note that still any of the match_types (as specified by the user, not the return type) may end up here.
-
-        let mut match_type: MatchType = MatchType::EmptyMatch;
-        let mut prefix = None;
-        if let Some(pfx_idx) = match_prefix_idx {
-            prefix = self.retrieve_prefix(pfx_idx);
-            match_type = if prefix.unwrap().len == search_pfx.get_len() {
-                MatchType::ExactMatch
-            } else {
-                MatchType::LongestMatch
-            }
-        };
-
-        QuerySingleResult {
-            prefix: if let Some(pfx) = prefix {
-                Prefix::new(pfx.net.into_ipaddr(), pfx.len).ok()
-            } else {
-                None
-            },
-            prefix_meta: prefix.map(|pfx| pfx.meta.clone()),
-            match_type,
-            less_specifics: if options.include_less_specifics {
-                less_specifics_vec.map(|vec| {
-                    vec.iter()
-                        .map(|p| self.retrieve_prefix(p.get_part()).unwrap())
-                        .collect::<RecordSingleSet<Store::Meta>>()
-                })
-            } else {
-                None
-            },
-            more_specifics: if options.include_more_specifics {
-                more_specifics_vec.map(|vec| {
-                    vec.iter()
-                        .map(|p| self.retrieve_prefix(p.get_part()).unwrap())
-                        .collect()
-                })
-            } else {
-                None
-            },
-        }
-    }
-}
diff --git a/src/local_vec/storage_backend.rs b/src/local_vec/storage_backend.rs
deleted file mode 100644
index 7da0f319..00000000
--- a/src/local_vec/storage_backend.rs
+++ /dev/null
@@ -1,260 +0,0 @@
-use crate::node_id::{InMemNodeId, SortableNodeId};
-use crate::prefix_record::InternalPrefixRecord;
-
-use crate::local_vec::tree::*;
-
-use crate::af::AddressFamily;
-// use crate::prefix_record::MergeUpdate;
-
-use std::fmt::Debug;
-use std::io::{Error, ErrorKind};
-
-// pub(crate) type PrefixIter<'a, AF, Meta> = Result<
-//     std::slice::Iter<'a, InternalPrefixRecord<AF, Meta>>,
-//     Box<dyn std::error::Error>,
-// >;
-
-pub(crate) trait StorageBackend
-where
-    Self::NodeType: SortableNodeId + Copy,
-{
-    type NodeType;
-    type AF: AddressFamily;
-    type Meta: crate::prefix_record::Meta;
-
-    fn init(
-        start_node: Option<SizedStrideNode<Self::AF, Self::NodeType>>,
-    ) -> Self;
-    fn acquire_new_node_id(
-        &self,
-        sort: <<Self as StorageBackend>::NodeType as SortableNodeId>::Sort,
-        part: <<Self as StorageBackend>::NodeType as SortableNodeId>::Part,
-    ) -> <Self as StorageBackend>::NodeType;
-    fn store_node(
-        &mut self,
-        id: Option<Self::NodeType>,
-        next_node: SizedStrideNode<Self::AF, Self::NodeType>,
-    ) -> Option<Self::NodeType>;
-    fn update_node(
-        &mut self,
-        current_node_id: Self::NodeType,
-        updated_node: SizedStrideNode<Self::AF, Self::NodeType>,
-    );
-    fn retrieve_node(
-        &self,
-        index: Self::NodeType,
-    ) -> Option<&SizedStrideNode<Self::AF, Self::NodeType>>;
-    fn retrieve_node_mut(
-        &mut self,
-        index: Self::NodeType,
-    ) -> SizedNodeResult<Self::AF, Self::NodeType>;
-    fn _retrieve_node_with_guard(
-        &self,
-        index: Self::NodeType,
-    ) -> CacheGuard<Self::AF, Self::NodeType>;
-    fn get_root_node_id(&self) -> Self::NodeType;
-    fn _get_root_node_mut(
-        &mut self,
-    ) -> Option<&mut SizedStrideNode<Self::AF, Self::NodeType>>;
-    fn _get_nodes(&self) -> &Vec<SizedStrideNode<Self::AF, Self::NodeType>>;
-    fn get_nodes_len(&self) -> usize;
-    fn acquire_new_prefix_id(
-        &self,
-        sort: &<<Self as StorageBackend>::NodeType as SortableNodeId>::Sort,
-        part: &InternalPrefixRecord<Self::AF, Self::Meta>,
-    ) -> <Self as StorageBackend>::NodeType;
-    fn store_prefix(
-        &mut self,
-        next_node: InternalPrefixRecord<Self::AF, Self::Meta>,
-    ) -> Result<
-        <<Self as StorageBackend>::NodeType as SortableNodeId>::Part,
-        Box<dyn std::error::Error>,
-    >;
-    fn retrieve_prefix(
-        &self,
-        index: <<Self as StorageBackend>::NodeType as SortableNodeId>::Part,
-    ) -> Option<&InternalPrefixRecord<Self::AF, Self::Meta>>;
-    fn retrieve_prefix_mut(
-        &mut self,
-        index: <<Self as StorageBackend>::NodeType as SortableNodeId>::Part,
-    ) -> Option<&mut InternalPrefixRecord<Self::AF, Self::Meta>>;
-    fn _retrieve_prefix_with_guard(
-        &self,
-        index: Self::NodeType,
-    ) -> PrefixCacheGuard<Self::AF, Self::Meta>;
-    fn _get_prefixes(
-        &self,
-    ) -> &Vec<InternalPrefixRecord<Self::AF, Self::Meta>>;
-    #[cfg(feature = "cli")]
-    fn get_prefixes_len(&self) -> usize;
-    // fn prefixes_iter(&self) -> PrefixIter<'_, Self::AF, Self::Meta>;
-}
-
-#[derive(Debug)]
-pub(crate) struct InMemStorage<
-    AF: AddressFamily,
-    Meta: crate::prefix_record::Meta,
-> {
-    pub nodes: Vec<SizedStrideNode<AF, InMemNodeId>>,
-    pub prefixes: Vec<InternalPrefixRecord<AF, Meta>>,
-}
-
-impl<AF: AddressFamily, Meta: crate::prefix_record::Meta>
-    StorageBackend for InMemStorage<AF, Meta>
-{
-    type NodeType = InMemNodeId;
-    type AF = AF;
-    type Meta = Meta;
-
-    fn init(
-        start_node: Option<SizedStrideNode<Self::AF, Self::NodeType>>,
-    ) -> InMemStorage<AF, Meta> {
-        let mut nodes = vec![];
-        if let Some(n) = start_node {
-            nodes = vec![n];
-        }
-        InMemStorage {
-            nodes,
-            prefixes: vec![],
-        }
-    }
-
-    fn acquire_new_node_id(
-        &self,
-        sort: <<Self as StorageBackend>::NodeType as SortableNodeId>::Sort,
-        _part: <<Self as StorageBackend>::NodeType as SortableNodeId>::Part,
-    ) -> <Self as StorageBackend>::NodeType {
-        // We're ignoring the part parameter here, because we want to store
-        // the index into the global self.nodes vec in the local vec.
-        InMemNodeId(sort, self.nodes.len() as u32)
-    }
-
-    fn store_node(
-        &mut self,
-        _id: Option<Self::NodeType>,
-        next_node: SizedStrideNode<Self::AF, Self::NodeType>,
-    ) -> Option<Self::NodeType> {
-        let id = self.nodes.len() as u32;
-        self.nodes.push(next_node);
-        //Store::NodeType::new(&bit_id, &i.into())
-        //Store::NodeType::new(&((1 << $nibble_len) + $nibble as u16).into(), &i)
-        Some(InMemNodeId::new(&0, &id))
-    }
-
-    fn update_node(
-        &mut self,
-        current_node_id: Self::NodeType,
-        updated_node: SizedStrideNode<Self::AF, Self::NodeType>,
-    ) {
-        let _default_val = std::mem::replace(
-            self.retrieve_node_mut(current_node_id).unwrap(),
-            updated_node,
-        );
-    }
-
-    fn retrieve_node(
-        &self,
-        id: Self::NodeType,
-    ) -> Option<&SizedStrideNode<Self::AF, Self::NodeType>> {
-        self.nodes.get(id.get_part() as usize)
-    }
-
-    fn retrieve_node_mut(
-        &mut self,
-        index: Self::NodeType,
-    ) -> SizedNodeResult<Self::AF, Self::NodeType> {
-        self.nodes
-            .get_mut(index.get_part() as usize)
-            .ok_or_else(|| {
-                Box::new(Error::new(ErrorKind::Other, "Retrieve Node Error"))
-                    .into()
-            })
-    }
-
-    // Don't use this function, this is just a placeholder and a really
-    // inefficient implementation.
-    fn _retrieve_node_with_guard(
-        &self,
-        _id: Self::NodeType,
-    ) -> CacheGuard<Self::AF, Self::NodeType> {
-        panic!("Not Implemented for InMeMStorage");
-    }
-
-    fn get_root_node_id(&self) -> Self::NodeType {
-        InMemNodeId(0, 0)
-    }
-
-    fn _get_root_node_mut(
-        &mut self,
-    ) -> Option<&mut SizedStrideNode<Self::AF, Self::NodeType>> {
-        Some(&mut self.nodes[0])
-    }
-
-    fn _get_nodes(&self) -> &Vec<SizedStrideNode<Self::AF, Self::NodeType>> {
-        &self.nodes
-    }
-
-    fn get_nodes_len(&self) -> usize {
-        self.nodes.len()
-    }
-
-    fn acquire_new_prefix_id(
-        &self,
-        sort: &<<Self as StorageBackend>::NodeType as SortableNodeId>::Sort,
-        _part: &InternalPrefixRecord<
-            <Self as StorageBackend>::AF,
-            <Self as StorageBackend>::Meta,
-        >,
-    ) -> <Self as StorageBackend>::NodeType {
-        // We're ignoring the part parameter here, because we want to store
-        // the index into the global self.prefixes vec in the local vec.
-        InMemNodeId(*sort, self.prefixes.len() as u32)
-    }
-
-    fn store_prefix(
-        &mut self,
-        next_node: InternalPrefixRecord<Self::AF, Self::Meta>,
-    ) -> Result<u32, Box<dyn std::error::Error>> {
-        let id = self.prefixes.len() as u32;
-        self.prefixes.push(next_node);
-        Ok(id)
-    }
-
-    fn retrieve_prefix(
-        &self,
-        index: u32,
-    ) -> Option<&InternalPrefixRecord<Self::AF, Self::Meta>> {
-        self.prefixes.get(index as usize)
-    }
-
-    fn retrieve_prefix_mut(
-        &mut self,
-        index: u32,
-    ) -> Option<&mut InternalPrefixRecord<Self::AF, Self::Meta>> {
-        self.prefixes.get_mut(index as usize)
-    }
-
-    fn _retrieve_prefix_with_guard(
-        &self,
-        _index: Self::NodeType,
-    ) -> PrefixCacheGuard<Self::AF, Self::Meta> {
-        panic!("nOt ImPlEmEnTed for InMemNode");
-    }
-
-    fn _get_prefixes(
-        &self,
-    ) -> &Vec<InternalPrefixRecord<Self::AF, Self::Meta>> {
-        &self.prefixes
-    }
-
-    #[cfg(feature = "cli")]
-    fn get_prefixes_len(&self) -> usize {
-        self.prefixes.len()
-    }
-
-    // fn prefixes_iter(
-    //     &self,
-    // ) -> PrefixIter<'_, AF, Meta> {
-    //     Ok(self.prefixes.iter())
-    // }
-}
diff --git a/src/local_vec/store.rs b/src/local_vec/store.rs
deleted file mode 100644
index 080ce669..00000000
--- a/src/local_vec/store.rs
+++ /dev/null
@@ -1,148 +0,0 @@
-use super::query::QuerySingleResult;
-use crate::local_vec::storage_backend::{InMemStorage, StorageBackend};
-use crate::local_vec::TreeBitMap;
-use crate::node_id::InMemNodeId;
-use crate::prefix_record::InternalPrefixRecord;
-use crate::{AddressFamily, MatchOptions, Stats, Strides};
-
-use crate::af::{IPv4, IPv6};
-use inetnum::addr::Prefix;
-
-use super::query::PrefixId;
-use super::tree::SizedStrideNode;
-/// A fast, memory-efficient Prefix Store, for use in single-threaded contexts.
-///
-/// Can be used in multi-threaded contexts by wrapping it in a `Arc<Mutex<_>>`.
-/// Be aware that this is undesirable in cases with high contention.
-/// Use cases with high contention are best served by the [`crate::MultiThreadedStore`].
-pub struct Store<M: crate::prefix_record::Meta> {
-    v4: TreeBitMap<InMemStorage<IPv4, M>>,
-    v6: TreeBitMap<InMemStorage<IPv6, M>>,
-}
-
-impl<M: crate::prefix_record::Meta> Store<M> {
-    pub fn new(v4_strides: Vec<u8>, v6_strides: Vec<u8>) -> Self {
-        Store {
-            v4: TreeBitMap::new(v4_strides),
-            v6: TreeBitMap::new(v6_strides),
-        }
-    }
-}
-
-impl<M: crate::prefix_record::Meta> Store<M> {
-    pub fn match_prefix(
-        &self,
-        search_pfx: &Prefix,
-        options: &MatchOptions,
-    ) -> QuerySingleResult<M> {
-        match search_pfx.addr() {
-            std::net::IpAddr::V4(addr) => self.v4.match_prefix(
-                PrefixId::<IPv4>::new(
-                    <IPv4 as AddressFamily>::from_ipaddr(addr),
-                    search_pfx.len(),
-                ),
-                options,
-            ),
-            std::net::IpAddr::V6(addr) => self.v6.match_prefix(
-                PrefixId::<IPv6>::new(
-                    <IPv6 as AddressFamily>::from_ipaddr(addr),
-                    search_pfx.len(),
-                ),
-                options,
-            ),
-        }
-    }
-
-    pub fn insert(
-        &mut self,
-        prefix: &Prefix,
-        meta: M,
-        // user_data: Option<&<M>::UserDataIn>,
-    ) -> Result<(), std::boxed::Box<dyn std::error::Error>> {
-        match prefix.addr() {
-            std::net::IpAddr::V4(addr) => {
-                self.v4.insert(InternalPrefixRecord::new_with_meta(
-                    <IPv4 as AddressFamily>::from_ipaddr(addr),
-                    prefix.len(),
-                    meta,
-                ))
-            }
-            std::net::IpAddr::V6(addr) => {
-                self.v6.insert(InternalPrefixRecord::new_with_meta(
-                    <IPv6 as AddressFamily>::from_ipaddr(addr),
-                    prefix.len(),
-                    meta,
-                ))
-            }
-        }
-    }
-
-    pub fn prefixes_iter(&self) -> crate::PrefixSingleRecordIter<M> {
-        let rs4: std::slice::Iter<InternalPrefixRecord<IPv4, M>> =
-            self.v4.store.prefixes[..].iter();
-        let rs6 = self.v6.store.prefixes[..].iter();
-
-        crate::PrefixSingleRecordIter::<M> {
-            v4: Some(rs4),
-            v6: rs6,
-        }
-    }
-
-    pub fn nodes_v4_iter(
-        &self,
-    ) -> impl Iterator<Item = &SizedStrideNode<IPv4, InMemNodeId>> + '_ {
-        self.v4.store.nodes.iter()
-    }
-
-    pub fn nodes_v6_iter(
-        &self,
-    ) -> impl Iterator<Item = &SizedStrideNode<IPv6, InMemNodeId>> {
-        self.v6.store.nodes.iter()
-    }
-
-    pub fn prefixes_len(&self) -> usize {
-        self.v4.store.prefixes.len() + self.v6.store.prefixes.len()
-    }
-
-    pub fn prefixes_v4_len(&self) -> usize {
-        self.v4.store.prefixes.len()
-    }
-
-    pub fn prefixes_v6_len(&self) -> usize {
-        self.v6.store.prefixes.len()
-    }
-
-    pub fn nodes_len(&self) -> usize {
-        self.v4.store.get_nodes_len() + self.v6.store.get_nodes_len()
-    }
-
-    pub fn nodes_v4_len(&self) -> usize {
-        self.v4.store.get_nodes_len()
-    }
-
-    pub fn nodes_v6_len(&self) -> usize {
-        self.v6.store.get_nodes_len()
-    }
-
-    #[cfg(feature = "cli")]
-    pub fn print_funky_stats(&self) {
-        println!("Stats for IPv4 multi-threaded store\n");
-        println!("{}", self.v4);
-        println!("Stats for IPv6 multi-threaded store\n");
-        println!("{}", self.v6);
-    }
-
-    pub fn stats(&self) -> Stats {
-        Stats {
-            v4: &self.v4.stats,
-            v6: &self.v6.stats,
-        }
-    }
-
-    pub fn strides(&self) -> Strides {
-        Strides {
-            v4: &self.v4.strides,
-            v6: &self.v6.strides,
-        }
-    }
-}
diff --git a/src/local_vec/tests/full_table_single.rs b/src/local_vec/tests/full_table_single.rs
deleted file mode 100644
index 7afe94f3..00000000
--- a/src/local_vec/tests/full_table_single.rs
+++ /dev/null
@@ -1,164 +0,0 @@
-#![cfg(test)]
-
-mod full_table {
-    use inetnum::addr::Prefix;
-
-    use crate::test_types::BeBytesAsn;
-    use crate::{prelude::*, PublicPrefixSingleRecord, SingleThreadedStore};
-
-    use std::error::Error;
-    use std::fs::File;
-    use std::process;
-
-    #[test]
-    fn test_full_table_from_csv() -> Result<(), Box<dyn Error>> {
-        // These constants are all contingent on the exact csv file,
-        // being loaded!
-        const CSV_FILE_PATH: &str = "./data/uniq_pfx_asn_dfz_rnd.csv";
-        const SEARCHES_NUM: u32 = 2080800;
-        const INSERTS_NUM: usize = 893943;
-        const GLOBAL_PREFIXES_VEC_SIZE: usize = 886117;
-        const FOUND_PREFIXES: u32 = 1322993;
-
-        fn load_prefixes(
-            pfxs: &mut Vec<PublicPrefixSingleRecord<BeBytesAsn>>,
-        ) -> Result<(), Box<dyn Error>> {
-            let file = File::open(CSV_FILE_PATH)?;
-
-            let mut rdr = csv::Reader::from_reader(file);
-            for result in rdr.records() {
-                let record = result?;
-                let ip: Vec<_> = record[0]
-                    .split('.')
-                    .map(|o| -> u8 { o.parse().unwrap() })
-                    .collect();
-                let net = std::net::Ipv4Addr::new(ip[0], ip[1], ip[2], ip[3]);
-                let len: u8 = record[1].parse().unwrap();
-                let asn: u32 = record[2].parse().unwrap();
-                let pfx = PublicPrefixSingleRecord::new(
-                    Prefix::new(net.into(), len)?,
-                    BeBytesAsn(asn.to_be_bytes()),
-                );
-                pfxs.push(pfx);
-            }
-            Ok(())
-        }
-
-        let strides_vec = [
-            // vec![8],
-            vec![4],
-            // vec![6, 6, 6, 6, 4, 4],
-            // vec![3, 4, 4, 6, 7, 8],
-        ];
-        for _strides in strides_vec.iter().enumerate() {
-            let mut pfxs: Vec<PublicPrefixSingleRecord<BeBytesAsn>> = vec![];
-            let v4_strides = vec![8];
-            let v6_strides = vec![8];
-            let mut tree_bitmap = SingleThreadedStore::<BeBytesAsn>::new(
-                v4_strides, v6_strides,
-            );
-
-            if let Err(err) = load_prefixes(&mut pfxs) {
-                println!("error running example: {}", err);
-                process::exit(1);
-            }
-
-            let inserts_num = pfxs.len();
-            for pfx in pfxs.into_iter() {
-                match tree_bitmap.insert(&pfx.prefix, pfx.meta) {
-                    Ok(_) => {}
-                    Err(e) => {
-                        println!("{}", e);
-                        panic!("STOP TESTING I CAN'T INSERT!");
-                    }
-                };
-
-                let query = tree_bitmap.match_prefix(
-                    &pfx.prefix,
-                    &MatchOptions {
-                        match_type: MatchType::LongestMatch,
-                        include_withdrawn: false,
-                        include_less_specifics: false,
-                        include_more_specifics: false,
-                        mui: None,
-                        include_history: IncludeHistory::None,
-                    },
-                );
-
-                if query.prefix.is_none() {
-                    panic!("STOPSTOPSTOPST");
-                } else {
-                    assert_eq!(query.prefix.unwrap(), pfx.prefix);
-                }
-            }
-
-            println!("done inserting {} prefixes", inserts_num);
-
-            let inet_max = 255;
-            let len_max = 32;
-
-            let mut found_counter = 0_u32;
-            let mut not_found_counter = 0_u32;
-            let mut inet_count = 0;
-            let mut len_count = 0;
-            (0..inet_max).for_each(|i_net| {
-                len_count = 0;
-                (0..len_max).for_each(|s_len| {
-                    (0..inet_max).for_each(|ii_net| {
-                        let pfx = Prefix::new_relaxed(
-                            std::net::Ipv4Addr::new(i_net, ii_net, 0, 0)
-                                .into(),
-                            s_len,
-                        );
-                        // print!(":{}.{}.0.0/{}:", i_net, ii_net, s_len);
-                        let res = tree_bitmap.match_prefix(
-                            &pfx.unwrap(),
-                            &MatchOptions {
-                                match_type: MatchType::LongestMatch,
-                                include_withdrawn: false,
-                                include_less_specifics: false,
-                                include_more_specifics: false,
-                                mui: None,
-                                include_history: IncludeHistory::None,
-                            },
-                        );
-                        if let Some(_pfx) = res.prefix {
-                            // println!("_pfx {:?}", _pfx);
-                            // println!("pfx {:?}", pfx);
-                            // println!("{:#?}", res);
-                            assert!(_pfx.len() <= pfx.unwrap().len());
-                            assert!(_pfx.addr() <= pfx.unwrap().addr());
-                            found_counter += 1;
-                        } else {
-                            // println!(
-                            //     "not found {:?}",
-                            //     if let Ok(e) = pfx {
-                            //         e.to_string()
-                            //     } else {
-                            //         "ok".to_string()
-                            //     }
-                            // );
-                            not_found_counter += 1;
-                        }
-                    });
-                    len_count += 1;
-                });
-                inet_count += 1;
-            });
-            println!("found pfx: {}", found_counter);
-            println!("not found pfx: {}", not_found_counter);
-            println!("inet counter {}", inet_count);
-            println!("len counter {}", len_count);
-
-            let searches_num =
-                inet_max as u128 * inet_max as u128 * len_max as u128;
-
-            assert_eq!(searches_num, SEARCHES_NUM as u128);
-            assert_eq!(inserts_num, INSERTS_NUM);
-            assert_eq!(tree_bitmap.prefixes_len(), GLOBAL_PREFIXES_VEC_SIZE);
-            assert_eq!(found_counter, FOUND_PREFIXES);
-            assert_eq!(not_found_counter, SEARCHES_NUM - FOUND_PREFIXES);
-        }
-        Ok(())
-    }
-}
diff --git a/src/local_vec/tests/mod.rs b/src/local_vec/tests/mod.rs
deleted file mode 100644
index 4f7fe29c..00000000
--- a/src/local_vec/tests/mod.rs
+++ /dev/null
@@ -1,2 +0,0 @@
-mod full_table_single;
-mod more_specifics_single;
\ No newline at end of file
diff --git a/src/local_vec/tests/more_specifics_single.rs b/src/local_vec/tests/more_specifics_single.rs
deleted file mode 100644
index 0f473db5..00000000
--- a/src/local_vec/tests/more_specifics_single.rs
+++ /dev/null
@@ -1,163 +0,0 @@
-#![cfg(test)]
-mod tests {
-    use inetnum::addr::Prefix;
-
-    use crate::{meta_examples::PrefixAs, prelude::*, SingleThreadedStore};
-
-    use std::error::Error;
-
-    #[test]
-    fn test_more_specifics_without_less_specifics(
-    ) -> Result<(), Box<dyn Error>> {
-        let v4_strides = vec![8];
-        let v6_strides = vec![8];
-        let mut tree_bitmap =
-            SingleThreadedStore::<PrefixAs>::new(v4_strides, v6_strides);
-        let pfxs = vec![
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18)?, // 0
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24)?, // 1
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 153, 0).into(), 24)?, // 2
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 21)?, // 3
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 176, 0).into(), 20)?, // 4
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8)?, // 5
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 184, 0).into(), 23)?, // 6
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 71, 0).into(), 24)?, // 7
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9)?, // 8
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 117, 0).into(), 24)?, // 9
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 99, 0).into(), 24)?, // 10
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 224, 0).into(), 24)?, // 11
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 128, 0).into(), 18)?, // 12
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 120, 0).into(), 24)?, // 13
-        ];
-
-        for pfx in pfxs.iter() {
-            tree_bitmap.insert(pfx, PrefixAs::new_from_u32(666))?;
-        }
-        println!("------ end of inserts\n");
-
-        // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
-        for spfx in &[
-            (
-                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9),
-                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9), // 0
-                vec![0, 1, 2, 3, 4, 6, 7, 9, 10, 11, 12, 13],
-            ),
-            (
-                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8),
-                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8), // 0
-                vec![0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13],
-            ),
-        ] {
-            println!("search for: {:?}", spfx.0);
-            let found_result = tree_bitmap.match_prefix(
-                &spfx.0.unwrap(),
-                &MatchOptions {
-                    match_type: MatchType::ExactMatch,
-                    include_withdrawn: false,
-                    include_less_specifics: false,
-                    include_more_specifics: true,
-                    mui: None,
-                    include_history: IncludeHistory::None,
-                },
-            );
-            println!("em/m-s: {:#?}", found_result);
-
-            let more_specifics = found_result.more_specifics.unwrap();
-
-            assert_eq!(found_result.prefix.unwrap(), spfx.1.unwrap());
-            assert_eq!(&more_specifics.len(), &spfx.2.len());
-
-            for i in spfx.2.iter() {
-                print!("{} ", i);
-
-                let result_pfx =
-                    more_specifics.iter().find(|pfx| pfx.prefix == pfxs[*i]);
-                assert!(result_pfx.is_some());
-            }
-            println!("-----------");
-        }
-        Ok(())
-    }
-
-    #[test]
-    fn test_more_specifics_with_less_specifics() -> Result<(), Box<dyn Error>>
-    {
-        let v4_strides = vec![8];
-        let v6_strides = vec![8];
-        let mut tree_bitmap =
-            SingleThreadedStore::<PrefixAs>::new(v4_strides, v6_strides);
-        let pfxs = vec![
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 64, 0).into(), 18), // 0
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 109, 0).into(), 24), // 1
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 153, 0).into(), 24), // 2
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 21), // 3
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 176, 0).into(), 20), // 4
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8), // 5
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 184, 0).into(), 23), // 6
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 71, 0).into(), 24), // 7
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9), // 8
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 117, 0).into(), 24), // 9
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 99, 0).into(), 24), // 10
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 224, 0).into(), 24), // 11
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 128, 0).into(), 18), // 12
-            Prefix::new(std::net::Ipv4Addr::new(17, 0, 120, 0).into(), 24), // 13
-        ];
-
-        for pfx in pfxs.iter() {
-            tree_bitmap.insert(&pfx.unwrap(), PrefixAs::new_from_u32(666))?;
-        }
-        println!("------ end of inserts\n");
-
-        for spfx in &[
-            (
-                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 9),
-                Some(&Prefix::new(
-                    std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
-                    9,
-                )), // 0
-                vec![0, 1, 2, 3, 4, 6, 7, 9, 10, 11, 12, 13],
-            ),
-            (
-                &Prefix::new(std::net::Ipv4Addr::new(17, 0, 0, 0).into(), 8),
-                Some(&Prefix::new(
-                    std::net::Ipv4Addr::new(17, 0, 0, 0).into(),
-                    8,
-                )), // 0
-                vec![0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13],
-            ),
-        ] {
-            println!("search for: {:#}", (*spfx.0)?);
-            let found_result = tree_bitmap.match_prefix(
-                &spfx.0.unwrap(),
-                &MatchOptions {
-                    match_type: MatchType::LongestMatch,
-                    include_withdrawn: false,
-                    include_less_specifics: false,
-                    include_more_specifics: true,
-                    mui: None,
-                    include_history: IncludeHistory::None,
-                },
-            );
-            println!("em/m-s: {}", found_result);
-
-            let more_specifics = found_result.more_specifics.unwrap();
-
-            assert_eq!(
-                found_result.prefix.unwrap(),
-                spfx.1.unwrap().unwrap()
-            );
-            assert_eq!(&more_specifics.len(), &spfx.2.len());
-
-            for i in spfx.2.iter() {
-                print!("{} ", i);
-
-                let result_pfx = more_specifics
-                    .iter()
-                    .find(|pfx| pfx.prefix == pfxs[*i].unwrap());
-                assert!(result_pfx.is_some());
-            }
-            println!("-----------");
-        }
-        Ok(())
-    }
-}
diff --git a/src/local_vec/tree.rs b/src/local_vec/tree.rs
deleted file mode 100644
index 031bd658..00000000
--- a/src/local_vec/tree.rs
+++ /dev/null
@@ -1,568 +0,0 @@
-use std::{
-    fmt::{Binary, Debug},
-    marker::PhantomData,
-};
-
-use crate::local_vec::node::TreeBitMapNode;
-use crate::local_vec::storage_backend::StorageBackend;
-use crate::match_node_for_strides_with_local_vec;
-use crate::node_id::SortableNodeId;
-use crate::prefix_record::InternalPrefixRecord;
-use crate::stats::{SizedStride, StrideStats};
-use crate::stride::*;
-use crate::synth_int::{U256, U512};
-use crate::{
-    af::{AddressFamily, Zero},
-    rib::UpsertReport,
-};
-
-#[cfg(feature = "cli")]
-use ansi_term::Colour;
-
-#[derive(Debug)]
-pub enum SizedStrideNode<AF: AddressFamily, NodeId: SortableNodeId + Copy> {
-    Stride3(TreeBitMapNode<AF, Stride3, NodeId>),
-    Stride4(TreeBitMapNode<AF, Stride4, NodeId>),
-    Stride5(TreeBitMapNode<AF, Stride5, NodeId>),
-    Stride6(TreeBitMapNode<AF, Stride6, NodeId>),
-    Stride7(TreeBitMapNode<AF, Stride7, NodeId>),
-    Stride8(TreeBitMapNode<AF, Stride8, NodeId>),
-}
-
-pub(crate) type SizedNodeResult<'a, AF, NodeType> =
-    Result<&'a mut SizedStrideNode<AF, NodeType>, Box<dyn std::error::Error>>;
-
-impl<AF, NodeId> Default for SizedStrideNode<AF, NodeId>
-where
-    AF: AddressFamily,
-    NodeId: SortableNodeId + Copy,
-{
-    fn default() -> Self {
-        SizedStrideNode::Stride3(TreeBitMapNode {
-            ptrbitarr: 0,
-            pfxbitarr: 0,
-            pfx_vec: vec![],
-            ptr_vec: vec![],
-            _af: PhantomData,
-        })
-    }
-}
-
-pub struct CacheGuard<
-    'a,
-    AF: 'static + AddressFamily,
-    NodeId: SortableNodeId + Copy,
-> {
-    pub guard: std::cell::Ref<'a, SizedStrideNode<AF, NodeId>>,
-}
-
-impl<AF: 'static + AddressFamily, NodeId: SortableNodeId + Copy>
-    std::ops::Deref for CacheGuard<'_, AF, NodeId>
-{
-    type Target = SizedStrideNode<AF, NodeId>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.guard
-    }
-}
-
-pub struct PrefixCacheGuard<
-    'a,
-    AF: 'static + AddressFamily,
-    Meta: crate::prefix_record::Meta,
-> {
-    pub guard: std::cell::Ref<'a, InternalPrefixRecord<AF, Meta>>,
-}
-
-impl<AF: 'static + AddressFamily, Meta: crate::prefix_record::Meta>
-    std::ops::Deref for PrefixCacheGuard<'_, AF, Meta>
-{
-    type Target = InternalPrefixRecord<AF, Meta>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.guard
-    }
-}
-
-pub(crate) enum NewNodeOrIndex<
-    AF: AddressFamily,
-    NodeId: SortableNodeId + Copy,
-> {
-    NewNode(SizedStrideNode<AF, NodeId>, NodeId::Sort), // New Node and bit_id of the new node
-    ExistingNode(NodeId),
-    NewPrefix,
-    ExistingPrefix(NodeId::Part),
-}
-
-pub(crate) struct TreeBitMap<Store>
-where
-    Store: StorageBackend,
-{
-    pub strides: Vec<u8>,
-    pub stats: Vec<StrideStats>,
-    pub store: Store,
-}
-
-impl<Store> TreeBitMap<Store>
-where
-    Store: StorageBackend,
-{
-    pub fn new(_strides_vec: Vec<u8>) -> TreeBitMap<Store> {
-        // Check if the strides division makes sense
-        let mut strides = vec![];
-        let mut strides_sum = 0;
-        for s in _strides_vec.iter().cycle() {
-            strides.push(*s);
-            strides_sum += s;
-            if strides_sum >= Store::AF::BITS - 1 {
-                break;
-            }
-        }
-        assert_eq!(strides.iter().sum::<u8>(), Store::AF::BITS);
-
-        let mut stride_stats: Vec<StrideStats> = vec![
-            StrideStats::new(SizedStride::Stride3, strides.len() as u8), // 0
-            StrideStats::new(SizedStride::Stride4, strides.len() as u8), // 1
-            StrideStats::new(SizedStride::Stride5, strides.len() as u8), // 2
-            StrideStats::new(SizedStride::Stride6, strides.len() as u8), // 3
-            StrideStats::new(SizedStride::Stride7, strides.len() as u8), // 4
-            StrideStats::new(SizedStride::Stride8, strides.len() as u8), // 5
-        ];
-
-        let node: SizedStrideNode<
-            <Store as StorageBackend>::AF,
-            <Store as StorageBackend>::NodeType,
-        >;
-
-        match strides[0] {
-            3 => {
-                node = SizedStrideNode::Stride3(TreeBitMapNode {
-                    ptrbitarr: 0,
-                    pfxbitarr: 0,
-                    ptr_vec: vec![],
-                    pfx_vec: vec![],
-                    _af: PhantomData,
-                });
-                stride_stats[0].inc(0);
-            }
-            4 => {
-                node = SizedStrideNode::Stride4(TreeBitMapNode {
-                    ptrbitarr: 0,
-                    pfxbitarr: 0,
-                    ptr_vec: vec![],
-                    pfx_vec: vec![],
-                    _af: PhantomData,
-                });
-                stride_stats[1].inc(0);
-            }
-            5 => {
-                node = SizedStrideNode::Stride5(TreeBitMapNode {
-                    ptrbitarr: 0,
-                    pfxbitarr: 0,
-                    ptr_vec: vec![],
-                    pfx_vec: vec![],
-                    _af: PhantomData,
-                });
-                stride_stats[2].inc(0);
-            }
-            6 => {
-                node = SizedStrideNode::Stride6(TreeBitMapNode {
-                    ptrbitarr: 0,
-                    pfxbitarr: 0,
-                    ptr_vec: vec![],
-                    pfx_vec: vec![],
-                    _af: PhantomData,
-                });
-                stride_stats[3].inc(0);
-            }
-            7 => {
-                node = SizedStrideNode::Stride7(TreeBitMapNode {
-                    ptrbitarr: 0,
-                    pfxbitarr: U256(0, 0),
-                    ptr_vec: vec![],
-                    pfx_vec: vec![],
-                    _af: PhantomData,
-                });
-                stride_stats[4].inc(0);
-            }
-            8 => {
-                node = SizedStrideNode::Stride8(TreeBitMapNode {
-                    ptrbitarr: U256(0, 0),
-                    pfxbitarr: U512(0, 0, 0, 0),
-                    ptr_vec: vec![],
-                    pfx_vec: vec![],
-                    _af: PhantomData,
-                });
-                stride_stats[5].inc(0);
-            }
-            _ => {
-                panic!("unknown stride size encountered in STRIDES array");
-            }
-        };
-
-        TreeBitMap {
-            strides,
-            stats: stride_stats,
-            store: Store::init(Some(node)),
-        }
-    }
-
-    // Partition for stride 4
-    //
-    // ptr bits never happen in the first half of the bitmap for the stride-size. Consequently the ptrbitarr can be an integer type
-    // half the size of the pfxbitarr.
-    //
-    // ptr bit arr (u16)                                                        0    1    2    3    4    5    6    7    8    9   10   11   12   13   14   15    x
-    // pfx bit arr (u32)   0 1 2  3  4  5  6   7   8   9  10  11  12  13  14   15   16   17   18   19   20   21   22   23   24   25   26   27   28   29   30   31
-    // nibble              * 0 1 00 01 10 11 000 001 010 011 100 101 110 111 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 1100 1101 1110 1111    x
-    // nibble len offset   0 1    2            3                                4
-    //
-    // stride 3: 1 + 2 + 4 + 8                              =  15 bits. 2^4 - 1 (1 << 4) - 1. ptrbitarr starts at pos  7 (1 << 3) - 1
-    // stride 4: 1 + 2 + 4 + 8 + 16                         =  31 bits. 2^5 - 1 (1 << 5) - 1. ptrbitarr starts at pos 15 (1 << 4) - 1
-    // stride 5: 1 + 2 + 4 + 8 + 16 + 32 + 64               =  63 bits. 2^6 - 1
-    // stride 6: 1 + 2 + 4 + 8 + 16 + 32 + 64               = 127 bits. 2^7 - 1
-    // stride 7: 1 + 2 + 4 + 8 + 16 + 32 + 64 = 128         = 256 bits. 2^8 - 1126
-    // stride 8: 1 + 2 + 4 + 8 + 16 + 32 + 64 + 128 + 256   = 511 bits. 2^9 - 1
-    //
-    // Ex.:
-    // pfx            65.0.0.252/30                                             0100_0001_0000_0000_0000_0000_1111_1100
-    //
-    // nibble 1       (pfx << 0) >> 28                                          0000_0000_0000_0000_0000_0000_0000_0100
-    // bit_pos        (1 << nibble length) - 1 + nibble                         0000_0000_0000_0000_0000_1000_0000_0000
-    //
-    // nibble 2       (pfx << 4) >> 24                                          0000_0000_0000_0000_0000_0000_0000_0001
-    // bit_pos        (1 << nibble length) - 1 + nibble                         0000_0000_0000_0000_1000_0000_0000_0000
-    // ...
-    // nibble 8       (pfx << 28) >> 0                                          0000_0000_0000_0000_0000_0000_0000_1100
-    // bit_pos        (1 << nibble length) - 1 + nibble = (1 << 2) - 1 + 2 = 5  0000_0010_0000_0000_0000_0000_0000_0000
-    // 5 - 5 - 5 - 4 - 4 - [4] - 5
-    // startpos (2 ^ nibble length) - 1 + nibble as usize
-
-    pub(crate) fn insert(
-        &mut self,
-        pfx: InternalPrefixRecord<Store::AF, Store::Meta>,
-        // user_data: Option<&<Store::Meta>::UserDataIn>,
-    ) -> Result<(), Box<dyn std::error::Error>> {
-        let mut stride_end: u8 = 0;
-        let mut cur_i = self.store.get_root_node_id();
-        let mut level: u8 = 0;
-
-        loop {
-            let stride = self.strides[level as usize];
-            stride_end += stride;
-            let nibble_len = if pfx.len < stride_end {
-                stride + pfx.len - stride_end
-            } else {
-                stride
-            };
-
-            let nibble = Store::AF::get_nibble(
-                pfx.net,
-                stride_end - stride,
-                nibble_len,
-            );
-            let is_last_stride = pfx.len <= stride_end;
-
-            let next_node_idx = match_node_for_strides_with_local_vec![
-                // applicable to the whole outer match in the marco
-                self;
-                // user_data;
-                nibble_len;
-                nibble;
-                is_last_stride;
-                pfx;
-                cur_i;
-                level;
-                // Strides to create match arm for; stats level
-                Stride3; 0,
-                Stride4; 1,
-                Stride5; 2,
-                Stride6; 3,
-                Stride7; 4,
-                Stride8; 5
-            ];
-
-            if let Some(i) = next_node_idx {
-                cur_i = i;
-                level += 1;
-            } else {
-                return Ok(());
-            }
-        }
-    }
-
-    pub(crate) fn store_node(
-        &mut self,
-        id: Option<Store::NodeType>,
-        next_node: SizedStrideNode<Store::AF, Store::NodeType>,
-    ) -> Option<Store::NodeType> {
-        self.store.store_node(id, next_node)
-    }
-
-    #[inline]
-    pub(crate) fn retrieve_node(
-        &self,
-        id: Store::NodeType,
-    ) -> Option<&SizedStrideNode<Store::AF, Store::NodeType>> {
-        self.store.retrieve_node(id)
-    }
-
-    pub(crate) fn get_root_node_id(&self) -> Store::NodeType {
-        self.store.get_root_node_id()
-    }
-
-    #[inline]
-    pub(crate) fn retrieve_node_mut(
-        &mut self,
-        index: Store::NodeType,
-    ) -> SizedNodeResult<Store::AF, Store::NodeType> {
-        self.store.retrieve_node_mut(index)
-    }
-
-    pub(crate) fn store_prefix(
-        &mut self,
-        next_node: InternalPrefixRecord<Store::AF, Store::Meta>,
-    ) -> Result<
-        <<Store as StorageBackend>::NodeType as SortableNodeId>::Part,
-        Box<dyn std::error::Error>,
-    > {
-        // let id = self.prefixes.len() as u32;
-        self.store.store_prefix(next_node)
-        // id
-    }
-
-    pub(crate) fn update_prefix_meta(
-        &mut self,
-        update_node_idx: <<Store as StorageBackend>::NodeType as SortableNodeId>::Part,
-        meta: Store::Meta,
-        // user_data: Option<&<Store::Meta>::UserDataIn>,
-    ) -> Result<UpsertReport, Box<dyn std::error::Error>> {
-        match self.store.retrieve_prefix_mut(update_node_idx) {
-            Some(update_pfx) => {
-                update_pfx.meta = meta;
-                Ok(UpsertReport {
-                    cas_count: 0,
-                    prefix_new: false,
-                    mui_new: false,
-                    mui_count: 0,
-                })
-                // <Store::Meta>::merge_update(&mut update_pfx.meta, meta)
-            }
-            // TODO
-            // Use/create proper error types
-            None => Err("Prefix not found".into()),
-        }
-    }
-
-    #[inline]
-    pub(crate) fn retrieve_prefix(
-        &self,
-        index: <<Store as StorageBackend>::NodeType as SortableNodeId>::Part,
-    ) -> Option<&InternalPrefixRecord<Store::AF, Store::Meta>> {
-        self.store.retrieve_prefix(index)
-    }
-
-    // This function assembles all entries in the `pfx_vec` of all child nodes of the
-    // `start_node` into one vec, starting from iself and then recursively assembling
-    // adding all `pfx_vec`s of its children.
-    fn get_all_more_specifics_for_node(
-        &self,
-        start_node: &SizedStrideNode<Store::AF, Store::NodeType>,
-        found_pfx_vec: &mut Vec<Store::NodeType>,
-    ) {
-        match start_node {
-            SizedStrideNode::Stride3(n) => {
-                found_pfx_vec.extend_from_slice(&n.pfx_vec);
-
-                for nn in n.ptr_vec.iter() {
-                    self.get_all_more_specifics_for_node(
-                        self.retrieve_node(*nn).unwrap(),
-                        found_pfx_vec,
-                    );
-                }
-            }
-            SizedStrideNode::Stride4(n) => {
-                found_pfx_vec.extend_from_slice(&n.pfx_vec);
-
-                for nn in n.ptr_vec.iter() {
-                    self.get_all_more_specifics_for_node(
-                        self.retrieve_node(*nn).unwrap(),
-                        found_pfx_vec,
-                    );
-                }
-            }
-            SizedStrideNode::Stride5(n) => {
-                found_pfx_vec.extend_from_slice(&n.pfx_vec);
-
-                for nn in n.ptr_vec.iter() {
-                    self.get_all_more_specifics_for_node(
-                        self.retrieve_node(*nn).unwrap(),
-                        found_pfx_vec,
-                    );
-                }
-            }
-            SizedStrideNode::Stride6(n) => {
-                found_pfx_vec.extend_from_slice(&n.pfx_vec);
-
-                for nn in n.ptr_vec.iter() {
-                    self.get_all_more_specifics_for_node(
-                        self.retrieve_node(*nn).unwrap(),
-                        found_pfx_vec,
-                    );
-                }
-            }
-            SizedStrideNode::Stride7(n) => {
-                found_pfx_vec.extend_from_slice(&n.pfx_vec);
-
-                for nn in n.ptr_vec.iter() {
-                    self.get_all_more_specifics_for_node(
-                        self.retrieve_node(*nn).unwrap(),
-                        found_pfx_vec,
-                    );
-                }
-            }
-            SizedStrideNode::Stride8(n) => {
-                found_pfx_vec.extend_from_slice(&n.pfx_vec);
-
-                for nn in n.ptr_vec.iter() {
-                    self.get_all_more_specifics_for_node(
-                        self.retrieve_node(*nn).unwrap(),
-                        found_pfx_vec,
-                    );
-                }
-            }
-        }
-    }
-
-    // This function assembles the prefixes of a child node starting on a specified bit position in a ptr_vec of
-    // `current_node` into a vec, then adds all prefixes of these children recursively into a vec and returns that.
-    pub fn get_all_more_specifics_from_nibble<S>(
-        &self,
-        current_node: &TreeBitMapNode<Store::AF, S, Store::NodeType>,
-        nibble: u32,
-        nibble_len: u8,
-    ) -> Option<Vec<Store::NodeType>>
-    where
-        S: Stride
-            + std::ops::BitAnd<Output = S>
-            + std::ops::BitOr<Output = S>
-            + Zero,
-        <S as Stride>::PtrSize: Debug
-            + Binary
-            + Copy
-            + std::ops::BitAnd<Output = S::PtrSize>
-            + PartialOrd
-            + Zero,
-    {
-        let (cnvec, mut msvec) =
-            current_node.add_more_specifics_at(nibble, nibble_len);
-
-        for child_node in cnvec.iter() {
-            self.get_all_more_specifics_for_node(
-                self.retrieve_node(*child_node).unwrap(),
-                &mut msvec,
-            );
-        }
-        Some(msvec)
-    }
-}
-
-// This implements the funky stats for a tree
-#[cfg(feature = "cli")]
-impl<Store: StorageBackend> std::fmt::Display for TreeBitMap<Store> {
-    fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let total_nodes = self.store.get_nodes_len();
-
-        writeln!(_f, "prefix vec size {}", self.store.get_prefixes_len())?;
-        writeln!(_f, "finished building tree...")?;
-        writeln!(_f, "{:?} nodes created", total_nodes)?;
-        // writeln!(
-        //     _f,
-        //     "size of node: {} bytes",
-        //     std::mem::size_of::<SizedStrideNode<u32, InMemNodeId>>()
-        // )?;
-        // writeln!(
-        //     _f,
-        //     "memory used by nodes: {}kb",
-        //     self.store.get_nodes_len()
-        //         * std::mem::size_of::<SizedStrideNode<u32, InMemNodeId>>()
-        //         / 1024
-        // )?;
-
-        writeln!(_f, "stride division {:?}", self.strides)?;
-        for s in &self.stats {
-            writeln!(_f, "{:?}", s)?;
-        }
-
-        writeln!(
-            _f,
-            "level\t[{}|{}] nodes occupied/max nodes percentage_max_nodes_occupied prefixes",
-            Colour::Blue.paint("nodes"),
-            Colour::Green.paint("prefixes")
-        )?;
-        let bars = ["▏", "▎", "▍", "▌", "▋", "▊", "▉"];
-        let mut stride_bits = [0, 0];
-        const SCALE: u32 = 5500;
-
-        for stride in self.strides.iter().enumerate() {
-            // let level = stride.0;
-            stride_bits = [stride_bits[1] + 1, stride_bits[1] + stride.1];
-            let nodes_num = self
-                .stats
-                .iter()
-                .find(|s| s.stride_len == *stride.1)
-                .unwrap()
-                .created_nodes[stride.0]
-                .count as u32;
-            let prefixes_num = self
-                .stats
-                .iter()
-                .find(|s| s.stride_len == *stride.1)
-                .unwrap()
-                .prefixes_num[stride.0]
-                .count as u32;
-
-            let n = (nodes_num / SCALE) as usize;
-            let max_pfx = u128::overflowing_pow(2, stride_bits[1] as u32);
-
-            write!(_f, "{}-{}\t", stride_bits[0], stride_bits[1])?;
-
-            for _ in 0..n {
-                write!(_f, "{}", Colour::Blue.paint("█"))?;
-            }
-
-            writeln!(
-                _f,
-                "{}",
-                Colour::Blue.paint(
-                    bars[((nodes_num % SCALE) / (SCALE / 7)) as usize]
-                ) //  = scale / 7
-            )?;
-
-            writeln!(
-                _f,
-                " {}/{} {:.2}%",
-                nodes_num,
-                max_pfx.0,
-                (nodes_num as f64 / max_pfx.0 as f64) * 100.0
-            )?;
-            write!(_f, "\n\t")?;
-
-            let n = (prefixes_num / SCALE) as usize;
-            for _ in 0..n {
-                write!(_f, "{}", Colour::Green.paint("█"))?;
-            }
-
-            write!(
-                _f,
-                "{}",
-                Colour::Green.paint(
-                    bars[((nodes_num % SCALE) / (SCALE / 7)) as usize]
-                ) //  = scale / 7
-            )?;
-
-            writeln!(_f, " {}", prefixes_num)?;
-        }
-        Ok(())
-    }
-}
diff --git a/src/macros.rs b/src/macros.rs
index 5920aa65..a033be94 100644
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -1,48 +1,3 @@
-#[macro_export]
-// This macro only works for stride with bitmaps that are <= u128,
-// the ones with synthetic integers (U256, U512) don't have the trait
-// implementations for left|right shift, counting ones etc.
-#[doc(hidden)]
-macro_rules! impl_primitive_stride {
-    ( $( $len: expr; $bits: expr; $pfxsize: ty; $ptrsize: ty ), * ) => {
-            $(
-                impl Stride for $pfxsize {
-                    type PtrSize = $ptrsize;
-                    const BITS: u8 = $bits;
-                    const STRIDE_LEN: u8 = $len;
-
-                    fn get_bit_pos(bs: BitSpan) -> $pfxsize {
-                        1 << (<Self as Stride>::BITS - ((1 << bs.len) - 1) as u8 - bs.bits as u8 - 1)
-                    }
-
-                    fn get_pfx_index(bitmap: $pfxsize, bs: BitSpan) -> usize {
-                        (bitmap >> ((<Self as Stride>::BITS - ((1 << bs.len) - 1) as u8 - bs.bits as u8 - 1) as usize))
-                            .count_ones() as usize
-                            - 1
-                    }
-                    fn get_ptr_index(bitmap: $ptrsize, nibble: u32) -> usize {
-                        (bitmap >> ((<Self as Stride>::BITS >> 1) - nibble as u8 - 1) as usize).count_ones()
-                            as usize
-                            - 1
-                    }
-
-                    fn into_stride_size(bitmap: $ptrsize) -> $pfxsize {
-                        bitmap as $pfxsize << 1
-                    }
-
-                    fn into_ptrbitarr_size(bitmap: $pfxsize) -> $ptrsize {
-                        (bitmap >> 1) as $ptrsize
-                    }
-
-                    #[inline]
-                    fn leading_zeros(self) -> u32 {
-                        self.leading_zeros()
-                    }
-                }
-            )*
-    };
-}
-
 #[macro_export]
 #[doc(hidden)]
 macro_rules! all_strategies {
@@ -55,7 +10,8 @@ macro_rules! all_strategies {
                 //------- Default (MemoryOnly)
                 println!("MemoryOnly strategy starting...");
                 let tree_bitmap =
-                    MultiThreadedStore::<$ty, MemoryOnlyConfig>::try_default()?;
+                    MultiThreadedStore::<
+                        $ty, MemoryOnlyConfig>::try_default()?;
 
                 $test_name(tree_bitmap)?;
 
@@ -115,76 +71,3 @@ macro_rules! all_strategies {
         )*
     };
 }
-
-#[macro_export]
-#[doc(hidden)]
-macro_rules! all_strategies_arced {
-    ( $( $fn_name: ident; $test_name: ident; $ty: ty ), * ) => {
-
-        $(
-            #[test]
-            fn $fn_name() -> Result<(), Box<dyn std::error::Error>> {
-                //------- Default (MemoryOnly)
-
-                println!("default strategy starting...");
-                let tree_bitmap =
-                    MultiThreadedStore::<$ty>::try_default()?;
-
-                $test_name(Arc::new(tree_bitmap))?;
-
-                //------- PersistOnly
-
-                println!("PersistOnly strategy starting...");
-                let store_config = StoreConfig {
-                    persist_strategy:
-                        rotonda_store::rib::PersistStrategy::PersistOnly,
-                    persist_path: "/tmp/rotonda/".into(),
-                };
-
-                let tree_bitmap = MultiThreadedStore::<
-                    $ty,
-                >::new_with_config(
-                    store_config
-                )?;
-
-                $test_name(Arc::new(tree_bitmap))?;
-
-                //------- PersistHistory
-
-                println!("PersistHistory strategy starting...");
-                let store_config = StoreConfig {
-                    persist_strategy:
-                        rotonda_store::rib::PersistStrategy::PersistHistory,
-                    persist_path: "/tmp/rotonda/".into(),
-                };
-
-                let tree_bitmap = MultiThreadedStore::<
-                    $ty,
-                >::new_with_config(
-                    store_config
-                )?;
-
-                $test_name(Arc::new(tree_bitmap))?;
-
-                //------- WriteAhead
-
-                println!("WriteAhead strategy starting...");
-                let store_config = StoreConfig {
-                    persist_strategy:
-                        rotonda_store::rib::PersistStrategy::WriteAhead,
-                    persist_path: "/tmp/rotonda/".into(),
-                };
-
-                let tree_bitmap = MultiThreadedStore::<
-                    $ty,
-                >::new_with_config(
-                    store_config
-                )?;
-
-                $test_name(Arc::new(tree_bitmap))?;
-
-                Ok(())
-            }
-        )*
-    };
-}
diff --git a/src/node_id.rs b/src/node_id.rs
deleted file mode 100644
index b1ecd071..00000000
--- a/src/node_id.rs
+++ /dev/null
@@ -1,62 +0,0 @@
-//------------ NodeId Types -------------------------------------------------
-
-pub trait SortableNodeId
-where
-    Self: std::cmp::Ord + std::fmt::Debug + Sized + Default,
-    Self::Sort:
-        std::cmp::Ord + std::convert::From<u16> + std::convert::Into<usize>,
-    Self::Part: std::cmp::Ord
-        + std::convert::From<u16>
-        + std::marker::Copy
-        + std::fmt::Debug,
-{
-    type Part;
-    type Sort;
-    // fn sort(&self, other: &Self) -> std::cmp::Ordering;
-    fn new(sort: &Self::Sort, part: &Self::Part) -> Self;
-    fn empty() -> Self;
-    fn get_sort(&self) -> Self::Sort;
-    fn get_part(&self) -> Self::Part;
-    fn is_empty(&self) -> bool;
-}
-
-#[derive(
-    Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Copy, Clone, Default,
-)]
-pub struct InMemNodeId(pub u16, pub u32);
-
-// This works for both IPv4 and IPv6 up to a certain point.
-// the u16 for Sort is used for ordering the local vecs
-// inside the nodes.
-// The u32 Part is used as an index to the backing global vecs,
-// so you CANNOT store all IPv6 prefixes that could exist!
-// If you really want that you should implement your own type with trait
-// SortableNodeId, e.g., Sort = u16, Part = u128.
-impl SortableNodeId for InMemNodeId {
-    type Sort = u16;
-    type Part = u32;
-
-    // fn sort(&self, other: &Self) -> std::cmp::Ordering {
-    //     self.0.cmp(&other.0)
-    // }
-
-    fn new(sort: &Self::Sort, part: &Self::Part) -> InMemNodeId {
-        InMemNodeId(*sort, *part)
-    }
-
-    fn get_sort(&self) -> Self::Sort {
-        self.0
-    }
-
-    fn get_part(&self) -> Self::Part {
-        self.1
-    }
-
-    fn is_empty(&self) -> bool {
-        self.0 == 0 && self.1 == 0
-    }
-
-    fn empty() -> Self {
-        Self::new(&0, &0)
-    }
-}
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index 906b4022..69c5f1fb 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -2,7 +2,6 @@ pub use crate::{AddressFamily, IPv4, IPv6};
 
 pub use crate::af::IntoIpAddr;
 pub use crate::prefix_record::{Meta, PublicPrefixRecord as PrefixRecord};
-pub use crate::stride::{Stride3, Stride4, Stride5};
 pub use crate::{IncludeHistory, MatchOptions, MatchType, QueryResult};
 pub use inetnum::addr::Prefix;
 
diff --git a/src/rotonda_store.rs b/src/rotonda_store.rs
index 14e74ac1..307d5f2e 100644
--- a/src/rotonda_store.rs
+++ b/src/rotonda_store.rs
@@ -21,11 +21,11 @@ pub use crate::local_array::rib::DefaultStore as MultiThreadedStore;
 
 //------------ Types for strides displaying/monitoring ----------------------
 
-type AfStrideStats = Vec<StrideStats>;
+type AfStrideStats<AF> = Vec<StrideStats<AF>>;
 
 pub struct Stats<'a> {
-    pub v4: &'a AfStrideStats,
-    pub v6: &'a AfStrideStats,
+    pub v4: &'a AfStrideStats<IPv4>,
+    pub v6: &'a AfStrideStats<IPv6>,
 }
 
 impl std::fmt::Display for Stats<'_> {
diff --git a/src/stats.rs b/src/stats.rs
index 8ac46c05..e8fe5d2d 100644
--- a/src/stats.rs
+++ b/src/stats.rs
@@ -1,82 +1,37 @@
 //------------ Types for Statistics -----------------------------------------
 
-use crate::stride::{Stride3, Stride4, Stride5, Stride6, Stride7, Stride8};
-use std::fmt::{Debug, Display};
+// use crate::stride::{Stride3, Stride4, Stride5, Stride6, Stride7, Stride8};
+use std::{
+    fmt::{Debug, Display},
+    marker::PhantomData,
+};
 
-#[derive(Debug, Copy, Clone)]
-pub enum SizedStride {
-    Stride3,
-    Stride4,
-    Stride5,
-    Stride6,
-    Stride7,
-    Stride8,
-}
-pub struct StrideStats {
-    pub stride_type: SizedStride,
-    pub stride_size: usize,
-    pub stride_len: u8,
+use crate::{
+    local_array::{
+        in_memory::node::TreeBitMapNode, rib::default_store::STRIDE_SIZE,
+    },
+    AddressFamily,
+};
+
+pub struct StrideStats<AF: AddressFamily> {
     pub node_size: usize,
     pub created_nodes: Vec<CreatedNodes>,
     pub prefixes_num: Vec<CreatedNodes>,
+    _af: PhantomData<AF>,
 }
 
-impl StrideStats {
-    pub fn new(stride_type: SizedStride, num_depth_levels: u8) -> Self {
-        match stride_type {
-            SizedStride::Stride3 => Self {
-                stride_type: SizedStride::Stride3,
-                stride_size: 16,
-                stride_len: 3,
-                node_size: std::mem::size_of::<Stride3>(),
-                created_nodes: Self::nodes_vec(num_depth_levels),
-                prefixes_num: Self::nodes_vec(num_depth_levels),
-            },
-            SizedStride::Stride4 => Self {
-                stride_type: SizedStride::Stride4,
-                stride_size: 32,
-                stride_len: 4,
-                node_size: std::mem::size_of::<Stride4>(),
-                created_nodes: Self::nodes_vec(num_depth_levels),
-                prefixes_num: Self::nodes_vec(num_depth_levels),
-            },
-            SizedStride::Stride5 => Self {
-                stride_type: SizedStride::Stride5,
-                stride_size: 64,
-                stride_len: 5,
-                node_size: std::mem::size_of::<Stride5>(),
-                created_nodes: Self::nodes_vec(num_depth_levels),
-                prefixes_num: Self::nodes_vec(num_depth_levels),
-            },
-            SizedStride::Stride6 => Self {
-                stride_type: SizedStride::Stride6,
-                stride_size: 128,
-                stride_len: 6,
-                node_size: std::mem::size_of::<Stride6>(),
-                created_nodes: Self::nodes_vec(num_depth_levels),
-                prefixes_num: Self::nodes_vec(num_depth_levels),
-            },
-            SizedStride::Stride7 => Self {
-                stride_type: SizedStride::Stride7,
-                stride_size: 256,
-                stride_len: 7,
-                node_size: std::mem::size_of::<Stride7>(),
-                created_nodes: Self::nodes_vec(num_depth_levels),
-                prefixes_num: Self::nodes_vec(num_depth_levels),
-            },
-            SizedStride::Stride8 => Self {
-                stride_type: SizedStride::Stride8,
-                stride_size: 512,
-                stride_len: 8,
-                node_size: std::mem::size_of::<Stride8>(),
-                created_nodes: Self::nodes_vec(num_depth_levels),
-                prefixes_num: Self::nodes_vec(num_depth_levels),
-            },
+impl<AF: AddressFamily> StrideStats<AF> {
+    pub fn new() -> Self {
+        Self {
+            node_size: std::mem::size_of::<TreeBitMapNode<AF>>(),
+            created_nodes: Self::nodes_vec(AF::BITS / STRIDE_SIZE),
+            prefixes_num: Self::nodes_vec(AF::BITS / STRIDE_SIZE),
+            _af: PhantomData,
         }
     }
 
     pub fn mem_usage(&self) -> usize {
-        self.stride_size
+        STRIDE_SIZE as usize
             * self.created_nodes.iter().fold(0, |mut acc, c| {
                 acc += c.count;
                 acc
@@ -103,12 +58,17 @@ impl StrideStats {
     }
 }
 
-impl Debug for StrideStats {
+impl<AF: AddressFamily> Default for StrideStats<AF> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<AF: AddressFamily> Debug for StrideStats<AF> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "{:?}:{:>8?} {:?} ({}k)",
-            &self.stride_type,
+            "Stride4:{:>8?} {:?} ({}k)",
             &self.created_nodes.iter().fold(0, |mut a, n| {
                 a += n.count;
                 a
@@ -119,12 +79,11 @@ impl Debug for StrideStats {
     }
 }
 
-impl Display for StrideStats {
+impl<AF: AddressFamily> Display for StrideStats<AF> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "{:?}:{:>8?} {:?} ({}k)",
-            &self.stride_type,
+            "Stride4:{:>8?} {:?} ({}k)",
             &self.created_nodes.iter().fold(0, |mut a, n| {
                 a += n.count;
                 a
diff --git a/src/stride.rs b/src/stride.rs
deleted file mode 100644
index f13911d1..00000000
--- a/src/stride.rs
+++ /dev/null
@@ -1,263 +0,0 @@
-use crate::impl_primitive_stride;
-use crate::local_array::bit_span::BitSpan;
-use crate::synth_int::{U256, U512};
-use std::fmt::{Binary, Debug};
-
-pub type Stride3 = u16;
-pub type Stride4 = u32;
-pub type Stride5 = u64;
-pub type Stride6 = u128;
-pub type Stride7 = U256;
-pub type Stride8 = U512;
-
-pub trait Stride:
-    Sized + Debug + Binary + Eq + PartialOrd + PartialEq + Copy
-{
-    type PtrSize;
-    const BITS: u8;
-    const STRIDE_LEN: u8;
-
-    // Get the bit position of the start of the given nibble.
-    // The nibble is defined as a `len` number of bits set from the right.
-    // bit_pos always has only one bit set in the complete array.
-    // e.g.:
-    // len: 4
-    // nibble: u16  =  0b0000 0000 0000 0111
-    // bit_pos: u16 =  0b0000 0000 0000 1000
-
-    // `<Self as Stride>::BITS`
-    // is the whole length of the bitmap, since we are shifting to the left,
-    // we have to start at the end of the bitmap.
-    // `((1 << len) - 1)`
-    // is the offset for this nibble length in the bitmap.
-    // `nibble`
-    // shifts to the right position withing the bit range for this nibble
-    // length, this follows from the fact that the `nibble` value represents
-    // *both* the bitmap part, we're considering here *and* the position
-    // relative to the nibble length offset in the bitmap.
-    fn get_bit_pos(bs: BitSpan) -> Self;
-
-    // Clear the bitmap to the right of the pointer and count the number of ones.
-    // This numbder represents the index to the corresponding prefix in the pfx_vec.
-
-    // Clearing is performed by shifting to the right until we have the nibble
-    // all the way at the right.
-
-    // `(<Self as Stride>::BITS >> 1)`
-    // The end of the bitmap (this bitmap is half the size of the pfx bitmap)
-
-    // `nibble`
-    // The bit position relative to the offset for the nibble length, this index
-    // is only used at the last (relevant) stride, so the offset is always 0.
-    fn get_pfx_index(bitmap: Self, bs: BitSpan) -> usize;
-
-    // Clear the bitmap to the right of the pointer and count the number of ones.
-    // This number represents the index to the corresponding child node in the ptr_vec.
-
-    // Clearing is performed by shifting to the right until we have the nibble
-    // all the way at the right.
-
-    // For ptrbitarr the only index we want is the one for a full-length nibble
-    // (stride length) at the last stride, so we don't need the length of the nibble
-
-    // `(<Self as Stride>::BITS >> 1)`
-    // The end of the bitmap (this bitmap is half the size of the pfx bitmap),
-    // ::BITS is the size of the pfx bitmap.
-
-    // `nibble`
-    // The bit position relative to the offset for the nibble length, this index
-    // is only used at the last (relevant) stride, so the offset is always 0.
-    fn get_ptr_index(bitmap: Self::PtrSize, nibble: u32) -> usize;
-
-    // Convert a ptrbitarr into a pfxbitarr sized bitmap,
-    // so we can do bitwise operations with a pfxbitarr sized
-    // bitmap on them.
-    // Since the last bit in the pfxbitarr isn't used, but the
-    // full ptrbitarr *is* used, the prtbitarr should be shifted
-    // one bit to the left.
-    #[allow(clippy::wrong_self_convention)]
-    fn into_stride_size(bitmap: Self::PtrSize) -> Self;
-
-    // Convert a pfxbitarr sized bitmap into a ptrbitarr sized
-    // Note that bitwise operators align bits of unsigend types with different
-    // sizes to the right, so we don't have to do anything to pad the smaller sized
-    // type. We do have to shift one bit to the left, to accomodate the unused pfxbitarr's
-    // last bit.
-    fn into_ptrbitarr_size(bitmap: Self) -> Self::PtrSize;
-
-    fn leading_zeros(self) -> u32;
-}
-
-impl_primitive_stride![3; 16; u16; u8];
-impl_primitive_stride![4; 32; u32; u16];
-impl_primitive_stride![5; 64; u64; u32];
-impl_primitive_stride![6; 128; u128; u64];
-
-impl Stride for Stride7 {
-    type PtrSize = u128;
-    const BITS: u8 = 255;
-    const STRIDE_LEN: u8 = 7;
-
-    fn get_bit_pos(bs: BitSpan) -> Self {
-        match 256 - ((1 << bs.len) - 1) as u16 - bs.bits as u16 - 1 {
-            n if n < 128 => U256(0, 1 << n),
-            n => U256(1 << (n - 128), 0),
-        }
-    }
-
-    fn get_pfx_index(bitmap: Self, bs: BitSpan) -> usize {
-        let n = 256 - ((1 << bs.len) - 1) as u16 - bs.bits as u16 - 1;
-        match n {
-            // if we move less than 128 bits to the right,
-            // all of bitmap.0 and a part of bitmap.1 will be used for counting zeros
-            // ex.
-            // ...1011_1010... >> 2 => ...0010_111010...
-            //    ____ ====                 -- --====
-            n if n < 128 => {
-                bitmap.0.count_ones() as usize
-                    + (bitmap.1 >> n).count_ones() as usize
-                    - 1
-            }
-            // if we move more than 128 bits to the right,
-            // all of bitmap.1 wil be shifted out of sight,
-            // so we only have to count bitmap.0 zeroes than (after shifting of course).
-            n => (bitmap.0 >> (n - 128)).count_ones() as usize - 1,
-        }
-    }
-
-    fn get_ptr_index(bitmap: Self::PtrSize, nibble: u32) -> usize {
-        (bitmap >> ((256 >> 1) - nibble as u16 - 1) as usize).count_ones()
-            as usize
-            - 1
-    }
-
-    fn into_stride_size(bitmap: Self::PtrSize) -> Self {
-        // One bit needs to move into the self.0 u128,
-        // since the last bit of the *whole* bitmap isn't used.
-        U256(bitmap >> 127, bitmap << 1)
-    }
-
-    fn into_ptrbitarr_size(bitmap: Self) -> Self::PtrSize {
-        // TODO expand:
-        // self.ptrbitarr =
-        // S::into_ptrbitarr_size(bit_pos | S::into_stride_size(self.ptrbitarr));
-        bitmap.0 << 127 | bitmap.1 >> 1
-    }
-
-    #[inline]
-    fn leading_zeros(self) -> u32 {
-        let lz = self.0.leading_zeros();
-        if lz == 128 {
-            lz + self.1.leading_zeros()
-        } else {
-            lz
-        }
-    }
-}
-
-impl Stride for Stride8 {
-    type PtrSize = U256;
-    const BITS: u8 = 255; // bogus
-    const STRIDE_LEN: u8 = 8;
-
-    fn get_bit_pos(bs: BitSpan) -> Self {
-        match 512 - ((1 << bs.len) - 1) as u16 - bs.bits as u16 - 1 {
-            n if n < 128 => U512(0, 0, 0, 1 << n),
-            n if n < 256 => U512(0, 0, 1 << (n - 128), 0),
-            n if n < 384 => U512(0, 1 << (n - 256), 0, 0),
-            n => U512(1 << (n - 384), 0, 0, 0),
-        }
-    }
-
-    fn get_pfx_index(bitmap: Self, bs: BitSpan) -> usize {
-        let n = 512 - ((1 << bs.len) - 1) as u16 - bs.bits as u16 - 1;
-        match n {
-            // if we move less than 128 bits to the right, all of bitmap.2
-            // and a part of bitmap.3 will be used for counting zeros.
-            // ex.
-            // ...1011_1010... >> 2 => ...0010_111010...
-            //    ____ ====                 -- --====
-            n if n < 128 => {
-                bitmap.0.count_ones() as usize
-                    + bitmap.1.count_ones() as usize
-                    + bitmap.2.count_ones() as usize
-                    + (bitmap.3 >> n).count_ones() as usize
-                    - 1
-            }
-
-            n if n < 256 => {
-                bitmap.0.count_ones() as usize
-                    + bitmap.1.count_ones() as usize
-                    + (bitmap.2 >> (n - 128)).count_ones() as usize
-                    - 1
-            }
-
-            n if n < 384 => {
-                bitmap.0.count_ones() as usize
-                    + (bitmap.1 >> (n - 256)).count_ones() as usize
-                    - 1
-            }
-
-            // if we move more than 384 bits to the right, all of bitmap.
-            // [1,2,3] will be shifted out of sight, so we only have to count
-            // bitmap.0 zeroes then (after shifting of course).
-            n => (bitmap.0 >> (n - 384)).count_ones() as usize - 1,
-        }
-    }
-
-    fn get_ptr_index(bitmap: Self::PtrSize, nibble: u32) -> usize {
-        let n = (512 >> 1) - nibble as u16 - 1;
-        match n {
-            // if we move less than 256 bits to the right, all of bitmap.0
-            // and a part of bitmap.1 will be used for counting zeros
-            // ex.
-            // ...1011_1010... >> 2 => ...0010_111010...
-            //    ____ ====                 -- --====
-            n if n < 128 => {
-                bitmap.0.count_ones() as usize
-                    + (bitmap.1 >> n).count_ones() as usize
-                    - 1
-            }
-            // if we move more than 256 bits to the right, all of bitmap.1
-            // wil be shifted out of sight, so we only have to count bitmap.0
-            // zeroes than (after) shifting of course).
-            n => (bitmap.0 >> (n - 128)).count_ones() as usize - 1,
-        }
-    }
-
-    fn into_stride_size(bitmap: Self::PtrSize) -> Self {
-        // One bit needs to move into the self.0 u128,
-        // since the last bit of the *whole* bitmap isn't used.
-        U512(
-            0,
-            bitmap.0 >> 127,
-            (bitmap.0 << 1) | (bitmap.1 >> 127),
-            bitmap.1 << 1,
-        )
-    }
-
-    fn into_ptrbitarr_size(bitmap: Self) -> Self::PtrSize {
-        // TODO expand:
-        // self.ptrbitarr =
-        // S::into_ptrbitarr_size(bit_pos | S::into_stride_size(self.ptrbitarr));
-        U256(
-            bitmap.1 << 127 | bitmap.2 >> 1,
-            bitmap.2 << 127 | bitmap.3 >> 1,
-        )
-    }
-
-    #[inline]
-    fn leading_zeros(self) -> u32 {
-        let mut lz = self.0.leading_zeros();
-        if lz == 128 {
-            lz += self.1.leading_zeros();
-            if lz == 256 {
-                lz += self.2.leading_zeros();
-                if lz == 384 {
-                    lz += self.3.leading_zeros();
-                }
-            }
-        }
-        lz
-    }
-}
diff --git a/src/synth_int.rs b/src/synth_int.rs
deleted file mode 100644
index b8601fbc..00000000
--- a/src/synth_int.rs
+++ /dev/null
@@ -1,444 +0,0 @@
-use std::cmp::Ordering;
-use std::convert::TryInto;
-use std::fmt::{Binary, Debug};
-use std::sync::atomic::AtomicU64;
-
-use crate::af::Zero;
-
-//------------ U256 synthetic integer type ----------------------------------
-
-#[derive(Copy, Clone)]
-pub struct U256(pub u128, pub u128);
-
-impl U256 {
-    pub fn to_be_bytes(self) -> [u8; 32] {
-        [self.0.to_be_bytes(), self.1.to_be_bytes()]
-            .concat()
-            .try_into()
-            .expect("U256 with incorrect length.")
-    }
-
-    pub fn from_bytes(bytes: &[u8]) -> U256 {
-        let nibble1: u128 = u128::from_be_bytes([
-            bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5],
-            bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], bytes[11],
-            bytes[12], bytes[13], bytes[14], bytes[15],
-        ]);
-        let nibble2: u128 = u128::from_be_bytes([
-            bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21],
-            bytes[22], bytes[23], bytes[24], bytes[25], bytes[26], bytes[27],
-            bytes[28], bytes[29], bytes[30], bytes[31],
-        ]);
-        U256(nibble1, nibble2)
-    }
-}
-
-impl Debug for U256 {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_fmt(format_args!(
-            "{:0128b}\n             {:0128b}",
-            self.0, self.1
-        ))
-    }
-}
-
-impl Eq for U256 {}
-
-impl std::ops::Add for U256 {
-    type Output = Self;
-
-    fn add(self, other: Self) -> Self {
-        U256(self.0.wrapping_add(other.0), self.1.wrapping_add(other.1))
-    }
-}
-
-impl Zero for U256 {
-    fn zero() -> Self {
-        U256(0_u128, 0_u128)
-    }
-    fn is_zero(&self) -> bool {
-        self.0 == 0_u128 && self.1 == 0_u128
-    }
-}
-
-impl Binary for U256 {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        std::fmt::Binary::fmt(&self, f)
-    }
-}
-
-impl PartialOrd for U256 {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for U256 {
-    fn cmp(&self, other: &Self) -> Ordering {
-        match (self.0, &other.0) {
-            (a, b) if &a > b => self.0.cmp(&other.0),
-            _ => self.1.cmp(&other.1),
-        }
-    }
-}
-
-impl std::ops::BitOr<Self> for U256 {
-    type Output = Self;
-    fn bitor(self, rhs: Self) -> Self::Output {
-        Self(self.0 | rhs.0, self.1 | rhs.1)
-    }
-}
-
-impl std::ops::BitAnd<Self> for U256 {
-    type Output = Self;
-    fn bitand(self, rhs: Self) -> Self::Output
-    where
-        Self: Eq,
-    {
-        Self(self.0 & rhs.0, self.1 & rhs.1)
-    }
-}
-
-impl PartialEq for U256 {
-    fn eq(&self, other: &Self) -> bool {
-        self.0 == other.0 && self.1 == other.1
-    }
-}
-
-//------------ U512 Synthetic Integer Type ----------------------------------
-
-#[derive(Debug, Copy, Clone)]
-pub struct U512(pub u128, pub u128, pub u128, pub u128);
-
-impl U512 {
-    pub fn to_be_bytes(self) -> [u8; 64] {
-        [
-            self.0.to_be_bytes(),
-            self.1.to_be_bytes(),
-            self.2.to_be_bytes(),
-            self.3.to_be_bytes(),
-        ]
-        .concat()
-        .try_into()
-        .expect("U512 with incorrect length.")
-    }
-
-    pub fn from_bytes(bytes: &[u8]) -> U512 {
-        let nibble1: u128 = u128::from_be_bytes([
-            bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5],
-            bytes[6], bytes[7], bytes[8], bytes[9], bytes[10], bytes[11],
-            bytes[12], bytes[13], bytes[14], bytes[15],
-        ]);
-        let nibble2: u128 = u128::from_be_bytes([
-            bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21],
-            bytes[22], bytes[23], bytes[24], bytes[25], bytes[26], bytes[27],
-            bytes[28], bytes[29], bytes[30], bytes[31],
-        ]);
-        let nibble3: u128 = u128::from_be_bytes([
-            bytes[32], bytes[33], bytes[34], bytes[35], bytes[36], bytes[37],
-            bytes[38], bytes[39], bytes[40], bytes[41], bytes[42], bytes[43],
-            bytes[44], bytes[45], bytes[46], bytes[47],
-        ]);
-        let nibble4: u128 = u128::from_be_bytes([
-            bytes[48], bytes[49], bytes[50], bytes[51], bytes[52], bytes[53],
-            bytes[54], bytes[55], bytes[56], bytes[57], bytes[58], bytes[59],
-            bytes[60], bytes[61], bytes[62], bytes[63],
-        ]);
-        U512(nibble1, nibble2, nibble3, nibble4)
-    }
-}
-
-impl PartialOrd for U512 {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        match (self.0, &other.0) {
-            (a, b) if &a > b => Some(self.0.cmp(&other.0)),
-            _ => match (self.1, &other.1) {
-                (a, b) if &a > b => Some(self.1.cmp(&other.1)),
-                _ => match (self.2, &other.2) {
-                    (a, b) if &a > b => Some(self.2.cmp(&other.2)),
-                    _ => Some(self.3.cmp(&other.3)),
-                },
-            },
-        }
-    }
-}
-
-impl PartialEq for U512 {
-    fn eq(&self, other: &Self) -> bool {
-        self.0 == other.0
-            && self.1 == other.1
-            && self.2 == other.2
-            && self.3 == other.3
-    }
-}
-
-impl Eq for U512 {}
-
-impl std::ops::Add for U512 {
-    type Output = Self;
-    fn add(self, rhs: Self) -> Self::Output {
-        Self(
-            self.0 + rhs.0,
-            self.1 + rhs.1,
-            self.2 + rhs.2,
-            self.3 + rhs.3,
-        )
-    }
-}
-
-impl Zero for U512 {
-    fn zero() -> Self {
-        U512(0_u128, 0_u128, 0_u128, 0_u128)
-    }
-    fn is_zero(&self) -> bool {
-        self.0 == 0_u128
-            && self.1 == 0_u128
-            && self.2 == 0_u128
-            && self.3 == 0_u128
-    }
-}
-
-impl Binary for U512 {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        std::fmt::Binary::fmt(&self, f)
-    }
-}
-
-impl std::ops::BitOr<Self> for U512 {
-    type Output = Self;
-    fn bitor(self, rhs: Self) -> Self::Output {
-        Self(
-            self.0 | rhs.0,
-            self.1 | rhs.1,
-            self.2 | rhs.2,
-            self.3 | rhs.3,
-        )
-    }
-}
-
-impl std::ops::BitAnd<Self> for U512 {
-    type Output = Self;
-    fn bitand(self, rhs: Self) -> Self::Output
-    where
-        Self: Eq,
-    {
-        Self(
-            self.0 & rhs.0,
-            self.1 & rhs.1,
-            self.2 & rhs.2,
-            self.3 & rhs.3,
-        )
-    }
-}
-
-//------------ Atomic U128 Synthetic Integer Type -------------------------------------
-
-#[allow(dead_code)]
-pub struct AtomicU128(pub AtomicU64, pub AtomicU64);
-
-#[allow(dead_code)]
-impl AtomicU128 {
-    pub fn new(value: u128) -> Self {
-        let (hi, lo) =
-            (((value << 64) >> 64) as u64, ((value >> 64) << 64) as u64);
-        AtomicU128(AtomicU64::new(hi), AtomicU64::new(lo))
-    }
-
-    pub fn into_be_bytes(self) -> [u8; 16] {
-        [
-            self.0.into_inner().to_be_bytes(),
-            self.1.into_inner().to_be_bytes(),
-        ]
-        .concat()
-        .try_into()
-        .expect("AtomicU128 with incorrect length.")
-    }
-
-    pub fn from_bytes(bytes: &[u8]) -> Self {
-        let hi = u64::from_be_bytes([
-            bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5],
-            bytes[6], bytes[7],
-        ]);
-        let lo = u64::from_be_bytes([
-            bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13],
-            bytes[14], bytes[15],
-        ]);
-        AtomicU128(AtomicU64::new(hi), AtomicU64::new(lo))
-    }
-}
-
-impl Debug for AtomicU128 {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_fmt(format_args!(
-            "{:016x}\n{:016x}",
-            self.0.load(std::sync::atomic::Ordering::SeqCst),
-            self.1.load(std::sync::atomic::Ordering::SeqCst)
-        ))
-    }
-}
-
-//------------ Atomic U256 Synthetic Integer Type -------------------------------------
-
-#[allow(dead_code)]
-pub struct AtomicU256(
-    pub AtomicU64,
-    pub AtomicU64,
-    pub AtomicU64,
-    pub AtomicU64,
-);
-
-#[allow(dead_code)]
-impl AtomicU256 {
-    pub fn new(value: U256) -> Self {
-        let (hihi, hilo, lohi, lolo) = (
-            ((value.0 << 64) >> 64) as u64,
-            ((value.0 >> 64) << 64) as u64,
-            ((value.1 << 64) >> 64) as u64,
-            ((value.1 >> 64) << 64) as u64,
-        );
-        AtomicU256(
-            AtomicU64::new(hihi),
-            AtomicU64::new(hilo),
-            AtomicU64::new(lohi),
-            AtomicU64::new(lolo),
-        )
-    }
-
-    pub fn into_be_bytes(self) -> [u8; 32] {
-        [
-            self.0.into_inner().to_be_bytes(),
-            self.1.into_inner().to_be_bytes(),
-            self.2.into_inner().to_be_bytes(),
-            self.3.into_inner().to_be_bytes(),
-        ]
-        .concat()
-        .try_into()
-        .expect("AtomicU256 with incorrect length.")
-    }
-
-    pub fn from_bytes(bytes: &[u8]) -> Self {
-        let hihi = u64::from_be_bytes([
-            bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5],
-            bytes[6], bytes[7],
-        ]);
-        let hilo = u64::from_be_bytes([
-            bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13],
-            bytes[14], bytes[15],
-        ]);
-        let lohi = u64::from_be_bytes([
-            bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21],
-            bytes[22], bytes[23],
-        ]);
-        let lolo = u64::from_be_bytes([
-            bytes[24], bytes[25], bytes[26], bytes[27], bytes[28], bytes[29],
-            bytes[30], bytes[31],
-        ]);
-        AtomicU256(
-            AtomicU64::new(hihi),
-            AtomicU64::new(hilo),
-            AtomicU64::new(lohi),
-            AtomicU64::new(lolo),
-        )
-    }
-}
-
-//------------ Atomic U512 Synthetic Integer Type -------------------------------------
-
-#[allow(dead_code)]
-pub struct AtomicU512(
-    pub AtomicU64,
-    pub AtomicU64,
-    pub AtomicU64,
-    pub AtomicU64,
-    pub AtomicU64,
-    pub AtomicU64,
-    pub AtomicU64,
-    pub AtomicU64,
-);
-
-#[allow(dead_code)]
-impl AtomicU512 {
-    pub fn new(value: U512) -> Self {
-        let (hihihi, hihilo, hilohi, hilolo, lohihi, lohilo, lolohi, lololo) = (
-            ((value.0 << 64) >> 64) as u64,
-            ((value.0 >> 64) << 64) as u64,
-            ((value.1 << 64) >> 64) as u64,
-            ((value.1 >> 64) << 64) as u64,
-            ((value.2 << 64) >> 64) as u64,
-            ((value.2 >> 64) << 64) as u64,
-            ((value.3 << 64) >> 64) as u64,
-            ((value.3 >> 64) << 64) as u64,
-        );
-        AtomicU512(
-            AtomicU64::new(hihihi),
-            AtomicU64::new(hihilo),
-            AtomicU64::new(hilohi),
-            AtomicU64::new(hilolo),
-            AtomicU64::new(lohihi),
-            AtomicU64::new(lohilo),
-            AtomicU64::new(lolohi),
-            AtomicU64::new(lololo),
-        )
-    }
-
-    pub fn into_be_bytes(self) -> [u8; 64] {
-        [
-            self.0.into_inner().to_be_bytes(),
-            self.1.into_inner().to_be_bytes(),
-            self.2.into_inner().to_be_bytes(),
-            self.3.into_inner().to_be_bytes(),
-            self.4.into_inner().to_be_bytes(),
-            self.5.into_inner().to_be_bytes(),
-            self.6.into_inner().to_be_bytes(),
-            self.7.into_inner().to_be_bytes(),
-        ]
-        .concat()
-        .try_into()
-        .expect("AtomicU512 with incorrect length.")
-    }
-
-    pub fn from_bytes(bytes: &[u8]) -> Self {
-        let hihihi = u64::from_be_bytes([
-            bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5],
-            bytes[6], bytes[7],
-        ]);
-        let hihilo = u64::from_be_bytes([
-            bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13],
-            bytes[14], bytes[15],
-        ]);
-        let hilohi = u64::from_be_bytes([
-            bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21],
-            bytes[22], bytes[23],
-        ]);
-        let hilolo = u64::from_be_bytes([
-            bytes[24], bytes[25], bytes[26], bytes[27], bytes[28], bytes[29],
-            bytes[30], bytes[31],
-        ]);
-        let lohihi = u64::from_be_bytes([
-            bytes[32], bytes[33], bytes[34], bytes[35], bytes[36], bytes[37],
-            bytes[38], bytes[39],
-        ]);
-        let lohilo = u64::from_be_bytes([
-            bytes[40], bytes[41], bytes[42], bytes[43], bytes[44], bytes[45],
-            bytes[46], bytes[47],
-        ]);
-        let lolohi = u64::from_be_bytes([
-            bytes[48], bytes[49], bytes[50], bytes[51], bytes[52], bytes[53],
-            bytes[54], bytes[55],
-        ]);
-        let lololo = u64::from_be_bytes([
-            bytes[56], bytes[57], bytes[58], bytes[59], bytes[60], bytes[61],
-            bytes[62], bytes[63],
-        ]);
-        AtomicU512(
-            AtomicU64::new(hihihi),
-            AtomicU64::new(hihilo),
-            AtomicU64::new(hilohi),
-            AtomicU64::new(hilolo),
-            AtomicU64::new(lohihi),
-            AtomicU64::new(lohilo),
-            AtomicU64::new(lolohi),
-            AtomicU64::new(lololo),
-        )
-    }
-}

From a90db9fa9ca4a5f6a1952edc98114cb4a22466ff Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 28 Feb 2025 14:04:52 +0100
Subject: [PATCH 093/147] use multithreaded store for examples

---
 examples/exact_matches_single.rs  | 18 ++++++++++++------
 examples/real_single_thread_24.rs | 22 +++++++++++++++-------
 2 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/examples/exact_matches_single.rs b/examples/exact_matches_single.rs
index d2fd69dd..25fe86ee 100644
--- a/examples/exact_matches_single.rs
+++ b/examples/exact_matches_single.rs
@@ -1,11 +1,12 @@
+use multi::MemoryOnlyConfig;
+use multi::Record;
 use rotonda_store::meta_examples::NoMeta;
 use rotonda_store::prelude::*;
-use rotonda_store::SingleThreadedStore;
+use rotonda_store::MultiThreadedStore;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let v4 = vec![8];
-    let v6 = vec![8];
-    let mut tree_bitmap = SingleThreadedStore::<NoMeta>::new(v4, v6);
+    let tree_bitmap =
+        MultiThreadedStore::<NoMeta, MemoryOnlyConfig>::try_default()?;
 
     let pfxs = vec![
         Prefix::new_relaxed(
@@ -260,7 +261,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     for pfx in pfxs.into_iter() {
         println!("insert {}", pfx?);
         // let p : rotonda_store::Prefix<u32, PrefixAs> = pfx.into();
-        tree_bitmap.insert(&pfx.unwrap(), NoMeta::Empty)?;
+        tree_bitmap.insert(
+            &pfx.unwrap(),
+            Record::new(1, 0, multi::RouteStatus::Active, NoMeta::Empty),
+            None,
+        )?;
     }
     println!("------ end of inserts\n");
     // println!(
@@ -336,7 +341,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         // Prefix::new_relaxed(std::net::Ipv4Addr::new(1, 0, 128, 0).into(), 24),
     ] {
         println!("search for: {:?}", spfx);
-        // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
+        let guard = &rotonda_store::epoch::pin();
         let s_spfx = tree_bitmap.match_prefix(
             // (&locks.0, &locks.1),
             &spfx.unwrap(),
@@ -348,6 +353,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 mui: None,
                 include_history: IncludeHistory::None,
             },
+            guard,
         );
         println!("exact match: {:?}", s_spfx);
         println!("-----------");
diff --git a/examples/real_single_thread_24.rs b/examples/real_single_thread_24.rs
index 2ea23533..996ec206 100644
--- a/examples/real_single_thread_24.rs
+++ b/examples/real_single_thread_24.rs
@@ -1,4 +1,5 @@
 use log::trace;
+use multi::{MemoryOnlyConfig, Record, RouteStatus};
 use std::thread;
 use std::time::Duration;
 
@@ -13,10 +14,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting one-threaded yolo testing....");
-    let v4 = vec![8];
-    let v6 = vec![8];
-    let mut tree_bitmap =
-        rotonda_store::SingleThreadedStore::<PrefixAs>::new(v4, v6);
+    let tree_bitmap = rotonda_store::MultiThreadedStore::<
+        PrefixAs,
+        MemoryOnlyConfig,
+    >::try_default()?;
 
     let mut pfx_int = 0_u32;
 
@@ -36,9 +37,16 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
                 print!("{}-", pfx_int);
                 let asn: u32 = rng.random();
-                match tree_bitmap
-                    .insert(&pfx.unwrap(), PrefixAs::new_from_u32(asn))
-                {
+                match tree_bitmap.insert(
+                    &pfx.unwrap(),
+                    Record::new(
+                        1,
+                        0,
+                        RouteStatus::Active,
+                        PrefixAs::new_from_u32(asn),
+                    ),
+                    None,
+                ) {
                     Ok(_) => {}
                     Err(e) => {
                         println!("{}", e);

From bbe4f3678a2a2fec2ebc2677c01589caaf1aea36 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 28 Feb 2025 15:01:12 +0100
Subject: [PATCH 094/147] sanitize StrideNodeId

---
 proc_macros/src/lib.rs                 |  6 +-
 src/local_array/in_memory/iterators.rs |  6 +-
 src/local_array/in_memory/node.rs      | 27 +++++----
 src/local_array/in_memory/tree.rs      | 80 ++++++++++++--------------
 src/local_array/tests.rs               | 22 +++----
 5 files changed, 67 insertions(+), 74 deletions(-)

diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
index 58c4ef19..93edeb8d 100644
--- a/proc_macros/src/lib.rs
+++ b/proc_macros/src/lib.rs
@@ -272,13 +272,13 @@ pub fn stride_sizes(
             }
 
             fn get_store(&self, id: StrideNodeId<#ip_af>) -> &NodeSet<#ip_af> {
-                match id.get_id().1 as usize {
+                match id.len() as usize {
                     #( #strides_len4 => &self.#strides_len4_l, )*
                     // ex.:
                     // 10 => &self.l10,
                     _ => panic!(
                         "unexpected sub prefix length {} in stride size 4 ({})",
-                        id.get_id().1,
+                        id.len(),
                         id
                     ),
                 }
@@ -291,7 +291,7 @@ pub fn stride_sizes(
 
             #[inline]
             fn get_stride_for_id(&self, id: StrideNodeId<#ip_af>) -> u8 {
-                [ #(#len_to_stride_arr, )* ][id.get_id().1 as usize]
+                [ #(#len_to_stride_arr, )* ][id.len() as usize]
             }
 
             #[inline]
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 3acec37b..60dd8802 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -286,9 +286,9 @@ impl<
             );
             trace!(
                 "start node id   {:032b} (bits {} len {})",
-                start_node_id.get_id().0,
-                start_node_id.get_id().0,
-                start_node_id.get_len()
+                start_node_id.bits(),
+                start_node_id.bits(),
+                start_node_id.len()
             );
             trace!(
                 "start pfx bit span  {:08b} {} len {}",
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index c58dc450..3e6c1d84 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -592,7 +592,7 @@ pub(crate) enum NewNodeOrIndex<AF: AddressFamily> {
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
 pub struct StrideNodeId<AF: AddressFamily> {
-    addr_bits: AF,
+    bits: AF,
     len: u8,
 }
 
@@ -601,23 +601,26 @@ impl<AF: AddressFamily> StrideNodeId<AF> {
         addr_bits: AF,
         len: u8,
     ) -> Self {
-        Self { addr_bits, len }
+        Self {
+            bits: addr_bits,
+            len,
+        }
     }
 
     #[inline]
     pub(crate) fn new_with_cleaned_id(addr_bits: AF, len: u8) -> Self {
         Self {
-            addr_bits: addr_bits.truncate_to_len(len),
+            bits: addr_bits.truncate_to_len(len),
             len,
         }
     }
 
-    pub fn get_id(&self) -> (AF, u8) {
-        (self.addr_bits, self.len)
+    pub(crate) fn len(&self) -> u8 {
+        self.len
     }
 
-    pub(crate) fn get_len(&self) -> u8 {
-        self.len
+    pub(crate) fn bits(&self) -> AF {
+        self.bits
     }
 
     pub fn set_len(mut self, len: u8) -> Self {
@@ -632,14 +635,14 @@ impl<AF: AddressFamily> StrideNodeId<AF> {
 
     #[inline]
     pub(crate) fn truncate_to_len(self) -> Self {
-        StrideNodeId::new_with_cleaned_id(self.addr_bits, self.len)
+        StrideNodeId::new_with_cleaned_id(self.bits, self.len)
     }
 
     // clean out all bits that are set beyond the len. This function should
     // be used before doing any ORing to add a nibble.
     #[inline]
     pub(crate) fn with_cleaned_id(&self) -> (AF, u8) {
-        (self.addr_bits.truncate_to_len(self.len), self.len)
+        (self.bits.truncate_to_len(self.len), self.len)
     }
 
     pub(crate) fn add_bit_span(&self, bs: BitSpan) -> Self {
@@ -651,7 +654,7 @@ impl<AF: AddressFamily> StrideNodeId<AF> {
 
 impl<AF: AddressFamily> std::fmt::Display for StrideNodeId<AF> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}-{}", self.addr_bits, self.len)
+        write!(f, "{}-{}", self.bits, self.len)
     }
 }
 
@@ -659,14 +662,14 @@ impl<AF: AddressFamily> std::convert::From<StrideNodeId<AF>>
     for PrefixId<AF>
 {
     fn from(id: StrideNodeId<AF>) -> Self {
-        PrefixId::new(id.addr_bits, id.len)
+        PrefixId::new(id.bits, id.len)
     }
 }
 
 impl<AF: AddressFamily> From<(AF, u8)> for StrideNodeId<AF> {
     fn from(value: (AF, u8)) -> Self {
         StrideNodeId {
-            addr_bits: value.0,
+            bits: value.0,
             len: value.1,
         }
     }
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index f50aef3a..c76d5e56 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -337,8 +337,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                                 && local_retry_count > 0
                             {
                                 trace!(
-                                    "{} contention: Node \
-                                                 already exists {}",
+                                    "{} contention: Node already exists {}",
                                     std::thread::current()
                                         .name()
                                         .unwrap_or("unnamed-thread"),
@@ -384,7 +383,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                     if log_enabled!(log::Level::Error) {
                         error!(
                             "{} failing to store (intermediate) node {}.
-                             Giving up this node. This shouldn't happen!",
+Giving up this node. This shouldn't happen!",
                             std::thread::current()
                                 .name()
                                 .unwrap_or("unnamed-thread"),
@@ -508,12 +507,15 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         let mut retry_count = 0;
 
         loop {
-            let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
-                id.get_id().1,
-                level,
-            );
-            trace!("{:032b}", id.get_id().0);
-            trace!("id {:?}", id.get_id());
+            let this_level =
+                <NB as NodeBuckets<AF>>::len_to_store_bits(id.len(), level);
+
+            // if this_level > (id.len() / 4) {
+            //     return Err(PrefixStoreError::StoreNotReadyError);
+            // }
+
+            trace!("{:032b}", id.len());
+            trace!("id {:?}", id);
             trace!("multi_uniq_id {}", multi_uniq_id);
 
             // HASHING FUNCTION
@@ -524,16 +526,16 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                     // No node exists, so we create one here.
                     let next_level =
                         <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            id.get_id().1,
+                            id.len(),
                             level + 1,
                         );
 
                     if log_enabled!(log::Level::Trace) {
                         trace!(
-                            "Empty node found,
-                            creating new node {} len{} lvl{}",
+                            "Empty node found,creating new node {} len{}
+lvl{}",
                             id,
-                            id.get_id().1,
+                            id.len(),
                             level + 1
                         );
                         trace!("Next level {}", next_level);
@@ -548,7 +550,10 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                     }
 
                     trace!("multi uniq id {}", multi_uniq_id);
+                    trace!("this level {}", this_level);
+                    trace!("next level {}", next_level);
 
+                    debug_assert!(next_level >= this_level);
                     let node_set = NodeSet::init(next_level - this_level);
 
                     let ptrbitarr = new_node.ptrbitarr.load();
@@ -592,13 +597,10 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                                 .unwrap_or("unnamed-thread"),
                             stored_node.node_id
                         );
-                        trace!("node_id {:?}", stored_node.node_id.get_id());
-                        trace!(
-                            "node_id {:032b}",
-                            stored_node.node_id.get_id().0
-                        );
+                        trace!("node_id {:?}", stored_node.node_id);
+                        trace!("node_id {:032b}", stored_node.node_id.bits());
                         trace!("id {}", id);
-                        trace!("     id {:032b}", id.get_id().0);
+                        trace!("     id {:032b}", id.bits());
                     }
 
                     // See if somebody beat us to creating our
@@ -635,13 +637,13 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                                  index {}",
                             stored_node.node_id,
                             id,
-                            id.get_id().1,
+                            id.len(),
                             level,
                             index
                         );
 
                         match <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            id.get_id().1,
+                            id.len(),
                             level,
                         ) {
                             // on to the next level!
@@ -808,12 +810,12 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                 None => {
                     let this_level =
                         <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            id.get_id().1,
+                            id.len(),
                             level,
                         );
                     let next_level =
                         <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            id.get_id().1,
+                            id.len(),
                             level + 1,
                         );
                     let node_set = NodeSet::init(next_level - this_level);
@@ -863,10 +865,8 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match <NB as NodeBuckets<AF>>::len_to_store_bits(
-                id.get_id().1,
-                level,
-            ) {
+            match <NB as NodeBuckets<AF>>::len_to_store_bits(id.len(), level)
+            {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -898,12 +898,12 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                 None => {
                     let this_level =
                         <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            id.get_id().1,
+                            id.len(),
                             level,
                         );
                     let next_level =
                         <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            id.get_id().1,
+                            id.len(),
                             level + 1,
                         );
                     let node_set = NodeSet::init(next_level - this_level);
@@ -951,10 +951,8 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match <NB as NodeBuckets<AF>>::len_to_store_bits(
-                id.get_id().1,
-                level,
-            ) {
+            match <NB as NodeBuckets<AF>>::len_to_store_bits(id.len(), level)
+            {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -986,12 +984,12 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
                 None => {
                     let this_level =
                         <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            id.get_id().1,
+                            id.len(),
                             level,
                         );
                     let next_level =
                         <NB as NodeBuckets<AF>>::len_to_store_bits(
-                            id.get_id().1,
+                            id.len(),
                             level + 1,
                         );
                     let node_set = NodeSet::init(next_level - this_level);
@@ -1049,10 +1047,8 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match <NB as NodeBuckets<AF>>::len_to_store_bits(
-                id.get_id().1,
-                level,
-            ) {
+            match <NB as NodeBuckets<AF>>::len_to_store_bits(id.len(), level)
+            {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -1179,11 +1175,11 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
     pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
         // And, this is all of our hashing function.
         let last_level = if level > 0 {
-            <NB>::len_to_store_bits(id.get_id().1, level - 1)
+            <NB>::len_to_store_bits(id.len(), level - 1)
         } else {
             0
         };
-        let this_level = <NB>::len_to_store_bits(id.get_id().1, level);
+        let this_level = <NB>::len_to_store_bits(id.len(), level);
         // trace!("bits division {}", this_level);
         // trace!(
         //     "calculated index ({} << {}) >> {}",
@@ -1192,7 +1188,7 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
         //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
         // );
         // HASHING FUNCTION
-        ((id.get_id().0 << AF::from_u8(last_level))
+        ((id.bits() << AF::from_u8(last_level))
             >> AF::from_u8(
                 (<AF>::BITS - (this_level - last_level)) % <AF>::BITS,
             ))
diff --git a/src/local_array/tests.rs b/src/local_array/tests.rs
index d494dcf5..646e2206 100644
--- a/src/local_array/tests.rs
+++ b/src/local_array/tests.rs
@@ -14,22 +14,18 @@ fn test_af_1() -> Result<(), Box<dyn Error>> {
     let base_prefix =
         StrideNodeId::dangerously_new_with_id_as_is(bit_addr, 32);
 
-    assert_eq!(base_prefix.get_id().0, bit_addr);
-    assert_eq!(
-        base_prefix.truncate_to_len().get_id().0,
-        base_prefix.get_id().0
-    );
+    assert_eq!(base_prefix.bits(), bit_addr);
+    assert_eq!(base_prefix.truncate_to_len().bits(), base_prefix.bits());
     assert_eq!(
         StrideNodeId::dangerously_new_with_id_as_is(
-            base_prefix.get_id().0.truncate_to_len(28),
+            base_prefix.bits().truncate_to_len(28),
             28
         )
         .add_bit_span(BitSpan {
             bits: 0b0101,
             len: 4
         })
-        .get_id()
-        .0,
+        .bits(),
         0b1111_1111_1111_1111_1111_1111_1111_0101
     );
 
@@ -45,9 +41,9 @@ fn test_af_2() -> Result<(), Box<dyn Error>> {
     let bit_addr: IPv4 = 0b1111_1111_1111_1111_1111_1111_1111_1111.into();
     let nu_prefix = StrideNodeId::dangerously_new_with_id_as_is(bit_addr, 8);
 
-    assert_eq!(nu_prefix.get_id().0, bit_addr);
+    assert_eq!(nu_prefix.bits(), bit_addr);
     assert_eq!(
-        nu_prefix.truncate_to_len().get_id().0,
+        nu_prefix.truncate_to_len().bits(),
         0b1111_1111_0000_0000_0000_0000_0000_0000
     );
 
@@ -57,8 +53,7 @@ fn test_af_2() -> Result<(), Box<dyn Error>> {
                 bits: 0b1010,
                 len: 4
             })
-            .get_id()
-            .0,
+            .bits(),
         0b1111_1111_1010_0000_0000_0000_0000_0000
     );
     assert_eq!(
@@ -68,8 +63,7 @@ fn test_af_2() -> Result<(), Box<dyn Error>> {
                 bits: 0b1010,
                 len: 4
             })
-            .get_id()
-            .0,
+            .bits(),
         0b1111_1111_1010_0000_0000_0000_0000_0000
     );
 

From 75365932a2ec446f3b9a31e18521fb6533432f37 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 28 Feb 2025 17:21:21 +0100
Subject: [PATCH 095/147] use saturating_sub instead of operator for
 Node|PrefixSet init

---
 src/local_array/in_memory/atomic_stride.rs |  4 ++--
 src/local_array/in_memory/atomic_types.rs  |  2 +-
 src/local_array/in_memory/tree.rs          | 15 ++++++++++-----
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index ca296f53..5ef4b5b2 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -122,13 +122,13 @@ pub(crate) fn into_pfxbitarr(bitmap: u16) -> u32 {
 }
 
 pub(crate) fn bit_pos_from_index(i: u8) -> u32 {
-    <u32>::try_from(1).unwrap().rotate_right(1) >> i
+    1_u32.rotate_right(1) >> i
 }
 
 pub(crate) fn ptr_bit_pos_from_index(i: u8) -> u16 {
     // trace!("pfx {} ptr {} strlen {}",
     // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
-    <u16>::try_from(1).unwrap().rotate_right(1) >> (i + 1)
+    1_u16.rotate_right(1) >> (i + 1)
 }
 
 impl AtomicBitmap for AtomicPfxBitArr {
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 5059d088..1720c743 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -160,7 +160,7 @@ impl<AF: AddressFamily, M: crate::prefix_record::Meta> StoredPrefix<AF, M> {
                 std::thread::current().name().unwrap_or("unnamed-thread"),
                 pfx_id.get_len()
             );
-            PrefixSet::init(next_level - this_level)
+            PrefixSet::init(next_level.saturating_sub(this_level))
         };
         // End of calculation
 
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index c76d5e56..aaa4e7ad 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -553,8 +553,10 @@ lvl{}",
                     trace!("this level {}", this_level);
                     trace!("next level {}", next_level);
 
-                    debug_assert!(next_level >= this_level);
-                    let node_set = NodeSet::init(next_level - this_level);
+                    // A weird trick to create either a NodeSet with 16 nodes,
+                    // or one without any (for the last stride)
+                    let node_set =
+                        NodeSet::init(next_level.saturating_sub(this_level));
 
                     let ptrbitarr = new_node.ptrbitarr.load();
                     let pfxbitarr = new_node.pfxbitarr.load();
@@ -818,7 +820,8 @@ lvl{}",
                             id.len(),
                             level + 1,
                         );
-                    let node_set = NodeSet::init(next_level - this_level);
+                    let node_set =
+                        NodeSet::init(next_level.saturating_sub(this_level));
 
                     // See if we can create the node
                     (node, _) =
@@ -906,7 +909,8 @@ lvl{}",
                             id.len(),
                             level + 1,
                         );
-                    let node_set = NodeSet::init(next_level - this_level);
+                    let node_set =
+                        NodeSet::init(next_level.saturating_sub(this_level));
 
                     // See if we can create the node
                     (node, _) =
@@ -992,7 +996,8 @@ lvl{}",
                             id.len(),
                             level + 1,
                         );
-                    let node_set = NodeSet::init(next_level - this_level);
+                    let node_set =
+                        NodeSet::init(next_level.saturating_sub(this_level));
 
                     // See if we can create the node
                     (node, _) =

From 915c3e82188a9537f2906f5687da4d3bff32f858 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 28 Feb 2025 17:59:53 +0100
Subject: [PATCH 096/147] fix debug test overflows on subtract and sh{r|l}

---
 src/af.rs                                  | 6 ++++--
 src/local_array/bit_span.rs                | 5 ++++-
 src/local_array/in_memory/atomic_stride.rs | 3 ++-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/af.rs b/src/af.rs
index 3cefab94..79ffeeb6 100644
--- a/src/af.rs
+++ b/src/af.rs
@@ -1,3 +1,4 @@
+use log::trace;
 use zerocopy::{IntoBytes, NetworkEndian, U128, U32};
 
 use crate::local_array::bit_span::BitSpan;
@@ -208,7 +209,8 @@ impl AddressFamily for IPv4 {
     }
 
     fn checked_shr_or_zero(self, rhs: u32) -> Self {
-        if rhs == 0 {
+        trace!("CHECKED_SHR_OR_ZERO {} >> {}", u32::from(self), rhs);
+        if rhs == 0 || rhs == 32 {
             return 0.into();
         }
         self >> U32::<NetworkEndian>::from(rhs)
@@ -351,7 +353,7 @@ impl AddressFamily for IPv6 {
     // }
 
     fn checked_shr_or_zero(self, rhs: u32) -> Self {
-        if rhs == 0 {
+        if rhs == 0 || rhs == 128 {
             return U128::from(0);
         };
 
diff --git a/src/local_array/bit_span.rs b/src/local_array/bit_span.rs
index c8038f9f..2a3dc2a6 100644
--- a/src/local_array/bit_span.rs
+++ b/src/local_array/bit_span.rs
@@ -28,7 +28,10 @@ impl BitSpan {
     }
 
     pub(crate) fn check(&self) -> bool {
-        // println!("check bit span: {:?}", self);
+        println!("check bit span: {:?}", self);
+        if self.len == 0 && self.bits == 0 {
+            return true;
+        };
         self.len < 5
             && self.bits < 16
             && (self.bits << (32 - self.len)) >> (32 - self.len) == self.bits
diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/local_array/in_memory/atomic_stride.rs
index 5ef4b5b2..98529109 100644
--- a/src/local_array/in_memory/atomic_stride.rs
+++ b/src/local_array/in_memory/atomic_stride.rs
@@ -128,7 +128,8 @@ pub(crate) fn bit_pos_from_index(i: u8) -> u32 {
 pub(crate) fn ptr_bit_pos_from_index(i: u8) -> u16 {
     // trace!("pfx {} ptr {} strlen {}",
     // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
-    1_u16.rotate_right(1) >> (i + 1)
+    trace!("PTR_BIT_POS_FROM_INDEX {i}");
+    1_u16.rotate_right(i as u32 + 2)
 }
 
 impl AtomicBitmap for AtomicPfxBitArr {

From 191162c8659553bfdb35e791b12aab1fffafdc15 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Mon, 3 Mar 2025 15:39:07 +0100
Subject: [PATCH 097/147] remove proc_macro, and create concrete store struct

---
 Cargo.toml                                |   2 +-
 src/local_array/in_memory/atomic_types.rs |  31 +-
 src/local_array/in_memory/iterators.rs    |  22 +-
 src/local_array/in_memory/query.rs        |   4 +-
 src/local_array/in_memory/tree.rs         | 102 +--
 src/local_array/prefix_cht/cht.rs         |  20 +-
 src/local_array/prefix_cht/iterators.rs   |   7 +-
 src/local_array/query.rs                  |   9 +-
 src/local_array/rib/default_store.rs      | 778 +++++++++++++++++++++-
 src/local_array/rib/rib.rs                |  21 +-
 src/prelude/mod.rs                        |   5 +-
 11 files changed, 876 insertions(+), 125 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 95262c6c..e9e1d1bd 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -30,7 +30,7 @@ parking_lot_core = "0.9.10"
 inetnum   = "0.1"
 log       = "^0.4"
 roaring 	= "0.10.3"
-rotonda-macros = { path = "proc_macros", version = "0.4.0" }
+# rotonda-macros = { path = "proc_macros", version = "0.4.0" }
 routecore       = { git = "https://github.com/nlnetlabs/routecore", branch = "dev", version = "0.5.2-dev", features = ["bgp", "bmp", "fsm", "serde", "mrt"] }
 ansi_term   = { version = "0.12", optional = true }
 csv         = { version = "1", optional = true }
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index 1720c743..f9a8618a 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -135,15 +135,15 @@ pub struct StoredPrefix<AF: AddressFamily, M: crate::prefix_record::Meta> {
 }
 
 impl<AF: AddressFamily, M: crate::prefix_record::Meta> StoredPrefix<AF, M> {
-    pub(crate) fn new<PB: PrefixBuckets<AF, M>>(
+    pub(crate) fn new<PB: FamilyCHT<AF, PrefixSet<AF, M>>>(
         pfx_id: PrefixId<AF>,
         level: u8,
     ) -> Self {
         // start calculation size of next set, it's dependent on the level
         // we're in.
         // let pfx_id = PrefixId::new(record.net, record.len);
-        let this_level = PB::get_bits_for_len(pfx_id.get_len(), level);
-        let next_level = PB::get_bits_for_len(pfx_id.get_len(), level + 1);
+        let this_level = PB::bits_for_len(pfx_id.get_len(), level);
+        let next_level = PB::bits_for_len(pfx_id.get_len(), level + 1);
 
         trace!("this level {} next level {}", this_level, next_level);
         let next_bucket: PrefixSet<AF, M> = if next_level > 0 {
@@ -565,30 +565,11 @@ impl<M: Meta> Clone for MultiMap<M> {
 }
 
 // ----------- FamilyBuckets Trait ------------------------------------------
-//
-// Implementations of this trait are done by a proc-macro called
-// `stride_sizes`from the `rotonda-macros` crate.
 
-pub trait NodeBuckets<AF: AddressFamily> {
+pub trait FamilyCHT<AF: AddressFamily, V> {
     fn init() -> Self;
-    fn len_to_store_bits(len: u8, level: u8) -> u8;
-    fn get_stride_sizes(&self) -> &[u8];
-    fn get_stride_for_id(&self, id: StrideNodeId<AF>) -> u8;
-    // fn get_store3(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride3>;
-    fn get_store(&self, id: StrideNodeId<AF>) -> &NodeSet<AF>;
-    // fn get_store5(&self, id: StrideNodeId<AF>) -> &NodeSet<AF, Stride5>;
-    fn get_strides_len() -> u8;
-    // fn get_first_stride_size() -> u8;
-}
-
-pub trait PrefixBuckets<AF: AddressFamily, M: Meta>
-where
-    Self: Sized,
-{
-    fn init() -> Self;
-    fn remove(&mut self, id: PrefixId<AF>) -> Option<M>;
-    fn get_root_prefix_set(&self, len: u8) -> &'_ PrefixSet<AF, M>;
-    fn get_bits_for_len(len: u8, level: u8) -> u8;
+    fn bits_for_len(len: u8, level: u8) -> u8;
+    fn root_for_len(&self, len: u8) -> &V;
 }
 
 //------------ PrefixSet ----------------------------------------------------
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 60dd8802..9c1b849f 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -19,7 +19,7 @@
 // contention, since every lookup has to go through the levels near the root
 // in the TreeBitMap.
 
-use crate::local_array::in_memory::atomic_types::NodeBuckets;
+use crate::local_array::in_memory::atomic_types::FamilyCHT;
 use crate::local_array::in_memory::tree::TreeBitMap;
 use crate::{
     af::AddressFamily,
@@ -35,6 +35,8 @@ use crate::{
 use inetnum::addr::Prefix;
 use log::{log_enabled, trace};
 
+use super::atomic_types::NodeSet;
+
 // ----------- MoreSpecificPrefixIter ------------------------------------
 
 // A iterator over all the more-specifics for a given prefix.
@@ -56,7 +58,7 @@ use log::{log_enabled, trace};
 pub(crate) struct MoreSpecificPrefixIter<
     'a,
     AF: AddressFamily,
-    NB: NodeBuckets<AF>,
+    NB: FamilyCHT<AF, NodeSet<AF>>,
 > {
     tree: &'a TreeBitMap<AF, NB>,
     cur_ptr_iter: NodeMoreSpecificChildIter<AF>,
@@ -64,7 +66,7 @@ pub(crate) struct MoreSpecificPrefixIter<
     parent_and_position: Vec<NodeMoreSpecificChildIter<AF>>,
 }
 
-impl<'a, AF: AddressFamily + 'a, NB: NodeBuckets<AF>> Iterator
+impl<'a, AF: AddressFamily + 'a, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
     for MoreSpecificPrefixIter<'a, AF, NB>
 {
     type Item = PrefixId<AF>;
@@ -178,12 +180,16 @@ impl<'a, AF: AddressFamily + 'a, NB: NodeBuckets<AF>> Iterator
     }
 }
 
-pub(crate) struct LMPrefixIter<'a, AF: AddressFamily, NB: NodeBuckets<AF>> {
+pub(crate) struct LMPrefixIter<
+    'a,
+    AF: AddressFamily,
+    NB: FamilyCHT<AF, NodeSet<AF>>,
+> {
     tree: &'a TreeBitMap<AF, NB>,
     prefix: PrefixId<AF>,
 }
 
-impl<AF: AddressFamily, NB: NodeBuckets<AF>> Iterator
+impl<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
     for LMPrefixIter<'_, AF, NB>
 {
     type Item = PrefixId<AF>;
@@ -215,14 +221,14 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> Iterator
 pub(crate) struct LessSpecificPrefixIter<
     'a,
     AF: AddressFamily,
-    NB: NodeBuckets<AF>,
+    NB: FamilyCHT<AF, NodeSet<AF>>,
 > {
     tree: &'a TreeBitMap<AF, NB>,
     prefix: PrefixId<AF>,
     cur_level: u8,
 }
 
-impl<AF: AddressFamily, NB: NodeBuckets<AF>> Iterator
+impl<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
     for LessSpecificPrefixIter<'_, AF, NB>
 {
     type Item = PrefixId<AF>;
@@ -259,7 +265,7 @@ impl<
         'a,
         AF: AddressFamily,
         // M: crate::prefix_record::Meta,
-        NB: NodeBuckets<AF>,
+        NB: FamilyCHT<AF, NodeSet<AF>>,
         // PB: PrefixBuckets<AF, M>,
     > TreeBitMap<AF, NB>
 {
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index c9a91956..fa2b2757 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -4,13 +4,13 @@ use crate::rib::query::TreeQueryResult;
 use crate::{MatchOptions, MatchType};
 
 use super::super::types::PrefixId;
-use super::atomic_types::NodeBuckets;
+use super::atomic_types::{FamilyCHT, NodeSet};
 use super::tree::TreeBitMap;
 
 impl<AF, NB> TreeBitMap<AF, NB>
 where
     AF: AddressFamily,
-    NB: NodeBuckets<AF>,
+    NB: FamilyCHT<AF, NodeSet<AF>>,
 {
     pub(crate) fn match_prefix(
         &self,
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index aaa4e7ad..2046c152 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -194,7 +194,7 @@ use roaring::RoaringBitmap;
 use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering};
 use std::{fmt::Debug, marker::PhantomData};
 
-use super::atomic_types::NodeBuckets;
+use super::atomic_types::FamilyCHT;
 use crate::af::AddressFamily;
 use crate::rib::Counters;
 
@@ -210,7 +210,7 @@ use ansi_term::Colour;
 //--------------------- TreeBitMap ------------------------------------------
 
 #[derive(Debug)]
-pub struct TreeBitMap<AF: AddressFamily, NB: NodeBuckets<AF>> {
+pub struct TreeBitMap<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> {
     pub(crate) node_buckets: NB,
     pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     counters: Counters,
@@ -218,10 +218,10 @@ pub struct TreeBitMap<AF: AddressFamily, NB: NodeBuckets<AF>> {
     _af: PhantomData<AF>,
 }
 
-impl<AF: AddressFamily, NB: NodeBuckets<AF>> TreeBitMap<AF, NB> {
+impl<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> TreeBitMap<AF, NB> {
     pub(crate) fn new() -> Result<Self, Box<dyn std::error::Error>> {
         let tree_bitmap = Self {
-            node_buckets: NodeBuckets::init(),
+            node_buckets: FamilyCHT::init(),
             withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
             default_route_exists: AtomicBool::new(false),
@@ -466,7 +466,7 @@ Giving up this node. This shouldn't happen!",
             self.retrieve_node_mut(self.get_root_node_id(), mui)
         {
             self.node_buckets
-                .get_store(self.get_root_node_id())
+                .root_for_len(self.get_root_node_id().len())
                 .update_rbm_index(mui)
         } else {
             Err(PrefixStoreError::StoreNotReadyError)
@@ -501,14 +501,16 @@ Giving up this node. This shouldn't happen!",
         }
         self.counters.inc_nodes_count();
 
-        let mut nodes = self.node_buckets.get_store(id);
+        let mut nodes = self.node_buckets.root_for_len(id.len());
         let new_node = next_node;
         let mut level = 0;
         let mut retry_count = 0;
 
         loop {
-            let this_level =
-                <NB as NodeBuckets<AF>>::len_to_store_bits(id.len(), level);
+            let this_level = <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
+                id.len(),
+                level,
+            );
 
             // if this_level > (id.len() / 4) {
             //     return Err(PrefixStoreError::StoreNotReadyError);
@@ -525,7 +527,7 @@ Giving up this node. This shouldn't happen!",
                 None => {
                     // No node exists, so we create one here.
                     let next_level =
-                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
                             id.len(),
                             level + 1,
                         );
@@ -644,7 +646,7 @@ lvl{}",
                             index
                         );
 
-                        match <NB as NodeBuckets<AF>>::len_to_store_bits(
+                        match <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
                             id.len(),
                             level,
                         ) {
@@ -798,7 +800,7 @@ lvl{}",
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
-        let mut nodes = self.node_buckets.get_store(id);
+        let mut nodes = self.node_buckets.root_for_len(id.len());
 
         loop {
             let index = Self::hash_node_id(id, level);
@@ -811,12 +813,12 @@ lvl{}",
                 // empty node in the store.
                 None => {
                     let this_level =
-                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
                             id.len(),
                             level,
                         );
                     let next_level =
-                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
                             id.len(),
                             level + 1,
                         );
@@ -868,8 +870,10 @@ lvl{}",
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match <NB as NodeBuckets<AF>>::len_to_store_bits(id.len(), level)
-            {
+            match <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
+                id.len(),
+                level,
+            ) {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -887,7 +891,7 @@ lvl{}",
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
-        let mut nodes = self.node_buckets.get_store(id);
+        let mut nodes = self.node_buckets.root_for_len(id.len());
 
         loop {
             let index = Self::hash_node_id(id, level);
@@ -900,12 +904,12 @@ lvl{}",
                 // empty node in the store.
                 None => {
                     let this_level =
-                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
                             id.len(),
                             level,
                         );
                     let next_level =
-                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
                             id.len(),
                             level + 1,
                         );
@@ -955,8 +959,10 @@ lvl{}",
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match <NB as NodeBuckets<AF>>::len_to_store_bits(id.len(), level)
-            {
+            match <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
+                id.len(),
+                level,
+            ) {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -974,7 +980,7 @@ lvl{}",
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
-        let mut nodes = self.node_buckets.get_store(id);
+        let mut nodes = self.node_buckets.root_for_len(id.len());
 
         loop {
             let index = Self::hash_node_id(id, level);
@@ -987,12 +993,12 @@ lvl{}",
                 // empty node in the store.
                 None => {
                     let this_level =
-                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
                             id.len(),
                             level,
                         );
                     let next_level =
-                        <NB as NodeBuckets<AF>>::len_to_store_bits(
+                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
                             id.len(),
                             level + 1,
                         );
@@ -1052,8 +1058,10 @@ lvl{}",
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match <NB as NodeBuckets<AF>>::len_to_store_bits(id.len(), level)
-            {
+            match <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
+                id.len(),
+                level,
+            ) {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -1082,9 +1090,9 @@ lvl{}",
 
     // Stride related methods
 
-    pub fn get_stride_sizes(&self) -> &[u8] {
-        self.node_buckets.get_stride_sizes()
-    }
+    // pub fn get_stride_sizes(&self) -> &[u8] {
+    //     self.node_buckets.get_stride_sizes()
+    // }
 
     // Calculates the id of the node that COULD host a prefix in its
     // ptrbitarr.
@@ -1180,11 +1188,11 @@ lvl{}",
     pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
         // And, this is all of our hashing function.
         let last_level = if level > 0 {
-            <NB>::len_to_store_bits(id.len(), level - 1)
+            <NB>::bits_for_len(id.len(), level - 1)
         } else {
             0
         };
-        let this_level = <NB>::len_to_store_bits(id.len(), level);
+        let this_level = <NB>::bits_for_len(id.len(), level);
         // trace!("bits division {}", this_level);
         // trace!(
         //     "calculated index ({} << {}) >> {}",
@@ -1234,7 +1242,7 @@ lvl{}",
 
 // This implements the funky stats for a tree
 #[cfg(feature = "cli")]
-impl<AF: AddressFamily, NB: NodeBuckets<AF>> std::fmt::Display
+impl<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> std::fmt::Display
     for TreeBitMap<AF, NB>
 {
     fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -1242,14 +1250,14 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> std::fmt::Display
         writeln!(_f, "{} nodes created", self.get_nodes_count())?;
         writeln!(_f)?;
 
-        writeln!(
-            _f,
-            "stride division {:?}",
-            self.get_stride_sizes()
-                .iter()
-                .map_while(|s| if s > &0 { Some(*s) } else { None })
-                .collect::<Vec<_>>()
-        )?;
+        // writeln!(
+        //     _f,
+        //     "stride division {:?}",
+        //     self.get_stride_sizes()
+        //         .iter()
+        //         .map_while(|s| if s > &0 { Some(*s) } else { None })
+        //         .collect::<Vec<_>>()
+        // )?;
 
         writeln!(
             _f,
@@ -1260,14 +1268,14 @@ impl<AF: AddressFamily, NB: NodeBuckets<AF>> std::fmt::Display
         let bars = ["▏", "▎", "▍", "▌", "▋", "▊", "▉"];
         const SCALE: u32 = 5500;
 
-        trace!(
-            "stride_sizes {:?}",
-            self.get_stride_sizes()
-                .iter()
-                .map_while(|s| if s > &0 { Some(*s) } else { None })
-                .enumerate()
-                .collect::<Vec<(usize, u8)>>()
-        );
+        // trace!(
+        //     "stride_sizes {:?}",
+        //     self.get_stride_sizes()
+        //         .iter()
+        //         .map_while(|s| if s > &0 { Some(*s) } else { None })
+        //         .enumerate()
+        //         .collect::<Vec<(usize, u8)>>()
+        // );
 
         for crate::stats::CreatedNodes {
             depth_level: len,
diff --git a/src/local_array/prefix_cht/cht.rs b/src/local_array/prefix_cht/cht.rs
index 263c9a93..778767bf 100644
--- a/src/local_array/prefix_cht/cht.rs
+++ b/src/local_array/prefix_cht/cht.rs
@@ -7,8 +7,10 @@ use log::{debug, log_enabled, trace};
 use roaring::RoaringBitmap;
 
 use crate::{
-    local_array::in_memory::atomic_types::{MultiMapValue, StoredPrefix},
-    prelude::multi::{PrefixBuckets, PrefixId, PrefixSet, PrefixStoreError},
+    local_array::in_memory::atomic_types::{
+        MultiMapValue, PrefixSet, StoredPrefix,
+    },
+    prelude::multi::{FamilyCHT, PrefixId, PrefixStoreError},
     rib::UpsertReport,
     AddressFamily, Meta, PublicRecord,
 };
@@ -17,14 +19,14 @@ use crate::{
 pub(crate) struct PrefixCHT<
     AF: AddressFamily,
     M: Meta,
-    PB: PrefixBuckets<AF, M>,
+    PB: FamilyCHT<AF, PrefixSet<AF, M>>,
 > {
     store: PB,
     _af: PhantomData<AF>,
     _m: PhantomData<M>,
 }
 
-impl<AF: AddressFamily, M: Meta, PB: PrefixBuckets<AF, M>>
+impl<AF: AddressFamily, M: Meta, PB: FamilyCHT<AF, PrefixSet<AF, M>>>
     PrefixCHT<AF, M, PB>
 {
     pub(crate) fn new() -> Self {
@@ -42,7 +44,7 @@ impl<AF: AddressFamily, M: Meta, PB: PrefixBuckets<AF, M>>
         include_withdrawn: bool,
         bmin: &RoaringBitmap,
     ) -> Option<Vec<PublicRecord<M>>> {
-        let mut prefix_set = self.store.get_root_prefix_set(prefix.get_len());
+        let mut prefix_set = self.store.root_for_len(prefix.get_len());
         let mut level: u8 = 0;
         let backoff = Backoff::new();
 
@@ -178,7 +180,7 @@ impl<AF: AddressFamily, M: Meta, PB: PrefixBuckets<AF, M>>
     ) -> (&StoredPrefix<AF, M>, bool) {
         trace!("non_recursive_retrieve_prefix_mut_with_guard");
         let mut prefix_set =
-            self.store.get_root_prefix_set(search_prefix_id.get_len());
+            self.store.root_for_len(search_prefix_id.get_len());
         let mut level: u8 = 0;
 
         trace!("root prefix_set {:?}", prefix_set);
@@ -260,7 +262,7 @@ impl<AF: AddressFamily, M: Meta, PB: PrefixBuckets<AF, M>>
             usize,
         )>,
     ) {
-        let mut prefix_set = self.store.get_root_prefix_set(id.get_len());
+        let mut prefix_set = self.store.root_for_len(id.get_len());
         let mut parents = [None; 32];
         let mut level: u8 = 0;
         let backoff = Backoff::new();
@@ -305,11 +307,11 @@ impl<AF: AddressFamily, M: Meta, PB: PrefixBuckets<AF, M>>
     pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
         // And, this is all of our hashing function.
         let last_level = if level > 0 {
-            <PB>::get_bits_for_len(id.get_len(), level - 1)
+            <PB>::bits_for_len(id.get_len(), level - 1)
         } else {
             0
         };
-        let this_level = <PB>::get_bits_for_len(id.get_len(), level);
+        let this_level = <PB>::bits_for_len(id.get_len(), level);
         // trace!(
         //     "bits division {}; no of bits {}",
         //     this_level,
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
index a11ed54a..ba20f7fa 100644
--- a/src/local_array/prefix_cht/iterators.rs
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -5,18 +5,19 @@ use crate::{
     local_array::{
         bit_span::BitSpan,
         in_memory::{
+            atomic_types::NodeSet,
             node::{NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter},
             tree::TreeBitMap,
         },
     },
-    prelude::multi::{NodeBuckets, PrefixId},
+    prelude::multi::{FamilyCHT, PrefixId},
     AddressFamily,
 };
 
 pub(crate) struct MoreSpecificPrefixIter<
     'a,
     AF: AddressFamily,
-    NB: NodeBuckets<AF>,
+    NB: FamilyCHT<AF, NodeSet<AF>>,
 > {
     store: &'a TreeBitMap<AF, NB>,
     cur_ptr_iter: NodeMoreSpecificChildIter<AF>,
@@ -31,7 +32,7 @@ pub(crate) struct MoreSpecificPrefixIter<
     include_withdrawn: bool,
 }
 
-impl<'a, AF: AddressFamily + 'a, NB: NodeBuckets<AF>> Iterator
+impl<'a, AF: AddressFamily + 'a, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
     for MoreSpecificPrefixIter<'a, AF, NB>
 {
     type Item = PrefixId<AF>;
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 3c289a7a..a10da38d 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -4,9 +4,7 @@ use log::trace;
 use zerocopy::TryFromBytes;
 
 use crate::af::AddressFamily;
-use crate::local_array::in_memory::atomic_types::{
-    NodeBuckets, PrefixBuckets,
-};
+use crate::local_array::in_memory::atomic_types::FamilyCHT;
 use crate::prefix_record::ZeroCopyRecord;
 use crate::rib::{Config, PersistStrategy, Rib};
 use crate::PublicRecord;
@@ -17,6 +15,7 @@ use crate::{Meta, QueryResult};
 use crate::{MatchOptions, MatchType};
 
 use super::errors::PrefixStoreError;
+use super::in_memory::atomic_types::{NodeSet, PrefixSet};
 use super::persist::lsm_tree::KeySize;
 use super::types::PrefixId;
 
@@ -28,8 +27,8 @@ where
     AF: AddressFamily,
     M: Meta,
     K: KeySize<AF, KEY_SIZE>,
-    NB: NodeBuckets<AF>,
-    PB: PrefixBuckets<AF, M>,
+    NB: FamilyCHT<AF, NodeSet<AF>>,
+    PB: FamilyCHT<AF, PrefixSet<AF, M>>,
 {
     pub(crate) fn get_value(
         &'a self,
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index ab4c2291..50b9031c 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -7,16 +7,6 @@ use rand::prelude::*;
 pub const STRIDE_SIZE: u8 = 4;
 pub const STRIDE_BITS: u8 = 32;
 
-// The default stride sizes for IPv4, IPv6, resp.
-#[create_store((
-    ([4, 4, 4, 4, 4, 4, 4, 4], 5, 18, LongKey),
-    ([4, 4, 4, 4, 4, 4, 4, 4,
-      4, 4, 4, 4, 4, 4, 4, 4,
-      4, 4, 4, 4, 4, 4, 4, 4,
-      4, 4, 4, 4, 4, 4, 4, 4], 17, 30, LongKey)
-))]
-struct DefaultStore;
-
 impl<M: Meta, C: Config> DefaultStore<M, C> {
     pub fn try_default() -> Result<Self, PrefixStoreError> {
         let config = C::default();
@@ -24,3 +14,771 @@ impl<M: Meta, C: Config> DefaultStore<M, C> {
             .map_err(|_| PrefixStoreError::StoreNotReadyError)
     }
 }
+
+#[derive(Debug)]
+pub(crate) struct NodeCHT<AF: AddressFamily, const SIZE: usize>(
+    [NodeSet<AF>; SIZE],
+);
+
+impl<AF: AddressFamily, const SIZE: usize> FamilyCHT<AF, NodeSet<AF>>
+    for NodeCHT<AF, SIZE>
+{
+    fn init() -> Self {
+        Self(std::array::from_fn::<_, SIZE, _>(|_| {
+            NodeSet::<AF>::init(16)
+        }))
+    }
+
+    fn bits_for_len(len: u8, lvl: u8) -> u8 {
+        let res = 4 * (lvl + 1);
+        if res < len {
+            res
+        } else if res >= len + 4 {
+            0
+        } else {
+            len
+        }
+    }
+
+    fn root_for_len(&self, len: u8) -> &NodeSet<AF> {
+        &self.0[len as usize / 4]
+    }
+}
+
+// create a range p0..p32 for IPv4, and p0..p128 for IPv6
+#[derive(Debug)]
+pub(crate) struct PrefixCHT<AF: AddressFamily, M: Meta, const SIZE: usize>(
+    [PrefixSet<AF, M>; SIZE],
+);
+
+impl<AF: AddressFamily, M: Meta, const SIZE: usize>
+    FamilyCHT<AF, PrefixSet<AF, M>> for PrefixCHT<AF, M, SIZE>
+{
+    fn init() -> Self {
+        Self(std::array::from_fn::<_, SIZE, _>(|_| {
+            PrefixSet::<AF, M>::init(16)
+        }))
+    }
+
+    fn root_for_len(&self, len: u8) -> &PrefixSet<AF, M> {
+        &self.0[len as usize]
+    }
+
+    fn bits_for_len(len: u8, lvl: u8) -> u8 {
+        let res = 4 * (lvl + 1);
+        if res < len {
+            res
+        } else if res >= len + 4 {
+            0
+        } else {
+            len
+        }
+    }
+}
+
+pub struct DefaultStore<M: Meta, C: Config> {
+    v4: Rib<
+        IPv4,
+        M,
+        LongKey<IPv4>,
+        NodeCHT<IPv4, 9>,
+        PrefixCHT<IPv4, M, 33>,
+        C,
+        18,
+    >,
+    v6: Rib<
+        IPv6,
+        M,
+        LongKey<IPv6>,
+        NodeCHT<IPv6, 33>,
+        PrefixCHT<IPv6, M, 129>,
+        C,
+        30,
+    >,
+    config: C,
+}
+
+impl<'a, M: Meta, C: Config> DefaultStore<M, C> {
+    pub fn new_with_config(
+        config: C,
+    ) -> Result<Self, Box<dyn std::error::Error>> {
+        let rng = rand::rng();
+        let uuid: String = rng
+            .sample_iter(rand::distr::Alphanumeric)
+            .take(12)
+            .map(char::from)
+            .collect();
+        let mut config_v4 = config.clone();
+        let mut config_v6 = config.clone();
+
+        if let Some(path) = config_v4.persist_path() {
+            let pp = format!("{}/{}/ipv4/", path, uuid);
+            config_v4.set_persist_path(pp);
+        };
+
+        if let Some(path) = config_v6.persist_path() {
+            config_v6.set_persist_path(format!("{}/{}/ipv6/", path, uuid));
+        }
+
+        Ok(Self {
+            v4: Rib::new(config_v4)?,
+            v6: Rib::new(config_v6)?,
+            config,
+        })
+    }
+
+    pub fn match_prefix(
+        &'a self,
+        search_pfx: &Prefix,
+        options: &MatchOptions,
+        guard: &'a Guard,
+    ) -> QueryResult<M> {
+        match search_pfx.addr() {
+            std::net::IpAddr::V4(addr) => self.v4.match_prefix(
+                PrefixId::<IPv4>::new(
+                    <IPv4 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
+                options,
+                guard,
+            ),
+            std::net::IpAddr::V6(addr) => self.v6.match_prefix(
+                PrefixId::<IPv6>::new(
+                    <IPv6 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
+                options,
+                guard,
+            ),
+        }
+    }
+
+    pub fn contains(&'a self, prefix: &Prefix, mui: Option<u32>) -> bool {
+        match prefix.addr() {
+            std::net::IpAddr::V4(_addr) => {
+                self.v4.contains(PrefixId::<IPv4>::from(*prefix), mui)
+            }
+            std::net::IpAddr::V6(_addr) => {
+                self.v6.contains(PrefixId::<IPv6>::from(*prefix), mui)
+            }
+        }
+    }
+
+    pub fn best_path(
+        &'a self,
+        search_pfx: &Prefix,
+        guard: &Guard,
+    ) -> Option<Result<Record<M>, PrefixStoreError>> {
+        match search_pfx.addr() {
+            std::net::IpAddr::V4(addr) => self.v4.best_path(
+                PrefixId::<IPv4>::new(
+                    <IPv4 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
+                guard,
+            ),
+            std::net::IpAddr::V6(addr) => self.v6.best_path(
+                PrefixId::<IPv6>::new(
+                    <IPv6 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
+                guard,
+            ),
+        }
+    }
+
+    pub fn calculate_and_store_best_and_backup_path(
+        &self,
+        search_pfx: &Prefix,
+        tbi: &<M as Meta>::TBI,
+        guard: &Guard,
+    ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
+        match search_pfx.addr() {
+            std::net::IpAddr::V4(addr) => {
+                self.v4.calculate_and_store_best_and_backup_path(
+                    PrefixId::<IPv4>::new(
+                        <IPv4 as AddressFamily>::from_ipaddr(addr),
+                        search_pfx.len(),
+                    ),
+                    tbi,
+                    guard,
+                )
+            }
+            std::net::IpAddr::V6(addr) => {
+                self.v6.calculate_and_store_best_and_backup_path(
+                    PrefixId::<IPv6>::new(
+                        <IPv6 as AddressFamily>::from_ipaddr(addr),
+                        search_pfx.len(),
+                    ),
+                    tbi,
+                    guard,
+                )
+            }
+        }
+    }
+
+    pub fn is_ps_outdated(
+        &self,
+        search_pfx: &Prefix,
+        guard: &Guard,
+    ) -> Result<bool, PrefixStoreError> {
+        match search_pfx.addr() {
+            std::net::IpAddr::V4(addr) => self.v4.is_ps_outdated(
+                PrefixId::<IPv4>::new(
+                    <IPv4 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
+                guard,
+            ),
+            std::net::IpAddr::V6(addr) => self.v6.is_ps_outdated(
+                PrefixId::<IPv6>::new(
+                    <IPv6 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
+                guard,
+            ),
+        }
+    }
+
+    pub fn more_specifics_from(
+        &'a self,
+        search_pfx: &Prefix,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> QueryResult<M> {
+        match search_pfx.addr() {
+            std::net::IpAddr::V4(addr) => self.v4.more_specifics_from(
+                PrefixId::<IPv4>::new(
+                    <IPv4 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
+                mui,
+                include_withdrawn,
+                guard,
+            ),
+            std::net::IpAddr::V6(addr) => self.v6.more_specifics_from(
+                PrefixId::<IPv6>::new(
+                    <IPv6 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
+                mui,
+                include_withdrawn,
+                guard,
+            ),
+        }
+    }
+
+    pub fn less_specifics_from(
+        &'a self,
+        search_pfx: &Prefix,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> QueryResult<M> {
+        match search_pfx.addr() {
+            std::net::IpAddr::V4(addr) => self.v4.less_specifics_from(
+                PrefixId::<IPv4>::new(
+                    <IPv4 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
+                mui,
+                include_withdrawn,
+                guard,
+            ),
+            std::net::IpAddr::V6(addr) => self.v6.less_specifics_from(
+                PrefixId::<IPv6>::new(
+                    <IPv6 as AddressFamily>::from_ipaddr(addr),
+                    search_pfx.len(),
+                ),
+                mui,
+                include_withdrawn,
+                guard,
+            ),
+        }
+    }
+
+    pub fn less_specifics_iter_from(
+        &'a self,
+        search_pfx: &Prefix,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+        let (left, right) = match search_pfx.addr() {
+            std::net::IpAddr::V4(addr) => (
+                Some(
+                    self.v4
+                        .less_specifics_iter_from(
+                            PrefixId::<IPv4>::new(
+                                <IPv4 as AddressFamily>::from_ipaddr(addr),
+                                search_pfx.len(),
+                            ),
+                            mui,
+                            include_withdrawn,
+                            guard,
+                        )
+                        .map(PrefixRecord::from),
+                ),
+                None,
+            ),
+            std::net::IpAddr::V6(addr) => (
+                None,
+                Some(
+                    self.v6
+                        .less_specifics_iter_from(
+                            PrefixId::<IPv6>::new(
+                                <IPv6 as AddressFamily>::from_ipaddr(addr),
+                                search_pfx.len(),
+                            ),
+                            mui,
+                            include_withdrawn,
+                            guard,
+                        )
+                        .map(PrefixRecord::from),
+                ),
+            ),
+        };
+
+        left.into_iter()
+            .flatten()
+            .chain(right.into_iter().flatten())
+    }
+
+    pub fn more_specifics_iter_from(
+        &'a self,
+        search_pfx: &Prefix,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+        let (left, right) = match search_pfx.addr() {
+            std::net::IpAddr::V4(addr) => (
+                Some(
+                    self.v4
+                        .more_specifics_iter_from(
+                            PrefixId::<IPv4>::new(
+                                <IPv4 as AddressFamily>::from_ipaddr(addr),
+                                search_pfx.len(),
+                            ),
+                            mui,
+                            include_withdrawn,
+                            guard,
+                        )
+                        .map(PrefixRecord::from),
+                ),
+                None,
+            ),
+            std::net::IpAddr::V6(addr) => (
+                None,
+                Some(
+                    self.v6
+                        .more_specifics_iter_from(
+                            PrefixId::<IPv6>::new(
+                                <IPv6 as AddressFamily>::from_ipaddr(addr),
+                                search_pfx.len(),
+                            ),
+                            mui,
+                            include_withdrawn,
+                            guard,
+                        )
+                        .map(PrefixRecord::from),
+                ),
+            ),
+        };
+
+        left.into_iter()
+            .flatten()
+            .chain(right.into_iter().flatten())
+    }
+
+    pub fn iter_records_for_mui_v4(
+        &'a self,
+        mui: u32,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+        if self.v4.mui_is_withdrawn(mui, guard) && !include_withdrawn {
+            None
+        } else {
+            Some(
+                self.v4
+                    .more_specifics_iter_from(
+                        PrefixId::<IPv4>::new(
+                            <IPv4 as AddressFamily>::zero(),
+                            0,
+                        ),
+                        Some(mui),
+                        include_withdrawn,
+                        guard,
+                    )
+                    .map(PrefixRecord::from),
+            )
+        }
+        .into_iter()
+        .flatten()
+    }
+
+    pub fn iter_records_for_mui_v6(
+        &'a self,
+        mui: u32,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+        if self.v6.mui_is_withdrawn(mui, guard) && !include_withdrawn {
+            None
+        } else {
+            Some(
+                self.v6
+                    .more_specifics_iter_from(
+                        PrefixId::<IPv6>::new(
+                            <IPv6 as AddressFamily>::zero(),
+                            0,
+                        ),
+                        Some(mui),
+                        include_withdrawn,
+                        guard,
+                    )
+                    .map(PrefixRecord::from),
+            )
+        }
+        .into_iter()
+        .flatten()
+    }
+
+    pub fn insert(
+        &self,
+        prefix: &Prefix,
+        record: Record<M>,
+        update_path_selections: Option<M::TBI>,
+    ) -> Result<UpsertReport, PrefixStoreError> {
+        match prefix.addr() {
+            std::net::IpAddr::V4(_addr) => self.v4.insert(
+                PrefixId::<IPv4>::from(*prefix),
+                record,
+                update_path_selections,
+            ),
+            std::net::IpAddr::V6(_addr) => self.v6.insert(
+                PrefixId::<IPv6>::from(*prefix),
+                record,
+                update_path_selections,
+            ),
+        }
+    }
+
+    pub fn prefixes_iter(
+        &'a self,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+        self.v4
+            .prefixes_iter(guard)
+            .map(PrefixRecord::from)
+            .chain(self.v6.prefixes_iter(guard).map(PrefixRecord::from))
+    }
+
+    pub fn prefixes_iter_v4(
+        &'a self,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+        self.v4.prefixes_iter(guard).map(PrefixRecord::from)
+    }
+
+    pub fn prefixes_iter_v6(
+        &'a self,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+        self.v6.prefixes_iter(guard).map(PrefixRecord::from)
+    }
+
+    /// Change the local status of the record for the combination of
+    /// (prefix, multi_uniq_id) to Withdrawn. Note that by default the
+    /// global `Withdrawn` status for a mui overrides the local status
+    /// of a record.
+    pub fn mark_mui_as_withdrawn_for_prefix(
+        &self,
+        prefix: &Prefix,
+        mui: u32,
+        ltime: u64,
+    ) -> Result<(), PrefixStoreError> {
+        match prefix.addr() {
+            std::net::IpAddr::V4(_addr) => {
+                self.v4.mark_mui_as_withdrawn_for_prefix(
+                    PrefixId::<IPv4>::from(*prefix),
+                    mui,
+                    ltime,
+                )
+            }
+            std::net::IpAddr::V6(_addr) => {
+                self.v6.mark_mui_as_withdrawn_for_prefix(
+                    PrefixId::<IPv6>::from(*prefix),
+                    mui,
+                    ltime,
+                )
+            }
+        }
+    }
+
+    /// Change the local status of the record for the combination of
+    /// (prefix, multi_uniq_id) to Active. Note that by default the
+    /// global `Withdrawn` status for a mui overrides the local status
+    /// of a record.
+    pub fn mark_mui_as_active_for_prefix(
+        &self,
+        prefix: &Prefix,
+        mui: u32,
+        ltime: u64,
+    ) -> Result<(), PrefixStoreError> {
+        match prefix.addr() {
+            std::net::IpAddr::V4(_addr) => {
+                self.v4.mark_mui_as_active_for_prefix(
+                    PrefixId::<IPv4>::from(*prefix),
+                    mui,
+                    ltime,
+                )
+            }
+            std::net::IpAddr::V6(_addr) => {
+                self.v6.mark_mui_as_active_for_prefix(
+                    PrefixId::<IPv6>::from(*prefix),
+                    mui,
+                    ltime,
+                )
+            }
+        }
+    }
+
+    /// Change the status of all records for IPv4 prefixes for this
+    /// `multi_uniq_id` globally to Active.  Note that the global
+    /// `Active` status will be overridden by the local status of the
+    /// record.
+    pub fn mark_mui_as_active_v4(
+        &self,
+        mui: u32,
+    ) -> Result<(), PrefixStoreError> {
+        let guard = &epoch::pin();
+
+        self.v4.mark_mui_as_active(mui, guard)
+    }
+
+    /// Change the status of all records for IPv4 prefixes for this
+    /// `multi_uniq_id` globally to Withdrawn. A global `Withdrawn`
+    /// status for a `multi_uniq_id` overrides the local status of
+    /// prefixes for this mui. However the local status can still be
+    /// modified. This modification will take effect if the global
+    /// status is changed to `Active`.
+    pub fn mark_mui_as_withdrawn_v4(
+        &self,
+        mui: u32,
+    ) -> Result<(), PrefixStoreError> {
+        let guard = &epoch::pin();
+
+        self.v4.mark_mui_as_withdrawn(mui, guard)
+    }
+
+    /// Change the status of all records for IPv6 prefixes for this
+    /// `multi_uniq_id` globally to Active.  Note that the global
+    /// `Active` status will be overridden by the local status of the
+    /// record.
+    pub fn mark_mui_as_active_v6(
+        &self,
+        mui: u32,
+    ) -> Result<(), PrefixStoreError> {
+        let guard = &epoch::pin();
+
+        self.v6.mark_mui_as_active(mui, guard)
+    }
+
+    /// Change the status of all records for IPv6 prefixes for this
+    /// `multi_uniq_id` globally to Withdrawn. A global `Withdrawn`
+    /// status for a `multi_uniq_id` overrides the local status of
+    /// prefixes for this mui. However the local status can still be
+    /// modified. This modification will take effect if the global
+    /// status is changed to `Active`.
+    pub fn mark_mui_as_withdrawn_v6(
+        &self,
+        mui: u32,
+    ) -> Result<(), PrefixStoreError> {
+        let guard = &epoch::pin();
+
+        self.v6.mark_mui_as_withdrawn(mui, guard)
+    }
+
+    /// Change the status of all records for this `multi_uniq_id` to
+    /// Withdrawn.
+    ///
+    /// This method tries to mark all records: first the IPv4 records,
+    /// then the IPv6 records. If marking of the IPv4 records fails,
+    /// the method continues and tries to mark the IPv6 records. If
+    /// either or both fail, an error is returned.
+    pub fn mark_mui_as_withdrawn(
+        &self,
+        mui: u32,
+    ) -> Result<(), PrefixStoreError> {
+        let guard = &epoch::pin();
+
+        let res_v4 = self.v4.mark_mui_as_withdrawn(mui, guard);
+        let res_v6 = self.v6.mark_mui_as_withdrawn(mui, guard);
+
+        res_v4.and(res_v6)
+    }
+
+    // Whether the global status for IPv4 prefixes and the specified
+    // `multi_uniq_id` is set to `Withdrawn`.
+    pub fn mui_is_withdrawn_v4(&self, mui: u32) -> bool {
+        let guard = &epoch::pin();
+
+        self.v4.mui_is_withdrawn(mui, guard)
+    }
+
+    // Whether the global status for IPv6 prefixes and the specified
+    // `multi_uniq_id` is set to `Active`.
+    pub fn mui_is_withdrawn_v6(&self, mui: u32) -> bool {
+        let guard = &epoch::pin();
+
+        self.v6.mui_is_withdrawn(mui, guard)
+    }
+
+    /// Returns the number of all prefixes in the store.
+    ///
+    /// Note that this method will actually traverse the complete
+    /// tree.
+    pub fn prefixes_count(&self) -> UpsertCounters {
+        self.v4.get_prefixes_count() + self.v6.get_prefixes_count()
+    }
+
+    /// Returns the number of all IPv4 prefixes in the store.
+    ///
+    /// Note that this counter may be lower than the actual
+    /// number in the store, due to contention at the time of
+    /// reading the value.
+    pub fn prefixes_v4_count(&self) -> UpsertCounters {
+        self.v4.get_prefixes_count()
+    }
+
+    /// Returns the number of all IPv4 prefixes with the
+    /// supplied prefix length in the store.
+    ///
+    /// Note that this counter may be lower than the actual
+    /// number in the store, due to contention at the time of
+    /// reading the value.
+    pub fn prefixes_v4_count_for_len(&self, len: u8) -> UpsertCounters {
+        self.v4.get_prefixes_count_for_len(len)
+    }
+
+    /// Returns the number of all IPv6 prefixes in the store.
+    ///
+    /// Note that this counter may be lower than the actual
+    /// number in the store, due to contention at the time of
+    /// reading the value.
+    pub fn prefixes_v6_count(&self) -> UpsertCounters {
+        self.v6.get_prefixes_count()
+    }
+
+    /// Returns the number of all IPv6 prefixes with the
+    /// supplied prefix length in the store.
+    ///
+    /// Note that this counter may be lower than the actual
+    /// number in the store, due to contention at the time of
+    /// reading the value.
+    pub fn prefixes_v6_count_for_len(&self, len: u8) -> UpsertCounters {
+        self.v6.get_prefixes_count_for_len(len)
+    }
+
+    /// Returns the number of nodes in the store.
+    ///
+    /// Note that this counter may be lower than the actual
+    /// number in the store, due to contention at the time of
+    /// reading the value.
+    pub fn nodes_count(&self) -> usize {
+        self.v4.get_nodes_count() + self.v6.get_nodes_count()
+    }
+
+    /// Returns the number of IPv4 nodes in the store.
+    ///
+    /// Note that this counter may be lower than the actual
+    /// number in the store, due to contention at the time of
+    /// reading the value.
+    pub fn nodes_v4_count(&self) -> usize {
+        self.v4.get_nodes_count()
+    }
+
+    /// Returns the number of IPv6 nodes in the store.
+    ///
+    /// Note that this counter may be lower than the actual
+    /// number in the store, due to contention at the time of
+    /// reading the value.
+    pub fn nodes_v6_count(&self) -> usize {
+        self.v6.get_nodes_count()
+    }
+
+    /// Print the store statistics to the standard output.
+    #[cfg(feature = "cli")]
+    pub fn print_funky_stats(&self) {
+        println!("");
+        println!("Stats for IPv4 multi-threaded store\n");
+        println!("{}", self.v4.in_memory_tree);
+        println!("Stats for IPv6 multi-threaded store\n");
+        println!("{}", self.v6.in_memory_tree);
+    }
+
+    // The Store statistics.
+    pub fn stats(&self) -> StoreStats {
+        StoreStats {
+            v4: self.v4.counters.get_prefix_stats(),
+            v6: self.v6.counters.get_prefix_stats(),
+        }
+    }
+
+    // Disk Persistence
+
+    pub fn persist_strategy(&self) -> PersistStrategy {
+        self.config.persist_strategy()
+    }
+
+    pub fn get_records_for_prefix(
+        &self,
+        prefix: &Prefix,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+    ) -> Option<Vec<Record<M>>> {
+        let guard = &epoch::pin();
+
+        match prefix.is_v4() {
+            true => self.v4.get_value(
+                PrefixId::<IPv4>::from(*prefix),
+                mui,
+                include_withdrawn,
+                guard,
+            ),
+            false => self.v6.get_value(
+                PrefixId::<IPv6>::from(*prefix),
+                mui,
+                include_withdrawn,
+                guard,
+            ),
+        }
+    }
+
+    /// Persist all the non-unique (prefix, mui, ltime) tuples
+    /// with their values to disk
+    pub fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
+        self.v4.flush_to_disk()?;
+        self.v6.flush_to_disk()?;
+
+        Ok(())
+    }
+
+    /// Return the approximate number of items that are persisted
+    /// to disk, for IPv4 and IPv6 respectively.
+    pub fn approx_persisted_items(&self) -> (usize, usize) {
+        (
+            self.v4.approx_persisted_items(),
+            self.v6.approx_persisted_items(),
+        )
+    }
+
+    /// Return an estimation of the disk space currently used by the
+    /// store in bytes.
+    pub fn disk_space(&self) -> u64 {
+        self.v4.disk_space() + self.v6.disk_space()
+    }
+}
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index a2b0187f..e64b591a 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -14,14 +14,13 @@ use crate::local_array::persist::lsm_tree::KeySize;
 use crate::local_array::prefix_cht::cht::PrefixCHT;
 use crate::local_array::types::PrefixId;
 use crate::prefix_record::{ValueHeader, ZeroCopyRecord};
-use crate::prelude::multi::RouteStatus;
+use crate::prelude::multi::{PrefixSet, RouteStatus};
 use crate::stats::CreatedNodes;
 use crate::{
     local_array::errors::PrefixStoreError, prefix_record::PublicRecord,
 };
 
-use crate::local_array::in_memory::atomic_types::NodeBuckets;
-use crate::local_array::in_memory::atomic_types::PrefixBuckets;
+use crate::local_array::in_memory::atomic_types::{FamilyCHT, NodeSet};
 
 // Make sure to also import the other methods for the Rib, so the proc macro
 // create_store can use them.
@@ -330,8 +329,8 @@ pub struct Rib<
     AF: AddressFamily,
     M: Meta,
     K: KeySize<AF, KEY_SIZE>,
-    NB: NodeBuckets<AF>,
-    PB: PrefixBuckets<AF, M>,
+    NB: FamilyCHT<AF, NodeSet<AF>>,
+    PB: FamilyCHT<AF, PrefixSet<AF, M>>,
     C: Config,
     const KEY_SIZE: usize,
 > {
@@ -347,8 +346,8 @@ impl<
         AF: AddressFamily,
         M: crate::prefix_record::Meta,
         K: KeySize<AF, KEY_SIZE>,
-        NB: NodeBuckets<AF>,
-        PB: PrefixBuckets<AF, M>,
+        NB: FamilyCHT<AF, NodeSet<AF>>,
+        PB: FamilyCHT<AF, PrefixSet<AF, M>>,
         C: Config,
         const KEY_SIZE: usize,
     > Rib<AF, M, K, NB, PB, C, KEY_SIZE>
@@ -1006,8 +1005,8 @@ impl<
 impl<
         M: Meta,
         K: KeySize<IPv4, KEY_SIZE>,
-        NB: NodeBuckets<IPv4>,
-        PB: PrefixBuckets<IPv4, M>,
+        NB: FamilyCHT<IPv4, NodeSet<IPv4>>,
+        PB: FamilyCHT<IPv4, PrefixSet<IPv4, M>>,
         // const PREFIX_SIZE: usize,
         C: Config,
         const KEY_SIZE: usize,
@@ -1021,8 +1020,8 @@ impl<
 impl<
         M: Meta,
         K: KeySize<IPv6, KEY_SIZE>,
-        NB: NodeBuckets<IPv6>,
-        PB: PrefixBuckets<IPv6, M>,
+        NB: FamilyCHT<IPv6, NodeSet<IPv6>>,
+        PB: FamilyCHT<IPv6, PrefixSet<IPv6, M>>,
         C: Config,
         // const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index 69c5f1fb..21326c67 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -10,14 +10,11 @@ pub mod multi {
 
     pub use std::sync::atomic::Ordering;
 
-    pub use rotonda_macros::create_store;
-    pub use rotonda_macros::stride_sizes;
-
     pub use crossbeam_epoch::{self as epoch, Guard};
 
     pub use crate::local_array::errors::PrefixStoreError;
     pub use crate::local_array::in_memory::atomic_types::{
-        NodeBuckets, PrefixBuckets, PrefixSet,
+        FamilyCHT, NodeSet, PrefixSet,
     };
     pub use crate::local_array::in_memory::iterators;
     pub use crate::local_array::in_memory::node::StrideNodeId;

From bb1cc2d222feea624ced78c0d758e43d741b0381 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Mon, 3 Mar 2025 16:05:16 +0100
Subject: [PATCH 098/147] remove K type argument

---
 src/local_array/query.rs             |  6 +++---
 src/local_array/rib/default_store.rs |  4 ++--
 src/local_array/rib/rib.rs           | 22 +++++++++++-----------
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index a10da38d..708410a2 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -21,12 +21,12 @@ use super::types::PrefixId;
 
 //------------ Prefix Matching ----------------------------------------------
 
-impl<'a, AF, M, K, NB, PB, C: Config, const KEY_SIZE: usize>
-    Rib<AF, M, K, NB, PB, C, KEY_SIZE>
+impl<'a, AF, M, NB, PB, C: Config, const KEY_SIZE: usize>
+    Rib<AF, M, NB, PB, C, KEY_SIZE>
 where
     AF: AddressFamily,
     M: Meta,
-    K: KeySize<AF, KEY_SIZE>,
+    // K: KeySize<AF, KEY_SIZE>,
     NB: FamilyCHT<AF, NodeSet<AF>>,
     PB: FamilyCHT<AF, PrefixSet<AF, M>>,
 {
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index 50b9031c..fbfb1a86 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -80,7 +80,7 @@ pub struct DefaultStore<M: Meta, C: Config> {
     v4: Rib<
         IPv4,
         M,
-        LongKey<IPv4>,
+        // LongKey<IPv4>,
         NodeCHT<IPv4, 9>,
         PrefixCHT<IPv4, M, 33>,
         C,
@@ -89,7 +89,7 @@ pub struct DefaultStore<M: Meta, C: Config> {
     v6: Rib<
         IPv6,
         M,
-        LongKey<IPv6>,
+        // LongKey<IPv6>,
         NodeCHT<IPv6, 33>,
         PrefixCHT<IPv6, M, 129>,
         C,
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index e64b591a..539548b7 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -10,7 +10,7 @@ use epoch::{Guard, Owned};
 use zerocopy::TryFromBytes;
 
 use crate::local_array::in_memory::tree::TreeBitMap;
-use crate::local_array::persist::lsm_tree::KeySize;
+use crate::local_array::persist::lsm_tree::{KeySize, LongKey};
 use crate::local_array::prefix_cht::cht::PrefixCHT;
 use crate::local_array::types::PrefixId;
 use crate::prefix_record::{ValueHeader, ZeroCopyRecord};
@@ -328,7 +328,7 @@ pub struct UpsertReport {
 pub struct Rib<
     AF: AddressFamily,
     M: Meta,
-    K: KeySize<AF, KEY_SIZE>,
+    // K: KeySize<AF, KEY_SIZE>,
     NB: FamilyCHT<AF, NodeSet<AF>>,
     PB: FamilyCHT<AF, PrefixSet<AF, M>>,
     C: Config,
@@ -338,26 +338,26 @@ pub struct Rib<
     pub(crate) in_memory_tree: TreeBitMap<AF, NB>,
     pub(crate) prefix_cht: PrefixCHT<AF, M, PB>,
     pub(in crate::local_array) persist_tree:
-        Option<PersistTree<AF, K, KEY_SIZE>>,
+        Option<PersistTree<AF, LongKey<AF>, KEY_SIZE>>,
     pub counters: Counters,
 }
 
 impl<
         AF: AddressFamily,
         M: crate::prefix_record::Meta,
-        K: KeySize<AF, KEY_SIZE>,
+        // K: KeySize<AF, KEY_SIZE>,
         NB: FamilyCHT<AF, NodeSet<AF>>,
         PB: FamilyCHT<AF, PrefixSet<AF, M>>,
         C: Config,
         const KEY_SIZE: usize,
-    > Rib<AF, M, K, NB, PB, C, KEY_SIZE>
+    > Rib<AF, M, NB, PB, C, KEY_SIZE>
 {
     #[allow(clippy::type_complexity)]
     pub(crate) fn new(
         config: C,
-    ) -> Result<Rib<AF, M, K, NB, PB, C, KEY_SIZE>, Box<dyn std::error::Error>>
+    ) -> Result<Rib<AF, M, NB, PB, C, KEY_SIZE>, Box<dyn std::error::Error>>
     {
-        Rib::<AF, M, K, NB, PB, C, KEY_SIZE>::init(config)
+        Rib::<AF, M, NB, PB, C, KEY_SIZE>::init(config)
     }
 
     // pub(crate) fn new_short_key(
@@ -1004,13 +1004,13 @@ impl<
 
 impl<
         M: Meta,
-        K: KeySize<IPv4, KEY_SIZE>,
+        // K: KeySize<IPv4, KEY_SIZE>,
         NB: FamilyCHT<IPv4, NodeSet<IPv4>>,
         PB: FamilyCHT<IPv4, PrefixSet<IPv4, M>>,
         // const PREFIX_SIZE: usize,
         C: Config,
         const KEY_SIZE: usize,
-    > std::fmt::Display for Rib<IPv4, M, K, NB, PB, C, KEY_SIZE>
+    > std::fmt::Display for Rib<IPv4, M, NB, PB, C, KEY_SIZE>
 {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "Rib<IPv4, {}>", std::any::type_name::<M>())
@@ -1019,13 +1019,13 @@ impl<
 
 impl<
         M: Meta,
-        K: KeySize<IPv6, KEY_SIZE>,
+        // K: KeySize<IPv6, KEY_SIZE>,
         NB: FamilyCHT<IPv6, NodeSet<IPv6>>,
         PB: FamilyCHT<IPv6, PrefixSet<IPv6, M>>,
         C: Config,
         // const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
-    > std::fmt::Display for Rib<IPv6, M, K, NB, PB, C, KEY_SIZE>
+    > std::fmt::Display for Rib<IPv6, M, NB, PB, C, KEY_SIZE>
 {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "Rib<IPv6, {}>", std::any::type_name::<M>())

From 2fbbcb8bf616774b9482390a34041fdd840bfacd Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Mon, 3 Mar 2025 16:14:34 +0100
Subject: [PATCH 099/147] cleanup

---
 src/local_array/rib/default_store.rs |  30 +----
 src/local_array/rib/rib.rs           | 189 +--------------------------
 2 files changed, 7 insertions(+), 212 deletions(-)

diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index fbfb1a86..910127b6 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -1,5 +1,3 @@
-use crate::local_array::in_memory::atomic_types::NodeSet;
-use crate::local_array::persist::lsm_tree::LongKey;
 use crate::prelude::multi::*;
 use crate::prelude::*;
 use rand::prelude::*;
@@ -29,6 +27,10 @@ impl<AF: AddressFamily, const SIZE: usize> FamilyCHT<AF, NodeSet<AF>>
         }))
     }
 
+    fn root_for_len(&self, len: u8) -> &NodeSet<AF> {
+        &self.0[len as usize / 4]
+    }
+
     fn bits_for_len(len: u8, lvl: u8) -> u8 {
         let res = 4 * (lvl + 1);
         if res < len {
@@ -39,10 +41,6 @@ impl<AF: AddressFamily, const SIZE: usize> FamilyCHT<AF, NodeSet<AF>>
             len
         }
     }
-
-    fn root_for_len(&self, len: u8) -> &NodeSet<AF> {
-        &self.0[len as usize / 4]
-    }
 }
 
 // create a range p0..p32 for IPv4, and p0..p128 for IPv6
@@ -77,24 +75,8 @@ impl<AF: AddressFamily, M: Meta, const SIZE: usize>
 }
 
 pub struct DefaultStore<M: Meta, C: Config> {
-    v4: Rib<
-        IPv4,
-        M,
-        // LongKey<IPv4>,
-        NodeCHT<IPv4, 9>,
-        PrefixCHT<IPv4, M, 33>,
-        C,
-        18,
-    >,
-    v6: Rib<
-        IPv6,
-        M,
-        // LongKey<IPv6>,
-        NodeCHT<IPv6, 33>,
-        PrefixCHT<IPv6, M, 129>,
-        C,
-        30,
-    >,
+    v4: Rib<IPv4, M, NodeCHT<IPv4, 9>, PrefixCHT<IPv4, M, 33>, C, 18>,
+    v6: Rib<IPv6, M, NodeCHT<IPv6, 33>, PrefixCHT<IPv6, M, 129>, C, 30>,
     config: C,
 }
 
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index 539548b7..e22c8776 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -328,7 +328,6 @@ pub struct UpsertReport {
 pub struct Rib<
     AF: AddressFamily,
     M: Meta,
-    // K: KeySize<AF, KEY_SIZE>,
     NB: FamilyCHT<AF, NodeSet<AF>>,
     PB: FamilyCHT<AF, PrefixSet<AF, M>>,
     C: Config,
@@ -344,8 +343,7 @@ pub struct Rib<
 
 impl<
         AF: AddressFamily,
-        M: crate::prefix_record::Meta,
-        // K: KeySize<AF, KEY_SIZE>,
+        M: Meta,
         NB: FamilyCHT<AF, NodeSet<AF>>,
         PB: FamilyCHT<AF, PrefixSet<AF, M>>,
         C: Config,
@@ -360,24 +358,6 @@ impl<
         Rib::<AF, M, NB, PB, C, KEY_SIZE>::init(config)
     }
 
-    // pub(crate) fn new_short_key(
-    //     config: StoreConfig,
-    // ) -> Result<
-    //     Rib<AF, M, ShortKey<AF>, NB, PB, KEY_SIZE>,
-    //     Box<dyn std::error::Error>,
-    // > {
-    //     Rib::<AF, M, ShortKey<AF>, NB, PB, KEY_SIZE>::init(config)
-    // }
-
-    // pub(crate) fn new_long_key(
-    //     config: StoreConfig,
-    // ) -> Result<
-    //     Rib<AF, M, LongKey<AF>, NB, PB, KEY_SIZE>,
-    //     Box<dyn std::error::Error>,
-    // > {
-    //     Rib::<AF, M, LongKey<AF>, NB, PB, KEY_SIZE>::init(config)
-    // }
-
     fn init(config: C) -> Result<Self, Box<dyn std::error::Error>> {
         info!("store: initialize store {}", AF::BITS);
 
@@ -537,51 +517,11 @@ impl<
                     p_tree.get_records_with_keys_for_prefix_mui(prefix, mui);
 
                 for s in stored_prefixes {
-                    // let key: [u8; KEY_SIZE] = PersistTree::<
-                    //     AF,
-                    //     // PREFIX_SIZE,
-                    //     K,
-                    //     KEY_SIZE,
-                    // >::persistence_key(
-                    //     prefix,
-                    //     mui,
-                    //     ltime,
-                    //     RouteStatus::Withdrawn,
-                    // );
-
-                    // let key = K::new_write_key(
-                    //     prefix,
-                    //     mui,
-                    //     ltime,
-                    //     RouteStatus::Withdrawn,
-                    // );
-                    // let record =
-                    //     ZeroCopyRecord::<AF>::try_ref_from_prefix(&s)
-                    //         .unwrap()
-                    //         .0;
-                    // let key = ShortKey::from((record.prefix, mui));
-                    // trace!("insert key {:?}", key);
-
-                    // // new header for this value
-                    // let mut value = ValueHeader {
-                    //     ltime,
-                    //     status: RouteStatus::Withdrawn,
-                    // }
-                    // .as_bytes()
-                    // .to_vec();
-                    // value.extend_from_slice(record.meta.as_ref());
-
-                    // p_tree.insert(key.as_bytes(), &value);
                     let header = ValueHeader {
                         ltime,
                         status: RouteStatus::Withdrawn,
                     };
                     p_tree.rewrite_header_for_record(header, &s);
-
-                    // remove the entry for the same (prefix, mui), but with
-                    // an older logical time
-                    // let key = K::short_key(&s);
-                    // p_tree.remove(key.as_bytes());
                 }
             }
             PersistStrategy::PersistHistory => {
@@ -607,39 +547,6 @@ impl<
                             return Err(PrefixStoreError::StoreNotReadyError);
                         };
 
-                    // let key: [u8; KEY_SIZE] = PersistTree::<
-                    //     AF,
-                    //     K,
-                    //     // PREFIX_SIZE,
-                    //     KEY_SIZE,
-                    // >::persistence_key(
-                    //     prefix,
-                    //     mui,
-                    //     ltime,
-                    //     RouteStatus::Withdrawn,
-                    // );
-                    // let key = K::new_write_key(
-                    //     prefix,
-                    //     mui,
-                    //     ltime,
-                    //     RouteStatus::Withdrawn,
-                    // );
-                    //
-
-                    // Here we are keeping persisted history, so no removal of
-                    // old (prefix, mui) records.
-                    // We are inserting an empty record, since this is a
-                    // withdrawal.
-                    // p_tree.insert(
-                    //     LongKey::from((
-                    //         prefix,
-                    //         mui,
-                    //         ltime,
-                    //         RouteStatus::Withdrawn,
-                    //     ))
-                    //     .as_bytes(),
-                    //     &[],
-                    // );
                     p_tree.insert_empty_record(prefix, mui, ltime);
                 }
             }
@@ -668,57 +575,15 @@ impl<
             }
             PersistStrategy::PersistOnly => {
                 let p_tree = self.persist_tree.as_ref().unwrap();
-                // let stored_prefixes = p_tree
-                //     .get_records_with_keys_for_prefix_mui::<M>(prefix, mui)
 
                 if let Some(record_b) =
                     p_tree.get_most_recent_record_for_prefix_mui(prefix, mui)
                 {
-                    // let new_key: [u8; KEY_SIZE] = PersistTree::<
-                    //     AF,
-                    //     // PREFIX_SIZE,
-                    //     KEY_SIZE,
-                    // >::persistence_key(
-                    //     prefix,
-                    //     mui,
-                    //     ltime,
-                    //     RouteStatus::Active,
-                    // );
-
-                    // let record: &mut ZeroCopyRecord<AF> =
-                    //     ZeroCopyRecord::try_mut_from_bytes(&mut record_b)
-                    //         .unwrap();
-                    // // record.prefix = prefix;
-                    // // record.multi_uniq_id = mui;
-                    // record.ltime = ltime;
-                    // record.status = RouteStatus::Active;
-
                     let header = ValueHeader {
                         ltime,
                         status: RouteStatus::Active,
                     };
-                    // .as_bytes()
-                    // .to_vec();
-                    // value.extend_from_slice(record.meta.as_ref());
-
-                    // p_tree.insert(
-                    //     ShortKey::from((prefix, mui)).as_bytes(),
-                    //     value.as_bytes(),
-                    // );
                     p_tree.rewrite_header_for_record(header, &record_b);
-                    // // remove the entry for the same (prefix, mui), but with
-                    // // an older logical time
-                    // let old_key = PersistTree::<
-                    //     AF,
-                    //     // PREFIX_SIZE,
-                    //     KEY_SIZE,
-                    // >::persistence_key(
-                    //     prefix,
-                    //     mui,
-                    //     record.ltime,
-                    //     record.status,
-                    // );
-                    // p_tree.remove(old_key);
                 }
             }
             PersistStrategy::PersistHistory => {
@@ -742,22 +607,6 @@ impl<
                             return Err(PrefixStoreError::StoreNotReadyError);
                         };
 
-                    // let key: [u8; KEY_SIZE] = PersistTree::<
-                    //     AF,
-                    //     // PREFIX_SIZE,
-                    //     KEY_SIZE,
-                    // >::persistence_key(
-                    //     prefix,
-                    //     mui,
-                    //     ltime,
-                    //     RouteStatus::Active,
-                    // );
-                    // let key = K::new_write_key(
-                    //     prefix,
-                    //     mui,
-                    //     ltime,
-                    //     RouteStatus::Active,
-                    // );
                     // Here we are keeping persisted history, so no removal of
                     // old (prefix, mui) records.
                     // We are inserting an empty record, since this is a
@@ -943,38 +792,6 @@ impl<
             .into_iter()
             .flatten()
     }
-    // pub fn get_records_for_prefix(
-    //     &self,
-    //     prefix: &Prefix,
-    //     mui: Option<u32>,
-    //     include_withdrawn: bool,
-    // ) -> Vec<PublicRecord<M>> {
-    //     trace!("get records for prefix in the right store");
-    //     let guard = epoch::pin();
-    //     match self.persist_strategy() {
-    //         PersistStrategy::PersistOnly => self
-    //             .persist_tree
-    //             .as_ref()
-    //             .map(|tree| {
-    //                 tree.get_records_for_prefix(
-    //                     PrefixId::from(*prefix),
-    //                     mui,
-    //                     include_withdrawn,
-    //                     self.in_memory_tree.withdrawn_muis_bmin(&guard),
-    //                 )
-    //             })
-    //             .unwrap_or_default(),
-    //         _ => self
-    //             .prefix_cht
-    //             .get_records_for_prefix(
-    //                 PrefixId::from(*prefix),
-    //                 mui,
-    //                 include_withdrawn,
-    //                 self.in_memory_tree.withdrawn_muis_bmin(&guard),
-    //             )
-    //             .unwrap_or_default(),
-    //     }
-    // }
 
     pub fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
         if let Some(p) = &self.persist_tree {
@@ -1004,10 +821,8 @@ impl<
 
 impl<
         M: Meta,
-        // K: KeySize<IPv4, KEY_SIZE>,
         NB: FamilyCHT<IPv4, NodeSet<IPv4>>,
         PB: FamilyCHT<IPv4, PrefixSet<IPv4, M>>,
-        // const PREFIX_SIZE: usize,
         C: Config,
         const KEY_SIZE: usize,
     > std::fmt::Display for Rib<IPv4, M, NB, PB, C, KEY_SIZE>
@@ -1019,11 +834,9 @@ impl<
 
 impl<
         M: Meta,
-        // K: KeySize<IPv6, KEY_SIZE>,
         NB: FamilyCHT<IPv6, NodeSet<IPv6>>,
         PB: FamilyCHT<IPv6, PrefixSet<IPv6, M>>,
         C: Config,
-        // const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
     > std::fmt::Display for Rib<IPv6, M, NB, PB, C, KEY_SIZE>
 {

From 0861c1dc1ffbd1b15fde4e7b3d929c92b5d6f216 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Mar 2025 09:22:10 +0100
Subject: [PATCH 100/147] make *sets 4 times as small (ahum)

---
 src/local_array/rib/default_store.rs | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index 910127b6..10ef66c1 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -5,14 +5,6 @@ use rand::prelude::*;
 pub const STRIDE_SIZE: u8 = 4;
 pub const STRIDE_BITS: u8 = 32;
 
-impl<M: Meta, C: Config> DefaultStore<M, C> {
-    pub fn try_default() -> Result<Self, PrefixStoreError> {
-        let config = C::default();
-        Self::new_with_config(config)
-            .map_err(|_| PrefixStoreError::StoreNotReadyError)
-    }
-}
-
 #[derive(Debug)]
 pub(crate) struct NodeCHT<AF: AddressFamily, const SIZE: usize>(
     [NodeSet<AF>; SIZE],
@@ -23,7 +15,7 @@ impl<AF: AddressFamily, const SIZE: usize> FamilyCHT<AF, NodeSet<AF>>
 {
     fn init() -> Self {
         Self(std::array::from_fn::<_, SIZE, _>(|_| {
-            NodeSet::<AF>::init(16)
+            NodeSet::<AF>::init(4)
         }))
     }
 
@@ -54,7 +46,7 @@ impl<AF: AddressFamily, M: Meta, const SIZE: usize>
 {
     fn init() -> Self {
         Self(std::array::from_fn::<_, SIZE, _>(|_| {
-            PrefixSet::<AF, M>::init(16)
+            PrefixSet::<AF, M>::init(4)
         }))
     }
 
@@ -81,6 +73,12 @@ pub struct DefaultStore<M: Meta, C: Config> {
 }
 
 impl<'a, M: Meta, C: Config> DefaultStore<M, C> {
+    pub fn try_default() -> Result<Self, PrefixStoreError> {
+        let config = C::default();
+        Self::new_with_config(config)
+            .map_err(|_| PrefixStoreError::StoreNotReadyError)
+    }
+
     pub fn new_with_config(
         config: C,
     ) -> Result<Self, Box<dyn std::error::Error>> {
@@ -695,8 +693,7 @@ impl<'a, M: Meta, C: Config> DefaultStore<M, C> {
     /// Print the store statistics to the standard output.
     #[cfg(feature = "cli")]
     pub fn print_funky_stats(&self) {
-        println!("");
-        println!("Stats for IPv4 multi-threaded store\n");
+        println!("\nStats for IPv4 multi-threaded store\n");
         println!("{}", self.v4.in_memory_tree);
         println!("Stats for IPv6 multi-threaded store\n");
         println!("{}", self.v6.in_memory_tree);

From 3bf832970c59e80715c6cb2b1cdd557f8ec15849 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Mar 2025 15:15:04 +0100
Subject: [PATCH 101/147] Cht struct instead of trait

---
 src/local_array/in_memory/atomic_types.rs |  89 +++++++--
 src/local_array/in_memory/iterators.rs    |  41 ++--
 src/local_array/in_memory/oncebox.rs      |   2 +-
 src/local_array/in_memory/query.rs        |   4 +-
 src/local_array/in_memory/tree.rs         | 216 +++-------------------
 src/local_array/prefix_cht/cht.rs         |  42 ++---
 src/local_array/prefix_cht/iterators.rs   |  13 +-
 src/local_array/query.rs                  |  20 +-
 src/local_array/rib/default_store.rs      | 120 ++++++------
 src/local_array/rib/init                  |   1 +
 src/local_array/rib/rib.rs                |  51 +++--
 src/prelude/mod.rs                        |   3 -
 src/rotonda_store.rs                      |  80 ++++----
 13 files changed, 272 insertions(+), 410 deletions(-)
 create mode 100644 src/local_array/rib/init

diff --git a/src/local_array/in_memory/atomic_types.rs b/src/local_array/in_memory/atomic_types.rs
index f9a8618a..97de4a23 100644
--- a/src/local_array/in_memory/atomic_types.rs
+++ b/src/local_array/in_memory/atomic_types.rs
@@ -13,8 +13,8 @@ use log::{debug, log_enabled, trace};
 use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
 
+use crate::local_array::rib::default_store::STRIDE_SIZE;
 use crate::local_array::types::{PrefixId, RouteStatus};
-// use crate::local_array::in_memory_tree::*;
 use crate::prefix_record::PublicRecord;
 use crate::prelude::Meta;
 use crate::AddressFamily;
@@ -121,7 +121,7 @@ impl PathSelections {
 // records that are stored inside it, so that iterators over its linked lists
 // don't have to go into them if there's nothing there and could stop early.
 #[derive(Debug)]
-pub struct StoredPrefix<AF: AddressFamily, M: crate::prefix_record::Meta> {
+pub struct StoredPrefix<AF: AddressFamily, M: Meta> {
     // the serial number
     // pub serial: usize,
     // the prefix itself,
@@ -135,15 +135,12 @@ pub struct StoredPrefix<AF: AddressFamily, M: crate::prefix_record::Meta> {
 }
 
 impl<AF: AddressFamily, M: crate::prefix_record::Meta> StoredPrefix<AF, M> {
-    pub(crate) fn new<PB: FamilyCHT<AF, PrefixSet<AF, M>>>(
-        pfx_id: PrefixId<AF>,
-        level: u8,
-    ) -> Self {
+    pub(crate) fn new(pfx_id: PrefixId<AF>, level: u8) -> Self {
         // start calculation size of next set, it's dependent on the level
         // we're in.
         // let pfx_id = PrefixId::new(record.net, record.len);
-        let this_level = PB::bits_for_len(pfx_id.get_len(), level);
-        let next_level = PB::bits_for_len(pfx_id.get_len(), level + 1);
+        let this_level = bits_for_len(pfx_id.get_len(), level);
+        let next_level = bits_for_len(pfx_id.get_len(), level + 1);
 
         trace!("this level {} next level {}", this_level, next_level);
         let next_bucket: PrefixSet<AF, M> = if next_level > 0 {
@@ -153,7 +150,7 @@ impl<AF: AddressFamily, M: crate::prefix_record::Meta> StoredPrefix<AF, M> {
                 1 << (next_level - this_level),
                 pfx_id.get_len()
             );
-            PrefixSet::init(next_level - this_level)
+            PrefixSet::init(next_level.saturating_sub(this_level))
         } else {
             debug!(
                 "{} store: INSERT at LAST LEVEL with empty bucket at prefix len {}",
@@ -566,11 +563,11 @@ impl<M: Meta> Clone for MultiMap<M> {
 
 // ----------- FamilyBuckets Trait ------------------------------------------
 
-pub trait FamilyCHT<AF: AddressFamily, V> {
-    fn init() -> Self;
-    fn bits_for_len(len: u8, level: u8) -> u8;
-    fn root_for_len(&self, len: u8) -> &V;
-}
+// pub trait FamilyCHT<AF: AddressFamily, V> {
+//     fn init() -> Self;
+//     fn bits_for_len(len: u8, level: u8) -> u8;
+//     fn root_for_len(&self, len: u8) -> &V;
+// }
 
 //------------ PrefixSet ----------------------------------------------------
 
@@ -593,3 +590,67 @@ impl<AF: AddressFamily, M: Meta> PrefixSet<AF, M> {
         PrefixSet(OnceBoxSlice::new(p2_size))
     }
 }
+
+//-----
+//
+pub(crate) trait Value {
+    fn init(size: usize) -> Self;
+}
+
+impl<AF: AddressFamily, M: Meta> Value for PrefixSet<AF, M> {
+    fn init(p2_size: usize) -> Self {
+        PrefixSet(OnceBoxSlice::new(p2_size as u8))
+    }
+}
+
+impl<AF: AddressFamily> Value for NodeSet<AF> {
+    fn init(p2_size: usize) -> Self {
+        if log_enabled!(log::Level::Debug) {
+            debug!(
+                "{} store: creating space for {} nodes",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                1 << p2_size
+            );
+        }
+
+        NodeSet(
+            OnceBoxSlice::new(p2_size as u8),
+            RoaringBitmap::new().into(),
+        )
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct Cht<
+    V,
+    const ROOT_SIZE: usize,
+    const STRIDES_PER_BUCKET: usize,
+>([V; ROOT_SIZE]);
+
+impl<V: Value, const ROOT_SIZE: usize, const STRIDES_PER_BUCKET: usize>
+    Cht<V, ROOT_SIZE, STRIDES_PER_BUCKET>
+{
+    pub(crate) fn init() -> Self {
+        Self(std::array::from_fn::<_, ROOT_SIZE, _>(|_| {
+            V::init(STRIDE_SIZE as usize)
+        }))
+    }
+
+    pub(crate) fn root_for_len(&self, len: u8) -> &V {
+        &self.0[len as usize / STRIDES_PER_BUCKET]
+    }
+}
+
+pub(crate) fn bits_for_len(len: u8, lvl: u8) -> u8 {
+    let res = STRIDE_SIZE * (lvl + 1);
+    if res < len {
+        res
+    } else if res >= len + STRIDE_SIZE {
+        0
+    } else {
+        len
+    }
+}
+
+pub(crate) type NodeCht<AF, const ROOT_SIZE: usize> =
+    Cht<NodeSet<AF>, ROOT_SIZE, 4>;
diff --git a/src/local_array/in_memory/iterators.rs b/src/local_array/in_memory/iterators.rs
index 9c1b849f..b924664f 100644
--- a/src/local_array/in_memory/iterators.rs
+++ b/src/local_array/in_memory/iterators.rs
@@ -19,7 +19,6 @@
 // contention, since every lookup has to go through the levels near the root
 // in the TreeBitMap.
 
-use crate::local_array::in_memory::atomic_types::FamilyCHT;
 use crate::local_array::in_memory::tree::TreeBitMap;
 use crate::{
     af::AddressFamily,
@@ -35,8 +34,6 @@ use crate::{
 use inetnum::addr::Prefix;
 use log::{log_enabled, trace};
 
-use super::atomic_types::NodeSet;
-
 // ----------- MoreSpecificPrefixIter ------------------------------------
 
 // A iterator over all the more-specifics for a given prefix.
@@ -58,16 +55,16 @@ use super::atomic_types::NodeSet;
 pub(crate) struct MoreSpecificPrefixIter<
     'a,
     AF: AddressFamily,
-    NB: FamilyCHT<AF, NodeSet<AF>>,
+    const ROOT_SIZE: usize,
 > {
-    tree: &'a TreeBitMap<AF, NB>,
+    tree: &'a TreeBitMap<AF, ROOT_SIZE>,
     cur_ptr_iter: NodeMoreSpecificChildIter<AF>,
     cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF>,
     parent_and_position: Vec<NodeMoreSpecificChildIter<AF>>,
 }
 
-impl<'a, AF: AddressFamily + 'a, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
-    for MoreSpecificPrefixIter<'a, AF, NB>
+impl<'a, AF: AddressFamily + 'a, const ROOT_SIZE: usize> Iterator
+    for MoreSpecificPrefixIter<'a, AF, ROOT_SIZE>
 {
     type Item = PrefixId<AF>;
 
@@ -180,17 +177,14 @@ impl<'a, AF: AddressFamily + 'a, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
     }
 }
 
-pub(crate) struct LMPrefixIter<
-    'a,
-    AF: AddressFamily,
-    NB: FamilyCHT<AF, NodeSet<AF>>,
-> {
-    tree: &'a TreeBitMap<AF, NB>,
+pub(crate) struct LMPrefixIter<'a, AF: AddressFamily, const ROOT_SIZE: usize>
+{
+    tree: &'a TreeBitMap<AF, ROOT_SIZE>,
     prefix: PrefixId<AF>,
 }
 
-impl<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
-    for LMPrefixIter<'_, AF, NB>
+impl<AF: AddressFamily, const ROOT_SIZE: usize> Iterator
+    for LMPrefixIter<'_, AF, ROOT_SIZE>
 {
     type Item = PrefixId<AF>;
     fn next(&mut self) -> Option<Self::Item> {
@@ -221,15 +215,15 @@ impl<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
 pub(crate) struct LessSpecificPrefixIter<
     'a,
     AF: AddressFamily,
-    NB: FamilyCHT<AF, NodeSet<AF>>,
+    const ROOT_SIZE: usize,
 > {
-    tree: &'a TreeBitMap<AF, NB>,
+    tree: &'a TreeBitMap<AF, ROOT_SIZE>,
     prefix: PrefixId<AF>,
     cur_level: u8,
 }
 
-impl<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
-    for LessSpecificPrefixIter<'_, AF, NB>
+impl<AF: AddressFamily, const ROOT_SIZE: usize> Iterator
+    for LessSpecificPrefixIter<'_, AF, ROOT_SIZE>
 {
     type Item = PrefixId<AF>;
 
@@ -261,13 +255,8 @@ impl<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
 // These are only the methods that are starting the iterations. All other
 // methods for Rib are in the main rib.rs file.
 
-impl<
-        'a,
-        AF: AddressFamily,
-        // M: crate::prefix_record::Meta,
-        NB: FamilyCHT<AF, NodeSet<AF>>,
-        // PB: PrefixBuckets<AF, M>,
-    > TreeBitMap<AF, NB>
+impl<'a, AF: AddressFamily, const ROOT_SIZE: usize>
+    TreeBitMap<AF, ROOT_SIZE>
 {
     // Iterator over all more-specific prefixes, starting from the given
     // prefix at the given level and cursor.
diff --git a/src/local_array/in_memory/oncebox.rs b/src/local_array/in_memory/oncebox.rs
index 9752ba5f..8234883e 100644
--- a/src/local_array/in_memory/oncebox.rs
+++ b/src/local_array/in_memory/oncebox.rs
@@ -62,7 +62,7 @@ impl<T> Drop for OnceBox<T> {
 }
 
 #[derive(Debug, Default)]
-pub struct OnceBoxSlice<T> {
+pub(crate) struct OnceBoxSlice<T> {
     ptr: AtomicPtr<OnceBox<T>>,
     p2_size: u8,
 }
diff --git a/src/local_array/in_memory/query.rs b/src/local_array/in_memory/query.rs
index fa2b2757..5cf7fcc1 100644
--- a/src/local_array/in_memory/query.rs
+++ b/src/local_array/in_memory/query.rs
@@ -4,13 +4,11 @@ use crate::rib::query::TreeQueryResult;
 use crate::{MatchOptions, MatchType};
 
 use super::super::types::PrefixId;
-use super::atomic_types::{FamilyCHT, NodeSet};
 use super::tree::TreeBitMap;
 
-impl<AF, NB> TreeBitMap<AF, NB>
+impl<AF, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE>
 where
     AF: AddressFamily,
-    NB: FamilyCHT<AF, NodeSet<AF>>,
 {
     pub(crate) fn match_prefix(
         &self,
diff --git a/src/local_array/in_memory/tree.rs b/src/local_array/in_memory/tree.rs
index 2046c152..f0d194b2 100644
--- a/src/local_array/in_memory/tree.rs
+++ b/src/local_array/in_memory/tree.rs
@@ -184,7 +184,9 @@
 // produce a fix for it).
 
 use crate::local_array::bit_span::BitSpan;
-use crate::local_array::in_memory::atomic_types::{NodeSet, StoredNode};
+use crate::local_array::in_memory::atomic_types::{
+    bits_for_len, NodeSet, StoredNode,
+};
 use crate::local_array::rib::default_store::STRIDE_SIZE;
 use crate::prelude::multi::PrefixId;
 use crossbeam_epoch::{Atomic, Guard};
@@ -194,7 +196,7 @@ use roaring::RoaringBitmap;
 use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering};
 use std::{fmt::Debug, marker::PhantomData};
 
-use super::atomic_types::FamilyCHT;
+use super::atomic_types::{Cht, NodeCht};
 use crate::af::AddressFamily;
 use crate::rib::Counters;
 
@@ -210,22 +212,20 @@ use ansi_term::Colour;
 //--------------------- TreeBitMap ------------------------------------------
 
 #[derive(Debug)]
-pub struct TreeBitMap<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> {
-    pub(crate) node_buckets: NB,
+pub struct TreeBitMap<AF: AddressFamily, const ROOT_SIZE: usize> {
+    pub(crate) node_buckets: NodeCht<AF, ROOT_SIZE>,
     pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     counters: Counters,
     default_route_exists: AtomicBool,
-    _af: PhantomData<AF>,
 }
 
-impl<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> TreeBitMap<AF, NB> {
+impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
     pub(crate) fn new() -> Result<Self, Box<dyn std::error::Error>> {
         let tree_bitmap = Self {
-            node_buckets: FamilyCHT::init(),
+            node_buckets: Cht::init(),
             withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
             default_route_exists: AtomicBool::new(false),
-            _af: PhantomData,
         };
 
         let _retry_count = tree_bitmap.store_node(
@@ -507,10 +507,7 @@ Giving up this node. This shouldn't happen!",
         let mut retry_count = 0;
 
         loop {
-            let this_level = <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                id.len(),
-                level,
-            );
+            let this_level = bits_for_len(id.len(), level);
 
             // if this_level > (id.len() / 4) {
             //     return Err(PrefixStoreError::StoreNotReadyError);
@@ -526,11 +523,7 @@ Giving up this node. This shouldn't happen!",
             match nodes.read().get(index) {
                 None => {
                     // No node exists, so we create one here.
-                    let next_level =
-                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                            id.len(),
-                            level + 1,
-                        );
+                    let next_level = bits_for_len(id.len(), level + 1);
 
                     if log_enabled!(log::Level::Trace) {
                         trace!(
@@ -646,10 +639,7 @@ lvl{}",
                             index
                         );
 
-                        match <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                            id.len(),
-                            level,
-                        ) {
+                        match bits_for_len(id.len(), level) {
                             // on to the next level!
                             next_bit_shift if next_bit_shift > 0 => {
                                 nodes = &stored_node.node_set;
@@ -667,131 +657,6 @@ lvl{}",
         }
     }
 
-    // Create a new node in the store with payload `next_node`.
-    //
-    // Next node will be ignored if a node with the same `id` already exists,
-    // but the multi_uniq_id will be added to the rbm_index of the NodeSet.
-    //
-    // Returns: a tuple with the node_id of the created node and the number of
-    // retry_count
-    // #[allow(clippy::type_complexity)]
-    // fn _old_old_store_node(
-    //     &self,
-    //     id: StrideNodeId<AF>,
-    //     multi_uniq_id: u32,
-    //     next_node: TreeBitMapNode<AF, Stride4>,
-    // ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
-    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-    //         f: &'s dyn Fn(
-    //             &SearchLevel<AF, S>,
-    //             &NodeSet<AF, S>,
-    //             TreeBitMapNode<AF, S>,
-    //             u32, // multi_uniq_id
-    //             u8,  // the store level
-    //             u32, // retry_count
-    //         ) -> Result<
-    //             (StrideNodeId<AF>, u32),
-    //             PrefixStoreError,
-    //         >,
-    //     }
-
-    //     let search_level_3 =
-    //         store_node_closure![Stride3; id; guard; back_off;];
-    //     let search_level_4 =
-    //         store_node_closure![Stride4; id; guard; back_off;];
-    //     let search_level_5 =
-    //         store_node_closure![Stride5; id; guard; back_off;];
-
-    //     if log_enabled!(log::Level::Trace) {
-    //         debug!(
-    //             "{} store: Store node {}: {:?} mui {}",
-    //             std::thread::current().name().unwrap_or("unnamed-thread"),
-    //             id,
-    //             next_node,
-    //             multi_uniq_id
-    //         );
-    //     }
-    //     self.counters.inc_nodes_count();
-
-    //     // match next_node {
-    //     // SizedStrideNode::Stride3(new_node) => (search_level_3.f)(
-    //     //     &search_level_3,
-    //     //     self.node_buckets.get_store3(id),
-    //     //     new_node,
-    //     //     multi_uniq_id,
-    //     //     0,
-    //     //     0,
-    //     // ),
-    //     (search_level_4.f)(
-    //         &search_level_4,
-    //         self.node_buckets.get_store4(id),
-    //         next_node,
-    //         multi_uniq_id,
-    //         0,
-    //         0,
-    //     )
-    //     // SizedStrideNode::Stride5(new_node) => (search_level_5.f)(
-    //     //     &search_level_5,
-    //     //     self.node_buckets.get_store5(id),
-    //     //     new_node,
-    //     //     multi_uniq_id,
-    //     //     0,
-    //     //     0,
-    //     // ),
-    //     // }
-    // }
-
-    // #[allow(clippy::type_complexity)]
-    // pub(crate) fn _retrieve_node_mut(
-    //     &self,
-    //     id: StrideNodeId<AF>,
-    //     multi_uniq_id: u32,
-    // ) -> Option<SizedStrideRef<'_, AF>> {
-    //     struct SearchLevel<'s, AF: AddressFamily, S: Stride> {
-    //         f: &'s dyn for<'a> Fn(
-    //             &SearchLevel<AF, S>,
-    //             &'a NodeSet<AF, S>,
-    //             u8,
-    //         )
-    //             -> Option<SizedStrideRef<'a, AF>>,
-    //     }
-
-    //     let search_level_3 =
-    //         retrieve_node_mut_closure![Stride3; id; multi_uniq_id;];
-    //     let search_level_4 =
-    //         retrieve_node_mut_closure![Stride4; id; multi_uniq_id;];
-    //     let search_level_5 =
-    //         retrieve_node_mut_closure![Stride5; id; multi_uniq_id;];
-
-    //     if log_enabled!(log::Level::Trace) {
-    //         trace!(
-    //             "{} store: Retrieve node mut {} from l{}",
-    //             std::thread::current().name().unwrap_or("unnamed-thread"),
-    //             id,
-    //             id.get_id().1
-    //         );
-    //     }
-
-    //     match self.node_buckets.get_stride_for_id(id) {
-    //         3 => (search_level_3.f)(
-    //             &search_level_3,
-    //             self.node_buckets.get_store3(id),
-    //             0,
-    //         ),
-
-    //         4 => (search_level_4.f)(
-    //             &search_level_4,
-    //             self.node_buckets.get_store4(id),
-    //             0,
-    //         ),
-    //         _ => (search_level_5.f)(
-    //             &search_level_5,
-    //             self.node_buckets.get_store5(id),
-    //             0,
-    //         ),
-    //     }
-    // }
-
     pub fn retrieve_node_mut(
         &self,
         id: StrideNodeId<AF>,
@@ -812,16 +677,8 @@ lvl{}",
                 // thread running this method, so our thread enounters an
                 // empty node in the store.
                 None => {
-                    let this_level =
-                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                            id.len(),
-                            level,
-                        );
-                    let next_level =
-                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                            id.len(),
-                            level + 1,
-                        );
+                    let this_level = bits_for_len(id.len(), level);
+                    let next_level = bits_for_len(id.len(), level + 1);
                     let node_set =
                         NodeSet::init(next_level.saturating_sub(this_level));
 
@@ -870,10 +727,7 @@ lvl{}",
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                id.len(),
-                level,
-            ) {
+            match bits_for_len(id.len(), level) {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -903,16 +757,8 @@ lvl{}",
                 // thread running this method, so our thread enounters an
                 // empty node in the store.
                 None => {
-                    let this_level =
-                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                            id.len(),
-                            level,
-                        );
-                    let next_level =
-                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                            id.len(),
-                            level + 1,
-                        );
+                    let this_level = bits_for_len(id.len(), level);
+                    let next_level = bits_for_len(id.len(), level + 1);
                     let node_set =
                         NodeSet::init(next_level.saturating_sub(this_level));
 
@@ -959,10 +805,7 @@ lvl{}",
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                id.len(),
-                level,
-            ) {
+            match bits_for_len(id.len(), level) {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -992,16 +835,8 @@ lvl{}",
                 // thread running this method, so our thread enounters an
                 // empty node in the store.
                 None => {
-                    let this_level =
-                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                            id.len(),
-                            level,
-                        );
-                    let next_level =
-                        <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                            id.len(),
-                            level + 1,
-                        );
+                    let this_level = bits_for_len(id.len(), level);
+                    let next_level = bits_for_len(id.len(), level + 1);
                     let node_set =
                         NodeSet::init(next_level.saturating_sub(this_level));
 
@@ -1058,10 +893,7 @@ lvl{}",
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match <NB as FamilyCHT<AF, NodeSet<AF>>>::bits_for_len(
-                id.len(),
-                level,
-            ) {
+            match bits_for_len(id.len(), level) {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -1188,11 +1020,11 @@ lvl{}",
     pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
         // And, this is all of our hashing function.
         let last_level = if level > 0 {
-            <NB>::bits_for_len(id.len(), level - 1)
+            bits_for_len(id.len(), level - 1)
         } else {
             0
         };
-        let this_level = <NB>::bits_for_len(id.len(), level);
+        let this_level = bits_for_len(id.len(), level);
         // trace!("bits division {}", this_level);
         // trace!(
         //     "calculated index ({} << {}) >> {}",
@@ -1242,8 +1074,8 @@ lvl{}",
 
 // This implements the funky stats for a tree
 #[cfg(feature = "cli")]
-impl<AF: AddressFamily, NB: FamilyCHT<AF, NodeSet<AF>>> std::fmt::Display
-    for TreeBitMap<AF, NB>
+impl<AF: AddressFamily, const ROOT_SIZE: usize> std::fmt::Display
+    for TreeBitMap<AF, ROOT_SIZE>
 {
     fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(_f, "{} prefixes created", self.get_prefixes_count())?;
diff --git a/src/local_array/prefix_cht/cht.rs b/src/local_array/prefix_cht/cht.rs
index 778767bf..4d9add20 100644
--- a/src/local_array/prefix_cht/cht.rs
+++ b/src/local_array/prefix_cht/cht.rs
@@ -1,5 +1,3 @@
-use std::marker::PhantomData;
-
 use crossbeam_epoch::Guard;
 use crossbeam_utils::Backoff;
 use inetnum::addr::Prefix;
@@ -8,33 +6,25 @@ use roaring::RoaringBitmap;
 
 use crate::{
     local_array::in_memory::atomic_types::{
-        MultiMapValue, PrefixSet, StoredPrefix,
+        bits_for_len, Cht, MultiMapValue, PrefixSet, StoredPrefix,
     },
-    prelude::multi::{FamilyCHT, PrefixId, PrefixStoreError},
+    prelude::multi::{PrefixId, PrefixStoreError},
     rib::UpsertReport,
     AddressFamily, Meta, PublicRecord,
 };
 
 #[derive(Debug)]
-pub(crate) struct PrefixCHT<
+pub(crate) struct PrefixCht<
     AF: AddressFamily,
     M: Meta,
-    PB: FamilyCHT<AF, PrefixSet<AF, M>>,
-> {
-    store: PB,
-    _af: PhantomData<AF>,
-    _m: PhantomData<M>,
-}
+    const ROOT_SIZE: usize,
+>(Cht<PrefixSet<AF, M>, ROOT_SIZE, 1>);
 
-impl<AF: AddressFamily, M: Meta, PB: FamilyCHT<AF, PrefixSet<AF, M>>>
-    PrefixCHT<AF, M, PB>
+impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
+    PrefixCht<AF, M, ROOT_SIZE>
 {
-    pub(crate) fn new() -> Self {
-        Self {
-            store: PB::init(),
-            _af: PhantomData,
-            _m: PhantomData,
-        }
+    pub(crate) fn init() -> Self {
+        Self(<Cht<PrefixSet<AF, M>, ROOT_SIZE, 1>>::init())
     }
 
     pub(crate) fn get_records_for_prefix(
@@ -44,7 +34,7 @@ impl<AF: AddressFamily, M: Meta, PB: FamilyCHT<AF, PrefixSet<AF, M>>>
         include_withdrawn: bool,
         bmin: &RoaringBitmap,
     ) -> Option<Vec<PublicRecord<M>>> {
-        let mut prefix_set = self.store.root_for_len(prefix.get_len());
+        let mut prefix_set = self.0.root_for_len(prefix.get_len());
         let mut level: u8 = 0;
         let backoff = Backoff::new();
 
@@ -173,14 +163,12 @@ impl<AF: AddressFamily, M: Meta, PB: FamilyCHT<AF, PrefixSet<AF, M>>>
     //
     // The error condition really shouldn't happen, because that basically
     // means the root node for that particular prefix length doesn't exist.
-    #[allow(clippy::type_complexity)]
     pub(crate) fn non_recursive_retrieve_prefix_mut(
         &self,
         search_prefix_id: PrefixId<AF>,
     ) -> (&StoredPrefix<AF, M>, bool) {
         trace!("non_recursive_retrieve_prefix_mut_with_guard");
-        let mut prefix_set =
-            self.store.root_for_len(search_prefix_id.get_len());
+        let mut prefix_set = self.0.root_for_len(search_prefix_id.get_len());
         let mut level: u8 = 0;
 
         trace!("root prefix_set {:?}", prefix_set);
@@ -210,7 +198,7 @@ impl<AF: AddressFamily, M: Meta, PB: FamilyCHT<AF, PrefixSet<AF, M>>>
                         prefix_set
                             .0
                             .get_or_init(index, || {
-                                StoredPrefix::new::<PB>(
+                                StoredPrefix::new(
                                     PrefixId::new(
                                         search_prefix_id.get_net(),
                                         search_prefix_id.get_len(),
@@ -262,7 +250,7 @@ impl<AF: AddressFamily, M: Meta, PB: FamilyCHT<AF, PrefixSet<AF, M>>>
             usize,
         )>,
     ) {
-        let mut prefix_set = self.store.root_for_len(id.get_len());
+        let mut prefix_set = self.0.root_for_len(id.get_len());
         let mut parents = [None; 32];
         let mut level: u8 = 0;
         let backoff = Backoff::new();
@@ -307,11 +295,11 @@ impl<AF: AddressFamily, M: Meta, PB: FamilyCHT<AF, PrefixSet<AF, M>>>
     pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
         // And, this is all of our hashing function.
         let last_level = if level > 0 {
-            <PB>::bits_for_len(id.get_len(), level - 1)
+            bits_for_len(id.get_len(), level - 1)
         } else {
             0
         };
-        let this_level = <PB>::bits_for_len(id.get_len(), level);
+        let this_level = bits_for_len(id.get_len(), level);
         // trace!(
         //     "bits division {}; no of bits {}",
         //     this_level,
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/local_array/prefix_cht/iterators.rs
index ba20f7fa..efbb7e22 100644
--- a/src/local_array/prefix_cht/iterators.rs
+++ b/src/local_array/prefix_cht/iterators.rs
@@ -5,21 +5,20 @@ use crate::{
     local_array::{
         bit_span::BitSpan,
         in_memory::{
-            atomic_types::NodeSet,
             node::{NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter},
             tree::TreeBitMap,
         },
     },
-    prelude::multi::{FamilyCHT, PrefixId},
+    prelude::multi::PrefixId,
     AddressFamily,
 };
 
-pub(crate) struct MoreSpecificPrefixIter<
+pub(crate) struct _MoreSpecificPrefixIter<
     'a,
     AF: AddressFamily,
-    NB: FamilyCHT<AF, NodeSet<AF>>,
+    const ROOT_SIZE: usize,
 > {
-    store: &'a TreeBitMap<AF, NB>,
+    store: &'a TreeBitMap<AF, ROOT_SIZE>,
     cur_ptr_iter: NodeMoreSpecificChildIter<AF>,
     cur_pfx_iter: NodeMoreSpecificsPrefixIter<AF>,
     parent_and_position: Vec<NodeMoreSpecificChildIter<AF>>,
@@ -32,8 +31,8 @@ pub(crate) struct MoreSpecificPrefixIter<
     include_withdrawn: bool,
 }
 
-impl<'a, AF: AddressFamily + 'a, NB: FamilyCHT<AF, NodeSet<AF>>> Iterator
-    for MoreSpecificPrefixIter<'a, AF, NB>
+impl<'a, AF: AddressFamily + 'a, const ROOT_SIZE: usize> Iterator
+    for _MoreSpecificPrefixIter<'a, AF, ROOT_SIZE>
 {
     type Item = PrefixId<AF>;
 
diff --git a/src/local_array/query.rs b/src/local_array/query.rs
index 708410a2..17424d2f 100644
--- a/src/local_array/query.rs
+++ b/src/local_array/query.rs
@@ -4,7 +4,6 @@ use log::trace;
 use zerocopy::TryFromBytes;
 
 use crate::af::AddressFamily;
-use crate::local_array::in_memory::atomic_types::FamilyCHT;
 use crate::prefix_record::ZeroCopyRecord;
 use crate::rib::{Config, PersistStrategy, Rib};
 use crate::PublicRecord;
@@ -15,20 +14,19 @@ use crate::{Meta, QueryResult};
 use crate::{MatchOptions, MatchType};
 
 use super::errors::PrefixStoreError;
-use super::in_memory::atomic_types::{NodeSet, PrefixSet};
-use super::persist::lsm_tree::KeySize;
 use super::types::PrefixId;
 
 //------------ Prefix Matching ----------------------------------------------
 
-impl<'a, AF, M, NB, PB, C: Config, const KEY_SIZE: usize>
-    Rib<AF, M, NB, PB, C, KEY_SIZE>
-where
-    AF: AddressFamily,
-    M: Meta,
-    // K: KeySize<AF, KEY_SIZE>,
-    NB: FamilyCHT<AF, NodeSet<AF>>,
-    PB: FamilyCHT<AF, PrefixSet<AF, M>>,
+impl<
+        'a,
+        AF: AddressFamily,
+        M: Meta,
+        const N_ROOT_SIZE: usize,
+        const P_ROOT_SIZE: usize,
+        C: Config,
+        const KEY_SIZE: usize,
+    > Rib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
 {
     pub(crate) fn get_value(
         &'a self,
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index 10ef66c1..1f3be8b9 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -5,70 +5,70 @@ use rand::prelude::*;
 pub const STRIDE_SIZE: u8 = 4;
 pub const STRIDE_BITS: u8 = 32;
 
-#[derive(Debug)]
-pub(crate) struct NodeCHT<AF: AddressFamily, const SIZE: usize>(
-    [NodeSet<AF>; SIZE],
-);
-
-impl<AF: AddressFamily, const SIZE: usize> FamilyCHT<AF, NodeSet<AF>>
-    for NodeCHT<AF, SIZE>
-{
-    fn init() -> Self {
-        Self(std::array::from_fn::<_, SIZE, _>(|_| {
-            NodeSet::<AF>::init(4)
-        }))
-    }
-
-    fn root_for_len(&self, len: u8) -> &NodeSet<AF> {
-        &self.0[len as usize / 4]
-    }
-
-    fn bits_for_len(len: u8, lvl: u8) -> u8 {
-        let res = 4 * (lvl + 1);
-        if res < len {
-            res
-        } else if res >= len + 4 {
-            0
-        } else {
-            len
-        }
-    }
-}
+// #[derive(Debug)]
+// pub(crate) struct NodeCHT<AF: AddressFamily, const SIZE: usize>(
+//     [NodeSet<AF>; SIZE],
+// );
+
+// impl<AF: AddressFamily, const SIZE: usize> FamilyCHT<AF, NodeSet<AF>>
+//     for NodeCHT<AF, SIZE>
+// {
+//     fn init() -> Self {
+//         Self(std::array::from_fn::<_, SIZE, _>(|_| {
+//             NodeSet::<AF>::init(STRIDE_SIZE)
+//         }))
+//     }
+
+//     fn root_for_len(&self, len: u8) -> &NodeSet<AF> {
+//         &self.0[len as usize / STRIDE_SIZE as usize]
+//     }
+
+//     fn bits_for_len(len: u8, lvl: u8) -> u8 {
+//         let res = STRIDE_SIZE * (lvl + 1);
+//         if res < len {
+//             res
+//         } else if res >= len + STRIDE_SIZE {
+//             0
+//         } else {
+//             len
+//         }
+//     }
+// }
 
 // create a range p0..p32 for IPv4, and p0..p128 for IPv6
-#[derive(Debug)]
-pub(crate) struct PrefixCHT<AF: AddressFamily, M: Meta, const SIZE: usize>(
-    [PrefixSet<AF, M>; SIZE],
-);
-
-impl<AF: AddressFamily, M: Meta, const SIZE: usize>
-    FamilyCHT<AF, PrefixSet<AF, M>> for PrefixCHT<AF, M, SIZE>
-{
-    fn init() -> Self {
-        Self(std::array::from_fn::<_, SIZE, _>(|_| {
-            PrefixSet::<AF, M>::init(4)
-        }))
-    }
-
-    fn root_for_len(&self, len: u8) -> &PrefixSet<AF, M> {
-        &self.0[len as usize]
-    }
-
-    fn bits_for_len(len: u8, lvl: u8) -> u8 {
-        let res = 4 * (lvl + 1);
-        if res < len {
-            res
-        } else if res >= len + 4 {
-            0
-        } else {
-            len
-        }
-    }
-}
+// #[derive(Debug)]
+// pub(crate) struct PrefixCHT<AF: AddressFamily, M: Meta, const SIZE: usize>(
+//     [PrefixSet<AF, M>; SIZE],
+// );
+
+// impl<AF: AddressFamily, M: Meta, const SIZE: usize>
+//     FamilyCHT<AF, PrefixSet<AF, M>> for PrefixCHT<AF, M, SIZE>
+// {
+//     fn init() -> Self {
+//         Self(std::array::from_fn::<_, SIZE, _>(|_| {
+//             PrefixSet::<AF, M>::init(STRIDE_SIZE)
+//         }))
+//     }
+
+//     fn root_for_len(&self, len: u8) -> &PrefixSet<AF, M> {
+//         &self.0[len as usize]
+//     }
+
+//     fn bits_for_len(len: u8, lvl: u8) -> u8 {
+//         let res = STRIDE_SIZE * (lvl + 1);
+//         if res < len {
+//             res
+//         } else if res >= len + 4 {
+//             0
+//         } else {
+//             len
+//         }
+//     }
+// }
 
 pub struct DefaultStore<M: Meta, C: Config> {
-    v4: Rib<IPv4, M, NodeCHT<IPv4, 9>, PrefixCHT<IPv4, M, 33>, C, 18>,
-    v6: Rib<IPv6, M, NodeCHT<IPv6, 33>, PrefixCHT<IPv6, M, 129>, C, 30>,
+    v4: Rib<IPv4, M, 9, 33, C, 18>,
+    v6: Rib<IPv6, M, 33, 129, C, 30>,
     config: C,
 }
 
diff --git a/src/local_array/rib/init b/src/local_array/rib/init
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/local_array/rib/init
@@ -0,0 +1 @@
+
diff --git a/src/local_array/rib/rib.rs b/src/local_array/rib/rib.rs
index e22c8776..99d91bfb 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/local_array/rib/rib.rs
@@ -10,18 +10,16 @@ use epoch::{Guard, Owned};
 use zerocopy::TryFromBytes;
 
 use crate::local_array::in_memory::tree::TreeBitMap;
-use crate::local_array::persist::lsm_tree::{KeySize, LongKey};
-use crate::local_array::prefix_cht::cht::PrefixCHT;
+use crate::local_array::persist::lsm_tree::LongKey;
+use crate::local_array::prefix_cht::cht::PrefixCht;
 use crate::local_array::types::PrefixId;
 use crate::prefix_record::{ValueHeader, ZeroCopyRecord};
-use crate::prelude::multi::{PrefixSet, RouteStatus};
+use crate::prelude::multi::RouteStatus;
 use crate::stats::CreatedNodes;
 use crate::{
     local_array::errors::PrefixStoreError, prefix_record::PublicRecord,
 };
 
-use crate::local_array::in_memory::atomic_types::{FamilyCHT, NodeSet};
-
 // Make sure to also import the other methods for the Rib, so the proc macro
 // create_store can use them.
 pub use crate::local_array::in_memory::iterators;
@@ -328,14 +326,14 @@ pub struct UpsertReport {
 pub struct Rib<
     AF: AddressFamily,
     M: Meta,
-    NB: FamilyCHT<AF, NodeSet<AF>>,
-    PB: FamilyCHT<AF, PrefixSet<AF, M>>,
+    const N_ROOT_SIZE: usize,
+    const P_ROOT_SIZE: usize,
     C: Config,
     const KEY_SIZE: usize,
 > {
     pub config: C,
-    pub(crate) in_memory_tree: TreeBitMap<AF, NB>,
-    pub(crate) prefix_cht: PrefixCHT<AF, M, PB>,
+    pub(crate) in_memory_tree: TreeBitMap<AF, N_ROOT_SIZE>,
+    pub(crate) prefix_cht: PrefixCht<AF, M, P_ROOT_SIZE>,
     pub(in crate::local_array) persist_tree:
         Option<PersistTree<AF, LongKey<AF>, KEY_SIZE>>,
     pub counters: Counters,
@@ -344,18 +342,19 @@ pub struct Rib<
 impl<
         AF: AddressFamily,
         M: Meta,
-        NB: FamilyCHT<AF, NodeSet<AF>>,
-        PB: FamilyCHT<AF, PrefixSet<AF, M>>,
+        const P_ROOT_SIZE: usize,
+        const N_ROOT_SIZE: usize,
         C: Config,
         const KEY_SIZE: usize,
-    > Rib<AF, M, NB, PB, C, KEY_SIZE>
+    > Rib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
 {
-    #[allow(clippy::type_complexity)]
     pub(crate) fn new(
         config: C,
-    ) -> Result<Rib<AF, M, NB, PB, C, KEY_SIZE>, Box<dyn std::error::Error>>
-    {
-        Rib::<AF, M, NB, PB, C, KEY_SIZE>::init(config)
+    ) -> Result<
+        Rib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>,
+        Box<dyn std::error::Error>,
+    > {
+        Rib::<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>::init(config)
     }
 
     fn init(config: C) -> Result<Self, Box<dyn std::error::Error>> {
@@ -372,10 +371,10 @@ impl<
 
         let store = Rib {
             config,
-            in_memory_tree: TreeBitMap::<AF, NB>::new()?,
+            in_memory_tree: TreeBitMap::<AF, N_ROOT_SIZE>::new()?,
             persist_tree,
             counters: Counters::default(),
-            prefix_cht: PrefixCHT::<AF, M, PB>::new(),
+            prefix_cht: PrefixCht::<AF, M, P_ROOT_SIZE>::init(),
         };
 
         Ok(store)
@@ -634,7 +633,6 @@ impl<
         let mut new = unsafe { current.as_ref() }.unwrap().clone();
         new.insert(mui);
 
-        #[allow(clippy::assigning_clones)]
         loop {
             match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
                 current,
@@ -667,7 +665,6 @@ impl<
         let mut new = unsafe { current.as_ref() }.unwrap().clone();
         new.remove(mui);
 
-        #[allow(clippy::assigning_clones)]
         loop {
             match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
                 current,
@@ -821,11 +818,12 @@ impl<
 
 impl<
         M: Meta,
-        NB: FamilyCHT<IPv4, NodeSet<IPv4>>,
-        PB: FamilyCHT<IPv4, PrefixSet<IPv4, M>>,
+        const N_ROOT_SIZE: usize,
+        const P_ROOT_SIZE: usize,
         C: Config,
         const KEY_SIZE: usize,
-    > std::fmt::Display for Rib<IPv4, M, NB, PB, C, KEY_SIZE>
+    > std::fmt::Display
+    for Rib<IPv4, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
 {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "Rib<IPv4, {}>", std::any::type_name::<M>())
@@ -834,11 +832,12 @@ impl<
 
 impl<
         M: Meta,
-        NB: FamilyCHT<IPv6, NodeSet<IPv6>>,
-        PB: FamilyCHT<IPv6, PrefixSet<IPv6, M>>,
+        const N_ROOT_SIZE: usize,
+        const P_ROOT_SIZE: usize,
         C: Config,
         const KEY_SIZE: usize,
-    > std::fmt::Display for Rib<IPv6, M, NB, PB, C, KEY_SIZE>
+    > std::fmt::Display
+    for Rib<IPv6, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
 {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "Rib<IPv6, {}>", std::any::type_name::<M>())
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
index 21326c67..243a62b3 100644
--- a/src/prelude/mod.rs
+++ b/src/prelude/mod.rs
@@ -13,9 +13,6 @@ pub mod multi {
     pub use crossbeam_epoch::{self as epoch, Guard};
 
     pub use crate::local_array::errors::PrefixStoreError;
-    pub use crate::local_array::in_memory::atomic_types::{
-        FamilyCHT, NodeSet, PrefixSet,
-    };
     pub use crate::local_array::in_memory::iterators;
     pub use crate::local_array::in_memory::node::StrideNodeId;
     pub use crate::local_array::persist::lsm_tree::KeySize;
diff --git a/src/rotonda_store.rs b/src/rotonda_store.rs
index 307d5f2e..58b002d4 100644
--- a/src/rotonda_store.rs
+++ b/src/rotonda_store.rs
@@ -1,10 +1,10 @@
 use std::{fmt, slice};
 
+use crate::prefix_record::InternalPrefixRecord;
 pub use crate::prefix_record::{
     Meta, PublicPrefixSingleRecord, RecordSingleSet,
 };
 use crate::prefix_record::{PublicRecord, RecordSet};
-use crate::{prefix_record::InternalPrefixRecord, stats::StrideStats};
 
 use inetnum::addr::Prefix;
 
@@ -21,45 +21,45 @@ pub use crate::local_array::rib::DefaultStore as MultiThreadedStore;
 
 //------------ Types for strides displaying/monitoring ----------------------
 
-type AfStrideStats<AF> = Vec<StrideStats<AF>>;
-
-pub struct Stats<'a> {
-    pub v4: &'a AfStrideStats<IPv4>,
-    pub v6: &'a AfStrideStats<IPv6>,
-}
-
-impl std::fmt::Display for Stats<'_> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(f, "v4 ")?;
-        for s in self.v4.iter() {
-            writeln!(f, "{} ", s)?;
-        }
-        writeln!(f, "v6 ")?;
-        for s in self.v6.iter() {
-            writeln!(f, "{} ", s)?;
-        }
-        Ok(())
-    }
-}
-
-pub struct Strides<'a> {
-    pub v4: &'a Vec<u8>,
-    pub v6: &'a Vec<u8>,
-}
-
-impl std::fmt::Debug for Strides<'_> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "v4 ")?;
-        for s in self.v4.iter() {
-            write!(f, "{} ", s)?;
-        }
-        writeln!(f, "v5 ")?;
-        for s in self.v6.iter() {
-            write!(f, "{} ", s)?;
-        }
-        Ok(())
-    }
-}
+// type AfStrideStats<AF> = Vec<StrideStats<AF>>;
+
+// pub struct Stats<'a> {
+//     pub v4: &'a AfStrideStats<IPv4>,
+//     pub v6: &'a AfStrideStats<IPv6>,
+// }
+
+// impl std::fmt::Display for Stats<'_> {
+//     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+//         writeln!(f, "v4 ")?;
+//         for s in self.v4.iter() {
+//             writeln!(f, "{} ", s)?;
+//         }
+//         writeln!(f, "v6 ")?;
+//         for s in self.v6.iter() {
+//             writeln!(f, "{} ", s)?;
+//         }
+//         Ok(())
+//     }
+// }
+
+// pub struct Strides<'a> {
+//     pub v4: &'a Vec<u8>,
+//     pub v6: &'a Vec<u8>,
+// }
+
+// impl std::fmt::Debug for Strides<'_> {
+//     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+//         write!(f, "v4 ")?;
+//         for s in self.v4.iter() {
+//             write!(f, "{} ", s)?;
+//         }
+//         writeln!(f, "v5 ")?;
+//         for s in self.v6.iter() {
+//             write!(f, "{} ", s)?;
+//         }
+//         Ok(())
+//     }
+// }
 
 //------------ MatchOptions / MatchType -------------------------------------
 

From bd5a8d33ca7131aa6c173667d6b210238cc82603 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Mar 2025 15:33:29 +0100
Subject: [PATCH 102/147] delete superfluous macros

---
 src/local_array/in_memory/macros.rs | 277 ---------------------
 src/local_array/rib/macros.rs       | 367 ----------------------------
 2 files changed, 644 deletions(-)
 delete mode 100644 src/local_array/in_memory/macros.rs
 delete mode 100644 src/local_array/rib/macros.rs

diff --git a/src/local_array/in_memory/macros.rs b/src/local_array/in_memory/macros.rs
deleted file mode 100644
index b01ef872..00000000
--- a/src/local_array/in_memory/macros.rs
+++ /dev/null
@@ -1,277 +0,0 @@
-// #[macro_export]
-// // This macro expands into a match node {}
-// // with match arms for all SizedStrideNode::Stride[3-8]
-// // for use in insert()
-// #[doc(hidden)]
-// macro_rules! insert_match {
-//     (
-//         $self: ident;
-//         $guard: ident;
-//         $bit_span: expr;
-//         $is_last_stride: expr;
-//         $pfx: ident; // the whole search prefix
-//         $mui: ident; // the reccord holding the metadata
-//         $truncate_len: ident; // the start of the length of this stride
-//         $stride_len: ident; // the length of this stride
-//         $cur_i: expr; // the id of the current node in this stride
-//         $level: expr;
-//         $acc_retry_count: expr;
-//         // The strides to generate match arms for,
-//         // $variant is the name of the enum varian (Stride[3..8]) and
-//         // $len is the index of the stats level, so 0..5
-//         $( $variant: ident; $stats_level: expr ), *
-//     ) => {
-//         // Look up the current node in the store. This should never fail,
-//         // since we're starting at the root node and retrieve that. If a node
-//         // does not exist, it is created here. BUT, BUT, in a multi-threaded
-//         // context, one thread creating the node may be outpaced by a thread
-//         // reading the same node. Because the creation of a node actually
-//         // consists of two independent atomic operations (first setting the
-//         // right bit in the parent bitarray, second storing the node in the
-//         // store with the meta-data), a thread creating a new node may have
-//         // altered the parent bitarray, but not it didn't create the node
-//         // in the store yet. The reading thread, however, saw the bit in the
-//         // parent and wants to read the node in the store, but that doesn't
-//         // exist yet. In that case, the reader thread needs to try again
-//         // until it is actually created
-
-//         // This macro counts the number of retries and adds that to the
-//         // $acc_retry_count variable, to be used by the incorporating
-//         // function.
-//         {
-//             // this counts the number of retry_count for this loop only,
-//             // but ultimately we will return the accumulated count of all
-//             // retry_count from this macro.
-//             let local_retry_count = 0;
-//             // retrieve_node_mut updates the bitmap index if necessary.
-//             if let Some(current_node) = $self.retrieve_node_mut(
-//                 $cur_i, $mui) {
-//                 match current_node {
-//                     $(
-//                         SizedStrideRef::$variant(current_node) => {
-//                             // eval_node_or_prefix_at mutates the node to
-//                             // reflect changes in the ptrbitarr & pfxbitarr.
-//                             match current_node.eval_node_or_prefix_at(
-//                                 $bit_span,
-//                                 // All the bits of the search prefix, but with
-//                                 // a length set to the start of the current
-//                                 // stride.
-//                                 StrideNodeId::dangerously_new_with_id_as_is(
-//                                     $pfx.get_net(), $truncate_len),
-//                                 // the length of THIS stride
-//                                 $stride_len,
-//                                 // the length of the next stride
-//                                 $self
-//                                     .get_stride_sizes()
-//                                     .get(($level + 1) as usize),
-//                                 $is_last_stride,
-//                             ) {
-//                                 (NewNodeOrIndex::NewNode(n), retry_count) => {
-//                                     // Stride3 logs to stats[0], Stride4 logs
-//                                     // to stats[1], etc.
-//                                     // $self.stats[$stats_level].inc($level);
-
-//                                     // get a new identifier for the node we're
-//                                     // going to create.
-//                                     let new_id =
-//                                         StrideNodeId::new_with_cleaned_id(
-//                                             $pfx.get_net(),
-//                                             $truncate_len + $bit_span.len
-//                                        );
-
-//                                     // store the new node in the in_memory
-//                                     // part of the RIB. It returns the created
-//                                     // id and the number of retries before
-//                                     // success.
-//                                     match $self.store_node(
-//                                         new_id,
-//                                         $mui, n
-//                                     ) {
-//                                         Ok((node_id, s_retry_count)) => {
-//                                             Ok((
-//                                                 node_id,
-//                                                 $acc_retry_count +
-//                                                 s_retry_count +
-//                                                 retry_count
-//                                             ))
-//                                         },
-//                                         Err(err) => {
-//                                             Err(err)
-//                                         }
-//                                     }
-//                                 }
-//                                 (NewNodeOrIndex::ExistingNode(node_id),
-//                                     retry_count
-//                                 ) => {
-//                                     if log_enabled!(log::Level::Trace) {
-//                                         if local_retry_count > 0 {
-//                                             trace!("{} contention: Node \
-//                                                  already exists {}",
-//                                             std::thread::current()
-//                                                 .name()
-//                                                 .unwrap_or("unnamed-thread"),
-//                                                 node_id
-//                                             )
-//                                         }
-//                                     }
-//                                     Ok((
-//                                         node_id,
-//                                         $acc_retry_count +
-//                                         local_retry_count +
-//                                         retry_count
-//                                     ))
-//                                 },
-//                                 (NewNodeOrIndex::NewPrefix, retry_count) => {
-//                                     break (
-//                                         $acc_retry_count +
-//                                         local_retry_count +
-//                                         retry_count,
-//                                         false
-//                                     )
-
-//                                 },
-//                                 (
-//                                     NewNodeOrIndex::ExistingPrefix,
-//                                      retry_count
-//                                  ) =>
-//                                      {
-//                                     break (
-//                                         $acc_retry_count +
-//                                         local_retry_count +
-//                                         retry_count,
-//                                         true
-//                                     )
-
-//                                 }
-//                             }   // end of eval_node_or_prefix_at
-//                         }
-//                     )*,
-//                 }
-//             } else {
-//                 Err(PrefixStoreError::NodeCreationMaxRetryError)
-//             }
-//         }
-//     }
-// }
-
-#[macro_export]
-// This macro only works for stride with bitmaps that are <= u128,
-// the ones with synthetic integers (U256, U512) don't have the trait
-// implementations for left|right shift, counting ones etc.
-#[doc(hidden)]
-macro_rules! impl_primitive_atomic_stride {
-    (
-        $(
-            $len: expr;
-            $bits: expr;
-            $pfxsize: ty;
-            $atomicpfxsize: ty;
-            $ptrsize: ty;
-            $atomicptrsize: ty
-        ),
-    *) => {
-        $(
-            impl Stride for $pfxsize {
-                type AtomicPfxSize = $atomicpfxsize;
-                type AtomicPtrSize = $atomicptrsize;
-                type PtrSize = $ptrsize;
-                const BITS: u8 = $bits;
-                const STRIDE_LEN: u8 = $len;
-
-                fn get_bit_pos(bs: BitSpan) -> u32 {
-                    // trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
-                    1 << (
-                            <Self as Stride>::BITS - ((1 << bs.len) - 1) as u8
-                            - bs.bits as u8 - 1
-                    )
-                }
-
-                fn bit_pos_from_index(i: u8) -> u32 {
-                    <u32>::try_from(1).unwrap().rotate_right(1) >> i
-                }
-
-                fn ptr_bit_pos_from_index(i: u8) -> u16 {
-                    // trace!("pfx {} ptr {} strlen {}",
-                    // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
-                    <u16>::try_from(1).unwrap().rotate_right(1)
-                        >> (i + 1)
-                }
-
-                fn cursor_from_bit_span(bs: BitSpan) -> u8 {
-                    Self::get_bit_pos(bs)
-                    .leading_zeros() as u8
-                }
-
-                fn ptr_range(
-                    ptrbitarr: u16,
-                    bs: BitSpan
-                ) -> (u16, u8) {
-                    let start: u8 = (bs.bits << (4 - bs.len)) as u8;
-                    let stop: u8 = start + (1 << (4 - bs.len));
-                    let mask: u16 = (
-                        (((1_u32 << (stop as u32 - start as u32)) - 1)
-                             as u32
-                    )
-                        .rotate_right(stop as u32) >> 16)
-                        .try_into()
-                        .unwrap();
-                    trace!("- mask      {:032b}", mask);
-                    trace!("- ptrbitarr {:032b}", ptrbitarr);
-                    trace!("- shl bitar {:032b}", ptrbitarr & mask);
-
-                    // if ptrbitarr & mask == <$ptrsize>::zero() { panic!("stop"); }
-
-                    (ptrbitarr & mask, start as u8)
-                }
-
-                fn ms_pfx_mask(
-                    pfxbitarr: u32,
-                    bs: BitSpan
-                ) -> u32 {
-                    <u32>::try_from(
-                        $crate::local_array::in_memory::node::
-                        ms_prefix_mask_arr(bs) & pfxbitarr as u32
-                    ).unwrap()
-                }
-
-                fn get_bit_pos_as_u8(nibble: u32, len: u8) -> u8 {
-                    1 << (
-                            STRIDE_BITS - ((1 << len) - 1) as u8
-                            - nibble as u8 - 1
-                    )
-                }
-
-                fn get_pfx_index(bs: BitSpan)
-                -> usize {
-                    (Self::get_bit_pos(bs).leading_zeros() - 1) as usize
-
-                }
-
-                fn get_ptr_index(_bitmap: $ptrsize, nibble: u32) -> usize {
-                    (nibble as u16).into()
-                }
-
-                fn into_node_id<AF: AddressFamily>(
-                    addr_bits: AF,
-                    len: u8
-                ) -> StrideNodeId<AF> {
-                    let id = StrideNodeId::new_with_cleaned_id(addr_bits, len);
-                    id
-                }
-
-                fn into_stride_size(bitmap: u16) -> u32 {
-                    (bitmap as u32) << 1
-                }
-
-                fn into_ptrbitarr_size(bitmap: u32) -> u16 {
-                    (bitmap >> 1) as u16
-                }
-
-                #[inline]
-                fn leading_zeros(self) -> u32 {
-                    self.leading_zeros()
-                }
-            }
-        )*
-    };
-}
diff --git a/src/local_array/rib/macros.rs b/src/local_array/rib/macros.rs
deleted file mode 100644
index c8bfb201..00000000
--- a/src/local_array/rib/macros.rs
+++ /dev/null
@@ -1,367 +0,0 @@
-// #[macro_export]
-// #[doc(hidden)]
-// macro_rules! impl_search_level {
-//     (
-//         $(
-//             $stride: ident;
-//             $id: ident;
-//         ),
-//     * ) => {
-//         $(
-//         SearchLevel {
-//             f: &|search_level: &SearchLevel<AF, $stride>,
-//                 nodes,
-//                 mut level: u8,
-//             | {
-//                     // HASHING FUNCTION
-//                     let index = Self::hash_node_id($id, level);
-
-//                     match nodes.read().get(index) {
-//                         None => None,
-//                         Some(stored_node) => {
-//                             let StoredNode {
-//                                 node_id, node, node_set, .. } = stored_node;
-//                             if $id == *node_id {
-//                                 // YES, It's the one we're looking for!
-//                                 return Some(SizedStrideRef::$stride(&node));
-//                             };
-//                             // Meh, it's not, but we can a go to the next
-//                             // level and see if it lives there.
-//                             level += 1;
-//                             match <NB as NodeBuckets<AF>>::len_to_store_bits(
-//                                 $id.get_id().1, level
-//                             ) {
-//                                 // on to the next level!
-//                                 next_bit_shift if next_bit_shift > 0 => {
-//                                     (search_level.f)(
-//                                         search_level,
-//                                         &node_set,
-//                                         level,
-//                                     )
-//                                 }
-//                                 // There's no next level, we found nothing.
-//                                 _ => None,
-//                             }
-//                         }
-//                     }
-//                 }
-//         }
-//         )*
-//     };
-// }
-
-// #[macro_export]
-// #[doc(hidden)]
-// macro_rules! impl_search_level_for_mui {
-//     (
-//         $(
-//             $stride: ident;
-//             $id: ident;
-//             $mui: ident;
-//         ),
-//     * ) => {
-//         $(
-//         SearchLevel {
-//             f: &|search_level: &SearchLevel<AF, $stride>,
-//                 nodes,
-//                 mut level: u8| {
-//                     // HASHING FUNCTION
-//                     let index = Self::hash_node_id($id, level);
-
-//                     match nodes.read().get(index) {
-//                         None => None,
-//                         Some(this_node) => {
-//                             let StoredNode {
-//                                 node_id, node, node_set, .. } = this_node;
-
-//                             // early return if the mui is not in the index
-//                             // stored in this node, meaning the mui does not
-//                             // appear anywhere in the sub-tree formed from
-//                             // this node.
-//                             let bmin = node_set.rbm().read().unwrap();
-//                             if !bmin.contains($mui) {
-//                                 return None;
-//                             }
-
-//                             if $id == *node_id {
-//                                 // YES, It's the one we're looking for!
-//                                 return Some(SizedStrideRef::$stride(&node));
-//                             };
-//                             // Meh, it's not, but we can a go to the next
-//                             // level and see if it lives there.
-//                             level += 1;
-//                             match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
-//                                 // on to the next level!
-//                                 next_bit_shift if next_bit_shift > 0 => {
-//                                     (search_level.f)(
-//                                         search_level,
-//                                         &node_set,
-//                                         level,
-//                                     )
-//                                 }
-//                                 // There's no next level, we found nothing.
-//                                 _ => None,
-//                             }
-//                         }
-//                     }
-//                 }
-//         }
-//         )*
-//     };
-// }
-
-// // This macro creates a closure that is used in turn in the macro
-// // 'eBox', that is used in the public `insert` method on a TreeBitMap.
-// //
-// // It retrieves the node specified by $id recursively, creates it if it does
-// // not exist. It is responsible for setting/updating the RBMIN, but is does
-// // *not* set/update the pfxbitarr or ptrbitarr of the TreeBitMapNode. The
-// // `insert_match` takes care of the latter.
-// //
-// // This closure should not be called repeatedly to create the same node, if it
-// // returns `None` that is basically a data race in the store and therefore an
-// // error. Also the caller should make sure to stay within the limit of the
-// // defined number of levels, although the closure will return at the end of
-// // the maximum depth.
-// #[macro_export]
-// #[doc(hidden)]
-// macro_rules! retrieve_node_mut_closure {
-//     (
-//         $(
-//             $stride: ident;
-//             $id: ident;
-//             $multi_uniq_id: ident;
-//         ),
-//     * ) => {$(
-//         SearchLevel {
-//             f: &|
-//                 search_level: &SearchLevel<AF, $stride>,
-//                 nodes,
-//                 mut level: u8,
-//             | {
-//                 // HASHING FUNCTION
-//                 let index = Self::hash_node_id($id, level);
-//                 let node;
-
-//                 match nodes.read().get(index) {
-//                     // This arm only ever gets called in multi-threaded code
-//                     // where our thread (running this code *now*), andgot
-//                     // ahead of another thread: After the other thread created
-//                     // the TreeBitMapNode first, it was overtaken by our
-//                     // thread running this method, so our thread enounters an
-//                     // empty node in the store.
-//                     None => {
-//                         let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
-//                             $id.get_id().1, level
-//                         );
-//                         let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits(
-//                             $id.get_id().1, level + 1
-//                         );
-//                         let node_set = NodeSet::init(next_level - this_level);
-
-//                         // See if we can create the node
-//                         (node, _) = nodes.read().get_or_init(index, || StoredNode {
-//                             node_id: $id,
-//                             node: TreeBitMapNode::new(),
-//                             node_set
-//                         });
-
-//                         // We may have lost, and a different node than we
-//                         // intended could live here, if so go a level deeper
-//                         if $id == node.node_id {
-//                             // Nope, its ours or at least the node we need.
-//                             let _retry_count = node.node_set.update_rbm_index(
-//                                 $multi_uniq_id
-//                             ).ok();
-
-//                             return Some(SizedStrideRef::$stride(&node.node));
-//                         };
-//                     },
-//                     Some(this_node) => {
-//                         node = this_node;
-//                         if $id == this_node.node_id {
-//                             // YES, It's the one we're looking for!
-
-//                             // Update the rbm_index in this node with the
-//                             // multi_uniq_id that the caller specified. This
-//                             // is the only atomic operation we need to do
-//                             // here. The NodeSet that the index is attached
-//                             // to, does not need to be written to, it's part
-//                             // of a trie, so it just needs to "exist" (and it
-//                             // already does).
-//                             let retry_count = this_node.node_set.update_rbm_index(
-//                                 $multi_uniq_id
-//                             ).ok();
-
-//                             trace!("Retry_count rbm index {:?}", retry_count);
-//                             trace!("add multi uniq id to bitmap index {} for node {}",
-//                                 $multi_uniq_id, this_node.node
-//                             );
-//                             return Some(SizedStrideRef::$stride(&this_node.node));
-//                         };
-//                     }
-//                 }
-//                 // It isn't ours. Move one level deeper.
-//                 level += 1;
-//                 match <NB as NodeBuckets<AF>>::len_to_store_bits(
-//                     $id.get_id().1, level
-//                 ) {
-//                     // on to the next level!
-//                     next_bit_shift if next_bit_shift > 0 => {
-//                         (search_level.f)(
-//                             search_level,
-//                             &node.node_set,
-//                             level,
-//                         )
-//                     }
-//                     // There's no next level, we found nothing.
-//                     _ => None,
-//                 }
-//             }
-//         }
-//     )*};
-// }
-
-// #[macro_export]
-// #[doc(hidden)]
-// macro_rules! store_node_closure {
-//     (
-//         $(
-//             $stride: ident;
-//             $id: ident;
-//             // $multi_uniq_id: ident;
-//             $guard: ident;
-//             $back_off: ident;
-//         ),
-//     *) => {
-//         $(
-//         SearchLevel {
-//             f: &|
-//             search_level: &SearchLevel<AF, $stride>,
-//             nodes,
-//             new_node: TreeBitMapNode<AF, $stride>,
-//             multi_uniq_id: u32,
-//             mut level: u8,
-//             retry_count: u32| {
-//                 let this_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level);
-//                 trace!("{:032b}", $id.get_id().0);
-//                 trace!("id {:?}", $id.get_id());
-//                 trace!("multi_uniq_id {}", multi_uniq_id);
-
-//                 // HASHING FUNCTION
-//                 let index = Self::hash_node_id($id, level);
-
-//                 match nodes.read().get(index) {
-//                     None => {
-//                         // No node exists, so we create one here.
-//                         let next_level = <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level + 1);
-
-//                         if log_enabled!(log::Level::Trace) {
-//                             trace!("Empty node found, creating new node {} len{} lvl{}",
-//                                 $id, $id.get_id().1, level + 1
-//                             );
-//                             trace!("Next level {}",
-//                                 next_level
-//                             );
-//                             trace!("Creating space for {} nodes",
-//                                 if next_level >= this_level { 1 << (next_level - this_level) } else { 1 }
-//                             );
-//                         }
-
-//                         trace!("multi uniq id {}", multi_uniq_id);
-
-//                         let node_set = NodeSet::init(next_level - this_level);
-
-//                         let ptrbitarr = new_node.ptrbitarr.load();
-//                         let pfxbitarr = new_node.pfxbitarr.load();
-
-//                         let (stored_node, its_us) = nodes.read().get_or_init(
-//                             index,
-//                             || StoredNode {
-//                                 node_id: $id,
-//                                 node: new_node,
-//                                 node_set
-//                             }
-//                         );
-
-//                         if stored_node.node_id == $id {
-//                             stored_node.node_set.update_rbm_index(
-//                                 multi_uniq_id
-//                             )?;
-
-//                             if !its_us && ptrbitarr != 0 {
-//                                 stored_node.node.ptrbitarr.merge_with(ptrbitarr);
-//                             }
-
-//                             if !its_us && pfxbitarr != 0 {
-//                                 stored_node.node.pfxbitarr.merge_with(pfxbitarr);
-//                             }
-//                         }
-
-//                         return Ok(($id, retry_count));
-//                     }
-//                     Some(stored_node) => {
-//                         // A node exists, might be ours, might be
-//                         // another one.
-
-//                         if log_enabled!(log::Level::Trace) {
-//                             trace!("
-//                                 {} store: Node here exists {:?}",
-//                                     std::thread::current().name().unwrap_or("unnamed-thread"),
-//                                     stored_node.node_id
-//                             );
-//                             trace!("node_id {:?}", stored_node.node_id.get_id());
-//                             trace!("node_id {:032b}", stored_node.node_id.get_id().0);
-//                             trace!("id {}", $id);
-//                             trace!("     id {:032b}", $id.get_id().0);
-//                         }
-
-//                         // See if somebody beat us to creating our
-//                         // node already, if so, we still need to do
-//                         // work: we have to update the bitmap index
-//                         // with the multi_uniq_id we've got from the
-//                         // caller.
-//                         if $id == stored_node.node_id {
-//                             stored_node.node_set.update_rbm_index(
-//                                 multi_uniq_id
-//                             )?;
-
-//                             if new_node.ptrbitarr.load() != 0 {
-//                                 stored_node.node.ptrbitarr.merge_with(new_node.ptrbitarr.load());
-//                             }
-//                             if new_node.pfxbitarr.load() != 0 {
-//                                 stored_node.node.pfxbitarr.merge_with(new_node.pfxbitarr.load());
-//                             }
-
-//                             return Ok(($id, retry_count));
-//                         } else {
-//                             // it's not "our" node, make a (recursive)
-//                             // call to create it.
-//                             level += 1;
-//                             trace!("Collision with node_id {}, move to next level: {} len{} next_lvl{} index {}",
-//                                 stored_node.node_id, $id, $id.get_id().1, level, index
-//                             );
-
-//                             return match <NB as NodeBuckets<AF>>::len_to_store_bits($id.get_id().1, level) {
-//                                 // on to the next level!
-//                                 next_bit_shift if next_bit_shift > 0 => {
-//                                     (search_level.f)(
-//                                         search_level,
-//                                         &stored_node.node_set,
-//                                         new_node,
-//                                         multi_uniq_id,
-//                                         level,
-//                                         retry_count
-//                                     )
-//                                 }
-//                                 // There's no next level!
-//                                 _ => panic!("out of storage levels, current level is {}", level),
-//                             }
-//                         }
-//                     }
-//                 }
-//             }
-//         }
-//     )*
-//     };
-// }

From eff603f989a925c9187f55f81c32bd97c0c62339 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Mar 2025 15:34:06 +0100
Subject: [PATCH 103/147] renaming some stuff

---
 src/local_array/bit_span.rs          |  7 ++-
 src/local_array/in_memory/node.rs    |  4 +-
 src/local_array/rib/default_store.rs | 69 ++--------------------------
 src/local_array/rib/mod.rs           |  5 +-
 src/rotonda_store.rs                 | 45 +-----------------
 5 files changed, 13 insertions(+), 117 deletions(-)

diff --git a/src/local_array/bit_span.rs b/src/local_array/bit_span.rs
index 2a3dc2a6..a23a6681 100644
--- a/src/local_array/bit_span.rs
+++ b/src/local_array/bit_span.rs
@@ -1,4 +1,4 @@
-use super::rib::default_store::STRIDE_BITS;
+use super::rib::default_store::BIT_SPAN_SIZE;
 
 #[derive(Copy, Clone, Debug)]
 pub struct BitSpan {
@@ -38,7 +38,10 @@ impl BitSpan {
     }
 
     pub(crate) fn into_bit_pos(self) -> u32 {
-        1 << (STRIDE_BITS - ((1 << self.len) - 1) as u8 - self.bits as u8 - 1)
+        1 << (BIT_SPAN_SIZE
+            - ((1 << self.len) - 1) as u8
+            - self.bits as u8
+            - 1)
     }
 
     pub(crate) fn cursor_from_bit_span(self) -> u8 {
diff --git a/src/local_array/in_memory/node.rs b/src/local_array/in_memory/node.rs
index 3e6c1d84..260f25fd 100644
--- a/src/local_array/in_memory/node.rs
+++ b/src/local_array/in_memory/node.rs
@@ -14,7 +14,7 @@ use crate::af::AddressFamily;
 use crate::local_array::in_memory::tree::{
     bit_pos_from_index, ptr_bit_pos_from_index,
 };
-use crate::local_array::rib::default_store::{STRIDE_BITS, STRIDE_SIZE};
+use crate::local_array::rib::default_store::{BIT_SPAN_SIZE, STRIDE_SIZE};
 use crate::local_array::types::PrefixId;
 
 //------------ TreeBitMap Node ----------------------------------------------
@@ -368,7 +368,7 @@ impl<AF: AddressFamily> std::iter::Iterator
             self.start_cursor,
             <u8>::min(
                 (1 << (STRIDE_SIZE - self.start_bs.len)) + self.start_cursor,
-                STRIDE_BITS - 2
+                BIT_SPAN_SIZE - 2
             )
         );
 
diff --git a/src/local_array/rib/default_store.rs b/src/local_array/rib/default_store.rs
index 1f3be8b9..4e011907 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/local_array/rib/default_store.rs
@@ -3,76 +3,15 @@ use crate::prelude::*;
 use rand::prelude::*;
 
 pub const STRIDE_SIZE: u8 = 4;
-pub const STRIDE_BITS: u8 = 32;
-
-// #[derive(Debug)]
-// pub(crate) struct NodeCHT<AF: AddressFamily, const SIZE: usize>(
-//     [NodeSet<AF>; SIZE],
-// );
-
-// impl<AF: AddressFamily, const SIZE: usize> FamilyCHT<AF, NodeSet<AF>>
-//     for NodeCHT<AF, SIZE>
-// {
-//     fn init() -> Self {
-//         Self(std::array::from_fn::<_, SIZE, _>(|_| {
-//             NodeSet::<AF>::init(STRIDE_SIZE)
-//         }))
-//     }
-
-//     fn root_for_len(&self, len: u8) -> &NodeSet<AF> {
-//         &self.0[len as usize / STRIDE_SIZE as usize]
-//     }
-
-//     fn bits_for_len(len: u8, lvl: u8) -> u8 {
-//         let res = STRIDE_SIZE * (lvl + 1);
-//         if res < len {
-//             res
-//         } else if res >= len + STRIDE_SIZE {
-//             0
-//         } else {
-//             len
-//         }
-//     }
-// }
-
-// create a range p0..p32 for IPv4, and p0..p128 for IPv6
-// #[derive(Debug)]
-// pub(crate) struct PrefixCHT<AF: AddressFamily, M: Meta, const SIZE: usize>(
-//     [PrefixSet<AF, M>; SIZE],
-// );
-
-// impl<AF: AddressFamily, M: Meta, const SIZE: usize>
-//     FamilyCHT<AF, PrefixSet<AF, M>> for PrefixCHT<AF, M, SIZE>
-// {
-//     fn init() -> Self {
-//         Self(std::array::from_fn::<_, SIZE, _>(|_| {
-//             PrefixSet::<AF, M>::init(STRIDE_SIZE)
-//         }))
-//     }
-
-//     fn root_for_len(&self, len: u8) -> &PrefixSet<AF, M> {
-//         &self.0[len as usize]
-//     }
-
-//     fn bits_for_len(len: u8, lvl: u8) -> u8 {
-//         let res = STRIDE_SIZE * (lvl + 1);
-//         if res < len {
-//             res
-//         } else if res >= len + 4 {
-//             0
-//         } else {
-//             len
-//         }
-//     }
-// }
-
-pub struct DefaultStore<M: Meta, C: Config> {
+pub const BIT_SPAN_SIZE: u8 = 32;
+
+pub struct StarCastDb<M: Meta, C: Config> {
     v4: Rib<IPv4, M, 9, 33, C, 18>,
     v6: Rib<IPv6, M, 33, 129, C, 30>,
     config: C,
 }
 
-impl<'a, M: Meta, C: Config> DefaultStore<M, C> {
+impl<'a, M: Meta, C: Config> StarCastDb<M, C> {
     pub fn try_default() -> Result<Self, PrefixStoreError> {
         let config = C::default();
         Self::new_with_config(config)
diff --git a/src/local_array/rib/mod.rs b/src/local_array/rib/mod.rs
index 96373e90..b163e60f 100644
--- a/src/local_array/rib/mod.rs
+++ b/src/local_array/rib/mod.rs
@@ -3,7 +3,4 @@ pub mod rib;
 
 pub(crate) mod default_store;
 
-pub use default_store::DefaultStore;
-
-#[macro_use]
-mod macros;
+pub use default_store::StarCastDb;
diff --git a/src/rotonda_store.rs b/src/rotonda_store.rs
index 58b002d4..21a7e14d 100644
--- a/src/rotonda_store.rs
+++ b/src/rotonda_store.rs
@@ -16,50 +16,7 @@ pub const RECORDS_MAX_NUM: usize = 3;
 
 //------------ The publicly available Rotonda Stores ------------------------
 
-pub use crate::local_array::rib::DefaultStore as MultiThreadedStore;
-// pub use crate::local_vec::store::Store as SingleThreadedStore;
-
-//------------ Types for strides displaying/monitoring ----------------------
-
-// type AfStrideStats<AF> = Vec<StrideStats<AF>>;
-
-// pub struct Stats<'a> {
-//     pub v4: &'a AfStrideStats<IPv4>,
-//     pub v6: &'a AfStrideStats<IPv6>,
-// }
-
-// impl std::fmt::Display for Stats<'_> {
-//     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-//         writeln!(f, "v4 ")?;
-//         for s in self.v4.iter() {
-//             writeln!(f, "{} ", s)?;
-//         }
-//         writeln!(f, "v6 ")?;
-//         for s in self.v6.iter() {
-//             writeln!(f, "{} ", s)?;
-//         }
-//         Ok(())
-//     }
-// }
-
-// pub struct Strides<'a> {
-//     pub v4: &'a Vec<u8>,
-//     pub v6: &'a Vec<u8>,
-// }
-
-// impl std::fmt::Debug for Strides<'_> {
-//     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-//         write!(f, "v4 ")?;
-//         for s in self.v4.iter() {
-//             write!(f, "{} ", s)?;
-//         }
-//         writeln!(f, "v5 ")?;
-//         for s in self.v6.iter() {
-//             write!(f, "{} ", s)?;
-//         }
-//         Ok(())
-//     }
-// }
+pub use crate::local_array::rib::StarCastDb as MultiThreadedStore;
 
 //------------ MatchOptions / MatchType -------------------------------------
 

From 2423dd0de4324bc63d022d2a13fe897b8aa0d239 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Mar 2025 15:35:45 +0100
Subject: [PATCH 104/147] remove proc macros

---
 proc_macros/Cargo.toml   |   19 -
 proc_macros/Changelog.md |   41 -
 proc_macros/LICENSE      |   30 -
 proc_macros/README.md    |   24 -
 proc_macros/src/lib.rs   | 1781 --------------------------------------
 proc_macros/src/maps.rs  |  547 ------------
 6 files changed, 2442 deletions(-)
 delete mode 100644 proc_macros/Cargo.toml
 delete mode 100644 proc_macros/Changelog.md
 delete mode 100644 proc_macros/LICENSE
 delete mode 100644 proc_macros/README.md
 delete mode 100644 proc_macros/src/lib.rs
 delete mode 100644 proc_macros/src/maps.rs

diff --git a/proc_macros/Cargo.toml b/proc_macros/Cargo.toml
deleted file mode 100644
index 4606c638..00000000
--- a/proc_macros/Cargo.toml
+++ /dev/null
@@ -1,19 +0,0 @@
-[package]
-name = "rotonda-macros"
-categories = ["network-programming"]
-description = "Procedural macros for the rotonda-store prefix store"
-homepage = "https://nlnetlabs.nl/projects/routing/rotonda/"
-repository = "https://github.com/NLnetLabs/rotonda-macros"
-keywords = ["routing", "bgp"]
-edition.workspace = true
-version.workspace = true
-authors.workspace = true
-rust-version.workspace = true
-license.workspace = true
-
-[lib]
-proc-macro = true
-
-[dependencies]
-syn = {version = "^2", features = ["proc-macro", "full", "parsing", "printing"] }
-quote = "^1"
diff --git a/proc_macros/Changelog.md b/proc_macros/Changelog.md
deleted file mode 100644
index ad136953..00000000
--- a/proc_macros/Changelog.md
+++ /dev/null
@@ -1,41 +0,0 @@
-# Change Log
-
-## Unreleased Version
-
-Released xxxx-xx-xx.
-
-Breaking Changes
-
-New
-
-Other Changes
-
-## 0.4.0-rc0
-
-Released 2024-06-12.
-
-Breaking Changes
-
-* remove MergeUpdate trait
-
-New
-
-* public API for best and backup path selection.
-* public API for searching and iterating paths for multi_uniq_ids.
-* public API for modifying local and global multi_uniq_ids ('mui').
-
-## 0.3.1
-
-Released 2021-03-25.
-
-Other Changes
-
-* Use inetnum create for Prefix, instead of routecore.
-
-## 0.1.1
-
-Released 2021-06-29
-
-First Release
-
-* Meant to be used by rotonda-store < 0.3
diff --git a/proc_macros/LICENSE b/proc_macros/LICENSE
deleted file mode 100644
index 3f451536..00000000
--- a/proc_macros/LICENSE
+++ /dev/null
@@ -1,30 +0,0 @@
-Copyright (c) 2021, NLnet Labs. All rights reserved.
-
-This software is open source.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-Redistributions of source code must retain the above copyright notice,
-this list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-
-Neither the name of the NLNET LABS nor the names of its contributors may
-be used to endorse or promote products derived from this software without
-specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
-TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/proc_macros/README.md b/proc_macros/README.md
deleted file mode 100644
index fad76fb9..00000000
--- a/proc_macros/README.md
+++ /dev/null
@@ -1,24 +0,0 @@
-# `rotonda-macros` – Procedural macros for Rotonda-store
-
-This crate provides a few procedural macros for the `rotonda-store`.
-
-`rotonda-store` is a part of the Rotonda project, a modular, analytical
-BGP engine.
-
-Read more about [Rotonda].
-
-## Contributing
-
-If you have comments, proposed changes, or would like to contribute,
-please open an issue in the [GitHub repository]. In particular, if you
-would like to use the crate but it is missing functionality for your use
-case, we would love to hear from you!
-
-[GitHub repository]: (https://github.com/NLnetLabs/rotonda-macros)
-[Rotonda]: (https://github.com/NLnetLabs/rotonda)
-
-
-## License
-
-The _rotonda-macros_ crate is distributed under the terms of the BSD-3-clause license.
-See LICENSE for details.
diff --git a/proc_macros/src/lib.rs b/proc_macros/src/lib.rs
deleted file mode 100644
index 93edeb8d..00000000
--- a/proc_macros/src/lib.rs
+++ /dev/null
@@ -1,1781 +0,0 @@
-extern crate proc_macro;
-
-mod maps;
-
-use proc_macro::TokenStream;
-use quote::{format_ident, quote};
-use std::iter::Iterator;
-use syn::parse_macro_input;
-
-#[proc_macro_attribute]
-pub fn stride_sizes(
-    attr: TokenStream,
-    struct_def: TokenStream,
-) -> TokenStream {
-    // The arguments for the macro invocation
-    let attrs = parse_macro_input!(attr as syn::ExprTuple);
-
-    let attrs = attrs.elems.iter().collect::<Vec<_>>();
-
-    let struct_def = parse_macro_input!(struct_def as syn::ItemStruct);
-    let type_name = &struct_def.ident;
-
-    let ip_af = match attrs
-        .first()
-        .unwrap_or_else(|| panic!("Missing Address Family"))
-    {
-        syn::Expr::Path(t) => t,
-        _ => panic!("Expected Adress Family Type"),
-    };
-
-    let _prefix_size = match attrs
-        .get(2)
-        .unwrap_or_else(|| panic!("Missing Prefix Size for Address Family"))
-    {
-        syn::Expr::Lit(l) => l,
-        l => panic!("Expected Prefix Size for Address Family, got {:?}", l),
-    };
-
-    let key_size = match attrs
-        .get(3)
-        .unwrap_or_else(|| panic!("Missing Key Size for Address Family"))
-    {
-        syn::Expr::Lit(l) => l,
-        l => panic!("Expected Key Size for Address Family, got {:?}", l),
-    };
-
-    let key_type = match attrs
-        .get(4)
-        .unwrap_or_else(|| panic!("Missing Key Type for Persist Strategy"))
-    {
-        syn::Expr::Path(l) => l,
-        l => panic!("Expected Key type for Persist Strategy, got {:?}", l),
-    };
-
-    let prefixes_all_len;
-    let all_len;
-    let prefixes_buckets_name: syn::Ident;
-    let get_root_prefix_set;
-
-    // The name of the Struct that we're going to generate
-    // We'll prepend it with the name of the TreeBitMap struct
-    // that the user wants, so that our macro is a little bit
-    // more hygienic, and the user can create multiple types
-    // of TreeBitMap structs with different stride sizes.
-    let buckets_name = if ip_af.path.is_ident("IPv4") {
-        format_ident!("{}NodeBuckets4", type_name)
-    } else {
-        format_ident!("{}NodeBuckets6", type_name)
-    };
-    let store_bits = if ip_af.path.is_ident("IPv4") {
-        all_len = (0..=32_u8).collect::<Vec<_>>();
-        prefixes_all_len = (0..=32_u8)
-            .map(|l| format_ident!("p{}", l))
-            .collect::<Vec<_>>();
-        prefixes_buckets_name = format_ident!("PrefixBuckets4");
-        // prefix_store_bits = format_ident!("prefix_store_bits_4");
-        get_root_prefix_set = quote! {
-            fn get_root_prefix_set(&self, len: u8) -> &'_ PrefixSet<IPv4, M> {
-                [
-                    &self.p0, &self.p1, &self.p2, &self.p3, &self.p4, &self.p5, &self.p6, &self.p7, &self.p8,
-                    &self.p9, &self.p10, &self.p11, &self.p12, &self.p13, &self.p14, &self.p15, &self.p16,
-                    &self.p17, &self.p18, &self.p19, &self.p20, &self.p21, &self.p22, &self.p23, &self.p24,
-                    &self.p25, &self.p26, &self.p27, &self.p28, &self.p29, &self.p30, &self.p31, &self.p32
-                ][len as usize]
-            }
-        };
-        crate::maps::node_buckets_map_v4()
-    } else {
-        all_len = (0..=128_u8).collect::<Vec<_>>();
-        prefixes_all_len = (0..=128_u8)
-            .map(|l| format_ident!("p{}", l))
-            .collect::<Vec<_>>();
-
-        prefixes_buckets_name = format_ident!("PrefixBuckets6");
-        // prefix_store_bits = format_ident!("prefix_store_bits_6");
-        get_root_prefix_set = quote! {
-            fn get_root_prefix_set(&self, len: u8) -> &'_ PrefixSet<IPv6, M> {
-                [
-                    &self.p0, &self.p1, &self.p2, &self.p3, &self.p4, &self.p5, &self.p6, &self.p7, &self.p8,
-                    &self.p9, &self.p10, &self.p11, &self.p12, &self.p13, &self.p14, &self.p15, &self.p16,
-                    &self.p17, &self.p18, &self.p19, &self.p20, &self.p21, &self.p22, &self.p23, &self.p24,
-                    &self.p25, &self.p26, &self.p27, &self.p28, &self.p29, &self.p30, &self.p31, &self.p32,
-                    &self.p33, &self.p34, &self.p35, &self.p36, &self.p37, &self.p38, &self.p39, &self.p40,
-                    &self.p41, &self.p42, &self.p43, &self.p44, &self.p45, &self.p46, &self.p47, &self.p48,
-                    &self.p49, &self.p50, &self.p51, &self.p52, &self.p53, &self.p54, &self.p55, &self.p56,
-                    &self.p57, &self.p58, &self.p59, &self.p60, &self.p61, &self.p62, &self.p63, &self.p64,
-                    &self.p65, &self.p66, &self.p67, &self.p68, &self.p69, &self.p70, &self.p71, &self.p72,
-                    &self.p73, &self.p74, &self.p75, &self.p76, &self.p77, &self.p78, &self.p79, &self.p80,
-                    &self.p81, &self.p82, &self.p83, &self.p84, &self.p85, &self.p86, &self.p87, &self.p88,
-                    &self.p89, &self.p90, &self.p91, &self.p92, &self.p93, &self.p94, &self.p95, &self.p96,
-                    &self.p97, &self.p98, &self.p99, &self.p100, &self.p101, &self.p102, &self.p103, &self.p104,
-                    &self.p105, &self.p106, &self.p107, &self.p108, &self.p109, &self.p110, &self.p111, &self.p112,
-                    &self.p113, &self.p114, &self.p115, &self.p116, &self.p117, &self.p118, &self.p119, &self.p120,
-                    &self.p121, &self.p122, &self.p123, &self.p124, &self.p125, &self.p126, &self.p127, &self.p128
-                    ][len as usize]
-            }
-        };
-        crate::maps::node_buckets_map_v6()
-    };
-
-    let mut strides_num: Vec<u8> = vec![];
-    let mut strides = vec![];
-    let mut strides_all_len = vec![];
-    let mut strides_all_len_accu: Vec<u8> = vec![];
-    let mut strides_all_len_level = vec![];
-    let mut strides_len3 = vec![];
-    let mut strides_len3_l = vec![];
-    let mut strides_len4 = vec![];
-    let mut strides_len4_l = vec![];
-    let mut strides_len5 = vec![];
-    let mut strides_len5_l = vec![];
-
-    let mut s_accu = 0_u8;
-
-    let attrs_s = match attrs[1] {
-        syn::Expr::Array(arr) => arr,
-        _ => panic!("Expected an array"),
-    };
-    let strides_len = attrs_s.elems.len() as u8;
-
-    for (len, stride) in attrs_s.elems.iter().enumerate() {
-        strides_all_len.push(format_ident!("l{}", len));
-
-        match stride {
-            syn::Expr::Lit(s) => {
-                if let syn::Lit::Int(i) = &s.lit {
-                    let stride_len = i.base10_digits().parse::<u8>().unwrap();
-                    strides_num.push(stride_len);
-                    strides_all_len_level.push(format_ident!("l{}", s_accu));
-
-                    match stride_len {
-                        3 => {
-                            strides_len3.push(s_accu as usize);
-                            strides_len3_l.push(format_ident!("l{}", s_accu));
-                        }
-                        4 => {
-                            strides_len4.push(s_accu as usize);
-                            strides_len4_l.push(format_ident!("l{}", s_accu));
-                        }
-                        5 => {
-                            strides_len5.push(s_accu as usize);
-                            strides_len5_l.push(format_ident!("l{}", s_accu));
-                        }
-                        _ => panic!("Expected a stride of 3, 4 or 5"),
-                    };
-                    strides_all_len_accu.push(s_accu);
-
-                    s_accu += stride_len;
-                    strides.push(format_ident!("Stride{}", stride_len))
-                } else {
-                    panic!("Expected an integer")
-                }
-            }
-            _ => {
-                panic!("Expected a literal")
-            }
-        }
-    }
-
-    // Check if the strides division makes sense
-    let mut len_to_stride_arr = [0_u8; 128];
-    strides_all_len_accu
-        .iter()
-        .zip(strides_num.iter())
-        .for_each(|(acc, s)| {
-            len_to_stride_arr[*acc as usize] = *s;
-        });
-
-    // These are the stride sizes as an array of u8s, padded with 0s to the
-    // right. It's bounded to 42 u8s to avoid having to set a const generic
-    // on the type (which would have to be carried over to its parent). So
-    // if a 0 is encountered, it's the end of the strides.
-    let mut stride_sizes = [0; 42];
-    let (left, _right) = stride_sizes.split_at_mut(strides_len as usize);
-    left.swap_with_slice(&mut strides_num);
-
-    let struct_creation = quote! {
-
-        #[derive(Debug)]
-        pub(crate) struct #buckets_name<AF: AddressFamily> {
-            // created fields for each sub-prefix (StrideNodeId) length,
-            // with hard-coded field-names, like this:
-            // l0: NodeSet<AF>,
-            // l4: NodeSet<AF>,
-            // l8: NodeSet<AF>,
-            // ...
-            // l28: NodeSet<AF, Stride4>
-            # ( #strides_all_len_level: NodeSet<#ip_af>, )*
-            _af: PhantomData<AF>,
-            stride_sizes: [u8; 42],
-            strides_len: u8
-        }
-
-        #[derive(Debug)]
-        pub(crate) struct #prefixes_buckets_name<AF: AddressFamily, M: Meta> {
-            // creates a bucket for each prefix (PrefixId) length, with
-            // hard-coded field-names, like this:
-            // p0: PrefixSet<AF, M>,
-            // p1: PrefixSet<AF, M>,
-            // ...
-            // p32: PrefixSet<AF, M>,
-            #( #prefixes_all_len: PrefixSet<#ip_af, M>, )*
-            _af: PhantomData<AF>,
-            _m: PhantomData<M>,
-        }
-
-    };
-
-    let prefix_buckets_map = if ip_af.path.is_ident("IPv4") {
-        crate::maps::prefix_buckets_map_v4()
-    } else {
-        crate::maps::prefix_buckets_map_v6()
-    };
-
-    let prefix_buckets_impl = quote! {
-
-        impl<AF: AddressFamily, M: Meta> PrefixBuckets<#ip_af, M> for #prefixes_buckets_name<AF, M> {
-            fn init() -> #prefixes_buckets_name<AF, M> {
-                #prefixes_buckets_name {
-                    #( #prefixes_all_len: PrefixSet::init(1 << #prefixes_buckets_name::<AF, M>::get_bits_for_len(#all_len, 0)), )*
-                    _af: PhantomData,
-                    _m: PhantomData,
-                }
-            }
-
-            fn remove(&mut self, id: PrefixId<#ip_af>) -> Option<M> { unimplemented!() }
-
-            #get_root_prefix_set
-
-            #prefix_buckets_map
-
-        }
-
-    };
-
-    let struct_impl = quote! {
-
-        impl<AF: AddressFamily> NodeBuckets<#ip_af> for #buckets_name<AF> {
-            fn init() -> Self {
-                #buckets_name {
-                    // creates l0, l1, ... l<AF::BITS>, but only for the
-                    // levels at the end of each stride, so for strides
-                    // [5,5,4,3,3,3,3,3,3] is will create l0, l5, l10, l14,
-                    // l17, l20, l23, l26, l29 last level will be omitted,
-                    // because that will never be used (l29 has children
-                    // with prefixes up to prefix-length 32 in this example).
-                    #( #strides_all_len_level: NodeSet::init(#buckets_name::<AF>::len_to_store_bits(#strides_all_len_accu, 0) ), )*
-                    _af: PhantomData,
-                    stride_sizes: [ #( #stride_sizes, )*],
-                    strides_len: #strides_len
-                }
-            }
-
-            fn get_store(&self, id: StrideNodeId<#ip_af>) -> &NodeSet<#ip_af> {
-                match id.len() as usize {
-                    #( #strides_len4 => &self.#strides_len4_l, )*
-                    // ex.:
-                    // 10 => &self.l10,
-                    _ => panic!(
-                        "unexpected sub prefix length {} in stride size 4 ({})",
-                        id.len(),
-                        id
-                    ),
-                }
-            }
-
-            #[inline]
-            fn get_stride_sizes(&self) -> &[u8] {
-                &self.stride_sizes[0..self.strides_len as usize]
-            }
-
-            #[inline]
-            fn get_stride_for_id(&self, id: StrideNodeId<#ip_af>) -> u8 {
-                [ #(#len_to_stride_arr, )* ][id.len() as usize]
-            }
-
-            #[inline]
-            #store_bits
-
-            fn get_strides_len() -> u8 {
-                #strides_len
-            }
-
-            // fn get_first_stride_size() -> u8 {
-            //     #first_stride_size
-            // }
-        }
-
-    };
-
-    let type_alias = quote! {
-        type #type_name<M, C> = Rib<#ip_af, M, #key_type<#ip_af>, #buckets_name<#ip_af>, #prefixes_buckets_name<#ip_af, M>, C, #key_size>;
-    };
-
-    let result = quote! {
-        #struct_creation
-        #struct_impl
-        #prefix_buckets_impl
-        #type_alias
-    };
-
-    TokenStream::from(result)
-}
-
-// ---------- Create Store struct -------------------------------------------
-
-// This macro creates the struct that will be the public API for the
-// PrefixStore. It aspires to be a thin wrapper around the v4 and v6 stores,
-// so that users can use it AF agnostically. All methods defined in here
-// should be public.
-
-/// Creates a new, user-named struct with user-defined specified stride sizes
-/// that can used as a store type. The size of the prefix and the total key in
-/// the persisted storage should als be included, although these should
-/// probably not be changed from their defaults.
-///
-/// # Usage
-/// ```ignore
-/// use rotonda_store::prelude::*;
-/// use rotonda_store::prelude::multi::*;
-/// use rotonda_store::meta_examples::PrefixAs;
-///
-/// const IP4_STRIDE_ARRAY: [u32; 8] = [4; 8];
-/// const IP6_STRIDE_ARRAY: [u32; 32] = [4; 32];
-///
-/// #[create_store(((IPV4_STRIDE_ARRAY, 5, 17), (IPV6_STRIDE_ARRAY, 17, 29)))]
-/// struct NuStorage;
-/// ```
-///
-/// This will create a `NuStorage` struct, that can be used as a regular
-/// store.
-///
-/// The stride-sizes can be any of \[3,4,5\], and they should add up to the
-/// total number of bits in the address family (32 for IPv4 and 128 for IPv6).
-/// Stride sizes in the array will be repeated if the sum of them falls short
-/// of the total number of bits for the address family.i
-///
-/// The numbers 5, 17 after the first array for IPv4, and the numbers 17,
-/// 29 after the second array, represent the number of bytes a prefix in the
-/// key of a record in the persisted storage (disk), and the total number of
-/// bytes of the key of a record in the persisted storage. An IPv4 prefix is
-/// therefore 5 bytes (address part + prefix length), and 17 bytes for IPv6.
-/// The total number of bytes of the key is calculated thus: prefix (5 or 17)
-/// + multi_unique_id (4 bytes) + logical time of reception of the PDU into Rotonda (8 bytes).
-///
-/// # Example
-/// ```ignore
-/// use rotonda_store::prelude::*;
-/// use rotonda_store::prelude::multi::*;
-/// use rotonda_store::meta_examples::PrefixAs;
-///
-/// // The default stride sizes for IPv4, IPv6, resp.
-/// #[create_store((
-///     ([5, 5, 4, 3, 3, 3, 3, 3, 3, 3], 5, 17),
-///     ([4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-///     4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], 17, 29)
-/// ))]
-/// struct NuStore;
-///
-/// let store = Arc::new(NuStore::<PrefixAs>::new().unwrap());
-/// ```
-#[proc_macro_attribute]
-pub fn create_store(
-    attr: TokenStream,
-    struct_def: TokenStream,
-) -> TokenStream {
-    let struct_def = parse_macro_input!(struct_def as syn::ItemStruct);
-    let rib_name = &struct_def.ident;
-
-    let attr = parse_macro_input!(attr as syn::ExprTuple);
-    let attrs = attr.elems.iter().collect::<Vec<_>>();
-
-    let tuple_4 = attrs
-        .first()
-        .unwrap_or_else(|| panic!("No tuple ([u8], usize) for IPv4 defined"));
-    let tuple_4 = match tuple_4 {
-        syn::Expr::Tuple(t) => t,
-        t => panic!("Expected tuple ([u8], usize), got {:?}", t),
-    };
-
-    let tuple_6 = attrs
-        .get(1)
-        .unwrap_or_else(|| panic!("No tuple ([u8], usize) for IPv6 defined"));
-    let tuple_6 = match tuple_6 {
-        syn::Expr::Tuple(t) => t,
-        t => panic!("Expected tuple ([u8], usize), got {:?}", t),
-    };
-
-    let strides4 = tuple_4.elems.first().unwrap_or_else(|| {
-        panic!(
-            "Expected stride sizes array for IPv4, got {:?}",
-            tuple_4.attrs
-        )
-    });
-    let strides6 = tuple_6.elems.first().unwrap_or_else(|| {
-        panic!(
-            "Expected stride sizes array for IPv6, got {:?}",
-            tuple_6.attrs
-        )
-    });
-
-    let key_size4 = tuple_4.elems.get(1).unwrap_or_else(|| {
-        panic!("Expected Key Size for IPv4, got {:?}", tuple_4.elems)
-    });
-    let key_size6 = tuple_6.elems.get(1).unwrap_or_else(|| {
-        panic!("Expected Key Size for IPv6, got {:?}", tuple_6.attrs)
-    });
-
-    let prefix_size4 = tuple_4.elems.get(2).unwrap_or_else(|| {
-        panic!("Expected Prefix Size for IPv4, got {:?}", tuple_4.elems)
-    });
-    let prefix_size6 = tuple_6.elems.get(2).unwrap_or_else(|| {
-        panic!("Expected Prefix Size for IPv6, got {:?}", tuple_6.attrs)
-    });
-
-    // let config_type4 = tuple_4.elems.get(3).unwrap_or_else(|| {
-    //     panic!("Expected config type for IPv4, got {:?}", tuple_4.attrs)
-    // });
-
-    // let config_type6 = tuple_6.elems.get(3).unwrap_or_else(|| {
-    //     panic!("Expected config type for IPv6, got {:?}", tuple_6.attrs)
-    // });
-
-    let key_type4 = tuple_4.elems.get(3).unwrap_or_else(|| {
-        panic!("Expected Key Type, got {:?}", tuple_4.elems)
-    });
-
-    let key_type6 = tuple_6.elems.get(3).unwrap_or_else(|| {
-        panic!("Expected Key Type, got {:?}", tuple_6.elems)
-    });
-
-    let ipv4_rib_name = format_ident!("{}IPv4", rib_name);
-    let ipv6_rib_name = format_ident!("{}IPv6", rib_name);
-
-    let create_strides = quote! {
-        use ::std::marker::PhantomData;
-        use ::inetnum::addr::Prefix;
-
-        #[stride_sizes((IPv4, #strides4, #key_size4, #prefix_size4, #key_type4))]
-        struct #ipv4_rib_name;
-
-        #[stride_sizes((IPv6, #strides6, #key_size6, #prefix_size6, #key_type6))]
-        struct #ipv6_rib_name;
-    };
-
-    let store = quote! {
-        /// A concurrently read/writable, lock-free Prefix Store, for use in a
-        /// multi-threaded context.
-        ///
-        /// This store will hold records keyed on Prefix, and with values
-        /// consisting of a multi-map (a map that can hold multiple values per
-        /// key), filled with Records.
-        ///
-        /// Records in the store contain the metadata, a `multi_uniq_id`,
-        /// logical time (to disambiguate the order of inserts into the store)
-        /// and the status of the Record.
-        ///
-        /// Effectively this means that the store holds values for the set of
-        /// `(prefix, multi_uniq_id)` pairs, where the primary key is the
-        /// prefix, and the secondary key is the `multi_uniq_id`. These
-        /// `multi_uniq_id`s are unique across all of the store. The store
-        /// facilitates iterating over and changing the status for all
-        /// prefixes per `multi_uniq_id`.
-        ///
-        /// The store has the concept of a global status for a
-        /// `multi_uniq_id`, e.g. to set all prefixes for a `multi_uniq_id` in
-        /// one atomic transaction to withdrawn. It also has local statuses
-        /// per `(prefix, multi_uniq_id)` pairs, e.g. to withdraw one value
-        /// for a `multi_uniq_id`.
-        ///
-        /// This way the store can hold RIBs for multiple peers in one
-        /// data-structure.
-        pub struct #rib_name<
-            M: Meta,
-            C: Config
-        > {
-            v4: #ipv4_rib_name<M, C>,
-            v6: #ipv6_rib_name<M, C>,
-            config: C
-        }
-
-        impl<
-                M: Meta,
-                C: Config
-            > #rib_name<M, C>
-        {
-            /// Creates a new empty store with a tree for IPv4 and on for IPv6.
-            ///
-            /// The store will be created with the default stride sizes. After
-            /// creation you can wrap the store in an Arc<_> and `clone()` that
-            /// for every thread that needs read access and/or write acces to
-            /// it. As a convenience both read and write methods take a `&self`
-            /// instead of `&mut self`.
-            ///
-            /// If you need custom stride sizes you can use the
-            /// [`#[create_store]`](rotonda_macros::create_store) macro to
-            /// create a struct with custom stride sizes.
-            ///
-            /// # Example
-            /// ```
-            /// use std::{sync::Arc, thread};
-            /// use std::net::Ipv4Addr;
-            ///
-            /// use rotonda_store::prelude::*;
-            /// use rotonda_store::prelude::multi::*;
-            /// use rotonda_store::meta_examples::{NoMeta, PrefixAs};
-            ///
-            /// let tree_bitmap = Arc::new(MultiThreadedStore::<NoMeta>::new().unwrap());
-            ///
-            /// let _: Vec<_> = (0..16)
-            ///      .map(|_| {
-            ///         let tree_bitmap = clone();
-            ///
-            ///         thread::spawn(move || {
-            ///              let pfxs = [
-            ///                 Prefix::new_relaxed(
-            ///                     Ipv4Addr::new(130, 55, 241, 0).into(),
-            ///                     24,
-            ///                 ),
-            ///                 Prefix::new_relaxed(
-            ///                     Ipv4Addr::new(130, 55, 240, 0).into(),
-            ///                     24,
-            ///                 )
-            ///              ];
-            ///
-            ///              for pfx in pfxs.into_iter() {
-            ///                  println!("insert {}", pfx.unwrap());
-            ///                  insert(
-            ///                      &pfx.unwrap(),
-            ///                      Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
-            ///                      None
-            ///                  ).unwrap();
-            ///              }
-            ///          })
-            ///      }).map(|t| t.join()).collect();
-            /// ```
-            pub fn new_with_config(
-                    mut config: C
-                ) -> Result<Self, Box<dyn std::error::Error>> {
-
-                let rng = rand::rng();
-                let uuid: String = rng
-                    .sample_iter(
-                        rand::distr::Alphanumeric
-                    )
-                    .take(12)
-                    .map(|b| char::from(b))
-                    .collect();
-                let mut config_v4 = config.clone();
-                let mut config_v6 = config.clone();
-
-                if let Some(path) = config_v4.persist_path() {
-                    let pp = format!("{}/{}/ipv4/", path, uuid);
-                    config_v4.set_persist_path(pp);
-                };
-
-                if let Some(path) = config_v6.persist_path() {
-                    config_v6.set_persist_path(
-                        format!("{}/{}/ipv6/", path, uuid)
-                    );
-                }
-
-                Ok(Self {
-                    v4: #ipv4_rib_name::new(config_v4)?,
-                    v6: #ipv6_rib_name::new(config_v6)?,
-                    config
-                })
-            }
-
-            // pub fn new_with_short_key_persist(
-            //         mut config: StoreConfig
-            //     ) -> Result<Self, Box<dyn std::error::Error>> {
-
-            //     let rng = rand::rng();
-            //     let uuid: String = rng
-            //         .sample_iter(
-            //             rand::distr::Alphanumeric
-            //         )
-            //         .take(12)
-            //         .map(|b| char::from(b))
-            //         .collect();
-            //     let mut config_v4 = config.clone();
-            //     let mut config_v6 = config.clone();
-
-            //     config_v4.persist_path = format!(
-            //         "{}/{}/ipv4/", config_v4.persist_path, uuid);
-
-            //     config_v6.persist_path = format!(
-            //         "{}/{}/ipv6/", config.persist_path, uuid);
-
-            //     Ok(Self {
-            //         v4: #ipv4_rib_name::new_short_key(config_v4)?,
-            //         v6: #ipv6_rib_name::new_short_key(config_v6)?,
-            //         config
-            //     })
-            // }
-
-            // pub fn new_with_long_key_persist(
-            //         mut config: StoreConfig
-            //     ) -> Result<Self, Box<dyn std::error::Error>> {
-
-            //     let rng = rand::rng();
-            //     let uuid: String = rng
-            //         .sample_iter(
-            //             rand::distr::Alphanumeric
-            //         )
-            //         .take(12)
-            //         .map(|b| char::from(b))
-            //         .collect();
-            //     let mut config_v4 = config.clone();
-            //     let mut config_v6 = config.clone();
-
-            //     config_v4.persist_path = format!(
-            //         "{}/{}/ipv4/", config_v4.persist_path, uuid);
-
-            //     config_v6.persist_path = format!(
-            //         "{}/{}/ipv6/", config.persist_path, uuid);
-
-            //     Ok(Self {
-            //         v4: #ipv4_rib_name::new_long_key(config_v4)?,
-            //         v6: #ipv6_rib_name::new_long_key(config_v6)?,
-            //         config
-            //     })
-            // }
-        }
-
-        impl<'a, M: Meta, C: Config
-            > #rib_name<M, C>
-        {
-            /// Search for and return one or more prefixes that match the
-            ///given  `search_pfx` argument.   The search will return a
-            ///[QueryResult] with the matching prefix,  if any, the type of
-            ///match for the found prefix and the more and  less specifics for
-            ///the requested prefix. The inclusion of more-  or less-specifics
-            ///and the requested `match_type` is configurable  through the
-            ///[MatchOptions] argument.
-            ///
-            /// The `match_type` in the `MatchOptions` indicates what match
-            /// types can appear in the [QueryResult] result.
-            ///
-            /// `ExactMatch` is the most strict, and will only allow exactly
-            /// matching prefixes in the result. Failing an exacly matching
-            /// prefix, it will return an `EmptyMatch`.
-            ///
-            /// `LongestMatch` is less strict, and either an exactly matching
-            /// prefix or - in case there is no exact match - a longest
-            /// matching prefix will be allowed in the result. Failing both an
-            /// EmptyMatch will be returned.
-            ///
-            /// For both `ExactMatch` and `LongestMatch` the
-            /// `include_less_specifics` and `include_more_specifics`
-            /// options will be respected and the result will contain the
-            /// more and less specifics according to the options for the
-            /// requested prefix, even if the result returns a `match_type`
-            /// of `EmptyMatch`.
-            ///
-            /// `EmptyMatch` is the least strict, and will *always* return
-            /// the requested prefix, be it exactly matching, longest matching
-            /// or not matching at all (empty match), again, together with
-            /// its less|more specifics (if requested). Note that the last
-            /// option, the empty match in the result will never return
-            /// less-specifics, but can return more-specifics for a prefix
-            /// that itself is not present in the store.
-            ///
-            ///
-            /// This table sums it up:
-            ///
-            /// | query match_type | possible result types                      | less-specifics? | more-specifics? |
-            /// | ---------------- | ------------------------------------------ | --------------- | --------------- |
-            /// | `ExactMatch`     | `ExactMatch`, `EmptyMatch`                 | maybe           | maybe           |
-            /// | `LongestMatch`   | `ExactMatch`, `LongestMatch`, `EmptyMatch` | maybe           | maybe           |
-            /// | `EmptyMatch`     | `ExactMatch`, `LongestMatch`, `EmptyMatch` | no for EmptyM res, maybe for others | yes for EmptyM for res, maybe for others |
-            ///
-            ///
-            /// Note that the behavior of the CLI command `show route exact` on
-            /// most router platforms can be modeled by setting the `match_type`
-            /// to `ExactMatch` and `include_less_specifics` to `true`.
-            ///
-            /// # Example
-            /// ```
-            /// use std::net::Ipv4Addr;
-            ///
-            /// use rotonda_store::prelude::*;
-            /// use rotonda_store::meta_examples::PrefixAs;
-            /// use rotonda_store::prelude::multi::*;
-            ///
-            /// let store = MultiThreadedStore::<PrefixAs>::new().unwrap();
-            /// let guard = &epoch::pin();
-            ///
-            /// let pfx_addr = "185.49.140.0".parse::<Ipv4Addr>()
-            ///         .unwrap()
-            ///         .into();
-            ///
-            /// store.insert(
-            ///     &Prefix::new(pfx_addr, 22).unwrap(),
-            ///     Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321)),
-            ///     None
-            /// );
-            ///
-            /// let res = store.match_prefix(
-            ///     &Prefix::new(pfx_addr, 24).unwrap(),
-            ///     &MatchOptions {
-            ///         match_type: MatchType::LongestMatch,
-            ///         include_withdrawn: false,
-            ///         include_less_specifics: false,
-            ///         include_more_specifics: false,
-            ///         mui: None
-            ///     },
-            ///     guard
-            /// );
-            ///
-            /// assert_eq!(res.prefix_meta[0].meta.asn(), 211321.into());
-            ///
-            /// let res = store.match_prefix(
-            ///     &Prefix::new(pfx_addr, 24).unwrap(),
-            ///         &MatchOptions {
-            ///             match_type: MatchType::ExactMatch,
-            ///             include_withdrawn: false,
-            ///             include_less_specifics: false,
-            ///             include_more_specifics: false,
-            ///             mui: None
-            ///         },
-            ///         guard
-            ///     );
-            ///
-            /// assert!(res.match_type.is_empty());
-            ///
-            /// ```
-            pub fn match_prefix(
-                &'a self,
-                search_pfx: &Prefix,
-                options: &MatchOptions,
-                guard: &'a Guard,
-            ) -> QueryResult<M> {
-
-                match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => {
-                        self.v4.match_prefix(
-                            PrefixId::<IPv4>::new(
-                                <IPv4 as AddressFamily>::
-                                    from_ipaddr(addr),
-                                search_pfx.len(),
-                            ),
-                            options,
-                            guard
-                        ).into()
-                    },
-                    std::net::IpAddr::V6(addr) => {
-                        self.v6.match_prefix(
-                            PrefixId::<IPv6>::new(
-                                <IPv6 as AddressFamily>::
-                                    from_ipaddr(addr),
-                                search_pfx.len(),
-                            ),
-                            options,
-                            guard
-                        ).into()
-                    }
-                }
-            }
-
-
-            /// Return whether the requested prefix was ever created in the
-            /// RIB. Specifying a multi unique id will return whether the
-            /// (prefix, multi_uniq_id) tuple was ever created.
-            ///
-            /// The result ignores the status of the prefix, e.g. whether it
-            /// was withdrawn, or whether the global withdrawn for the multi
-            /// unique id is withdrawn.
-            pub fn contains(
-                &'a self,
-                prefix: &Prefix,
-                mui: Option<u32>
-            ) -> bool {
-                match prefix.addr() {
-                    std::net::IpAddr::V4(addr) => {
-                        self.v4.contains(
-                            PrefixId::<IPv4>::from(*prefix),
-                            mui
-                        )
-                    },
-                    std::net::IpAddr::V6(addr) => {
-                        self.v6.contains(
-                            PrefixId::<IPv6>::from(*prefix),
-                            mui
-                        )
-                    }
-                }
-            }
-
-            /// Return the record that belongs to the pre-calculated and
-            /// stored best path for a given prefix.
-            ///
-            /// If the Prefix does not exist in the store `None` is returned.
-            /// If the prefix does exist, but no best path was calculated
-            /// (yet), a `PrefixStoreError::BestPathNotFound` error will be
-            /// returned. A returned result of
-            /// `PrefixError::StoreNotReadyError` should never happen: it
-            /// would indicate an internal inconsistency in the store.
-            pub fn best_path(&'a self,
-                search_pfx: &Prefix,
-                guard: &Guard
-            ) -> Option<Result<Record<M>, PrefixStoreError>> {
-                match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => self.v4.best_path(
-                        PrefixId::<IPv4>::new(
-                            <IPv4 as AddressFamily>::
-                                from_ipaddr(addr),
-                            search_pfx.len(),
-                        ),
-                        guard
-                    ),
-                    std::net::IpAddr::V6(addr) => self.v6.best_path(
-                        PrefixId::<IPv6>::new(
-                            <IPv6 as AddressFamily>::
-                                from_ipaddr(addr),
-                            search_pfx.len(),
-                        ),
-                        guard
-                    )
-                }
-            }
-
-            /// Calculate and store the best path for the specified Prefix.
-            ///
-            /// If the result of the calculation is successful it will be
-            /// stored for the prefix. If they were set, it will return the
-            /// multi_uniq_id of the best path and the one for the backup
-            /// path, respectively. If the prefix does not exist in the store,
-            /// `None` will be returned. If the best path cannot be
-            /// calculated, a `Ok(None, None)` will be returned.
-            ///
-            /// Failing to calculate a best path, may be caused by
-            /// unavailability of any active paths, or by a lack of data (in
-            /// either the paths, or the supplied `TiebreakerInfo`).
-            ///
-            /// An Error result indicates an inconsistency in the store.
-            pub fn calculate_and_store_best_and_backup_path(
-                &self,
-                search_pfx: &Prefix,
-                tbi: &<M as Meta>::TBI,
-                guard: &Guard
-            ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
-                match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => self.v4
-                        .calculate_and_store_best_and_backup_path(
-                            PrefixId::<IPv4>::new(
-                                <IPv4 as AddressFamily>::
-                                    from_ipaddr(addr),
-                                search_pfx.len(),
-                            ),
-                            tbi,
-                            guard
-                        ),
-                    std::net::IpAddr::V6(addr) => self.v6
-                        .calculate_and_store_best_and_backup_path(
-                            PrefixId::<IPv6>::new(
-                                <IPv6 as AddressFamily>::
-                                    from_ipaddr(addr),
-                                search_pfx.len(),
-                            ),
-                            tbi,
-                            guard
-                        )
-                }
-            }
-
-            /// Return whether the best path selection, stored for this prefix
-            /// in the store is up to date with all the routes stored for
-            /// this prefix.
-            ///
-            /// Will return an error if the prefix does not exist in the store
-            ///
-            /// The `guard` should be a `&epoch::pin()`. It allows the
-            /// QuerySet to contain references to the meta-data objects,
-            /// instead of cloning them into it.
-            pub fn is_ps_outdated(
-                &self,
-                search_pfx: &Prefix,
-                guard: &Guard
-            ) -> Result<bool, PrefixStoreError> {
-                match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => self.v4
-                        .is_ps_outdated(
-                            PrefixId::<IPv4>::new(
-                                <IPv4 as AddressFamily>::
-                                    from_ipaddr(addr),
-                                search_pfx.len(),
-                            ),
-                            guard
-                        ),
-                    std::net::IpAddr::V6(addr) => self.v6
-                        .is_ps_outdated(
-                            PrefixId::<IPv6>::new(
-                                <IPv6 as AddressFamily>::
-                                    from_ipaddr(addr),
-                                search_pfx.len(),
-                            ),
-                            guard
-                        )
-                }
-            }
-
-            /// Return a [QueryResult] that contains all the more-specific
-            /// prefixes of the `search_pfx` in the store, including the
-            /// meta-data of these prefixes.
-            ///
-            /// The `search_pfx` argument can be either a IPv4 or an IPv6
-            /// prefix. The `search_pfx` itself doesn't have to be present
-            /// in the store for an iterator to be non-empty, i.e. if
-            /// more-specific prefixes exist for a non-existent
-            /// `search_pfx` the iterator will yield these more-specific
-            /// prefixes.
-            ///
-            /// The `guard` should be a `&epoch::pin()`. It allows the
-            /// QuerySet to contain references to the meta-data objects,
-            /// instead of cloning them into it.
-            pub fn more_specifics_from(&'a self,
-                search_pfx: &Prefix,
-                mui: Option<u32>,
-                include_withdrawn: bool,
-                guard: &'a Guard,
-            ) -> QueryResult<M> {
-
-                match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => self.v4.more_specifics_from(
-                        PrefixId::<IPv4>::new(
-                            <IPv4 as AddressFamily>::
-                                from_ipaddr(addr),
-                            search_pfx.len(),
-                        ),
-                        mui,
-                        include_withdrawn,
-                        guard
-                    ),
-                    std::net::IpAddr::V6(addr) => self.v6.more_specifics_from(
-                        PrefixId::<IPv6>::new(
-                            <IPv6 as AddressFamily>::
-                                from_ipaddr(addr),
-                            search_pfx.len(),
-                        ),
-                        mui,
-                        include_withdrawn,
-                        guard
-                    ),
-                }
-            }
-
-            /// Return a `QuerySet` that contains all the less-specific
-            /// prefixes of the `search_pfx` in the store, including the
-            /// meta-data of these prefixes.
-            ///
-            /// The `search_pfx` argument can be either a IPv4 or an IPv6
-            /// prefix. The `search_pfx` itself doesn't have to be present
-            /// in the store for an iterator to be non-empty, i.e. if
-            /// less-specific prefixes exist for a non-existent
-            /// `search_pfx` the iterator will yield these less-specific
-            /// prefixes.
-            ///
-            /// The `guard` should be a `&epoch::pin()`. It allows the
-            /// QuerySet to contain references to the meta-data objects,
-            /// instead of cloning them into it.
-            pub fn less_specifics_from(&'a self,
-                search_pfx: &Prefix,
-                mui: Option<u32>,
-                include_withdrawn: bool,
-                guard: &'a Guard,
-            ) -> QueryResult<M> {
-
-                match search_pfx.addr() {
-                    std::net::IpAddr::V4(addr) => self.v4.less_specifics_from(
-                        PrefixId::<IPv4>::new(
-                            <IPv4 as AddressFamily>::
-                                from_ipaddr(addr),
-                            search_pfx.len(),
-                        ),
-                        mui,
-                        include_withdrawn,
-                        guard
-                    ),
-                    std::net::IpAddr::V6(addr) => self.v6.less_specifics_from(
-                        PrefixId::<IPv6>::new(
-                            <IPv6 as AddressFamily>::
-                                from_ipaddr(addr),
-                            search_pfx.len(),
-                        ),
-                        mui,
-                        include_withdrawn,
-                        guard
-                    ),
-                }
-            }
-
-            /// Returns an iterator over all the less-specific prefixes
-            /// of the `search_prefix`, if present in the store, including
-            /// the meta-data of these prefixes.
-            ///
-            /// The `search_pfx` argument can be either a IPv4 or an IPv6
-            /// prefix. The `search_pfx` itself doesn't have to be present
-            /// in the store for an iterator to be non-empty, i.e. if
-            /// less-specific prefixes exist for a non-existent
-            /// `search_pfx` the iterator will yield these less-specific
-            /// prefixes.
-            ///
-            /// The `guard` should be a `&epoch::pin()`. It allows the
-            /// iterator to create and return references to the meta-data
-            /// objects to the caller (instead of cloning them).
-            ///
-            /// # Example
-            /// ```
-            /// use std::net::Ipv4Addr;
-            ///
-            /// use rotonda_store::prelude::*;
-            /// use rotonda_store::meta_examples::PrefixAs;
-            /// use rotonda_store::prelude::multi::*;
-            ///
-            ///
-            /// let store = MultiThreadedStore::<PrefixAs>::new().unwrap();
-            /// let guard = epoch::pin();
-            ///
-            /// let pfx_addr = "185.49.140.0".parse::<Ipv4Addr>()
-            ///         .unwrap()
-            ///         .into();
-            ///
-            /// store.insert(
-            ///     &Prefix::new(pfx_addr, 22).unwrap(),
-            ///     Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321)),
-            ///     None
-            /// );
-            ///
-            /// for prefix_record in store.less_specifics_iter_from(
-            ///     &Prefix::new(pfx_addr, 24).unwrap(),
-            ///     None,
-            ///     false,
-            ///     &guard
-            /// ) {
-            ///    assert_eq!(prefix_record.meta[0].meta.asn(), 211321.into());
-            /// }
-            /// ```
-            pub fn less_specifics_iter_from(&'a self,
-                search_pfx: &Prefix,
-                mui: Option<u32>,
-                include_withdrawn: bool,
-                guard: &'a Guard,
-                ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                    let (left, right) = match search_pfx.addr() {
-                        std::net::IpAddr::V4(addr) => {
-                            (
-                                Some(self
-                                    .v4
-                                    .less_specifics_iter_from(
-                                        PrefixId::<IPv4>::new(
-                                            <IPv4 as AddressFamily>::
-                                                from_ipaddr(addr),
-                                            search_pfx.len(),
-                                        ),
-                                        mui,
-                                        include_withdrawn,
-                                        guard
-                                    )
-                                    .map(|p| PrefixRecord::from(p))
-                                ),
-                                None
-                            )
-                        }
-                        std::net::IpAddr::V6(addr) => {
-                            (
-                                None,
-                                Some(self
-                                    .v6
-                                    .less_specifics_iter_from(
-                                        PrefixId::<IPv6>::new(
-                                            <IPv6 as AddressFamily>::
-                                                from_ipaddr(addr),
-                                            search_pfx.len(),
-                                        ),
-                                        mui,
-                                        include_withdrawn,
-                                        guard
-                                    )
-                                    .map(|p| PrefixRecord::from(p))
-                                )
-                            )
-                        }
-                    };
-
-                    left
-                        .into_iter()
-                        .flatten()
-                        .chain(right.into_iter().flatten())
-                }
-
-            /// Returns an iterator over all the more-specifics prefixes
-            /// of the `search_prefix`, if present in the store, including
-            /// the meta-data of these prefixes.
-            ///
-            /// The `search_pfx` argument can be either a IPv4 or an IPv6
-            /// prefix. The `search_pfx` itself doesn't have to be present
-            /// in the store for an iterator to be non-empty, i.e. if
-            /// more-specific prefixes exist for a non-existent
-            /// `search_pfx` the iterator will yield these more-specific
-            /// prefixes.
-            ///
-            /// The `guard` should be a `&epoch::pin()`. It allows the
-            /// iterator to create and return references to the meta-data
-            /// objects to the caller (instead of cloning them).
-            ///
-            /// # Example
-            /// ```
-            /// use std::net::Ipv4Addr;
-            ///
-            /// use rotonda_store::prelude::*;
-            /// use rotonda_store::prelude::multi::*;
-            /// use rotonda_store::meta_examples::PrefixAs;
-            ///
-            /// let store = MultiThreadedStore::<PrefixAs>::new().unwrap();
-            /// let guard = epoch::pin();
-            ///
-            /// let pfx_addr = "185.49.140.0".parse::<Ipv4Addr>()
-            ///         .unwrap()
-            ///         .into();
-            ///
-            /// store.insert(
-            ///     &Prefix::new(pfx_addr, 24).unwrap(),
-            ///     Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321)),
-            ///     None
-            /// );
-            ///
-            /// for prefix_record in store.more_specifics_iter_from(
-            ///     &Prefix::new(pfx_addr, 22).unwrap(),
-            ///     None,
-            ///     false,
-            ///     &guard
-            /// ) {
-            ///    assert_eq!(prefix_record.meta[0].meta.asn(), 211321.into());
-            /// }
-            /// ```
-            pub fn more_specifics_iter_from(&'a self,
-                search_pfx: &Prefix,
-                mui: Option<u32>,
-                include_withdrawn: bool,
-                guard: &'a Guard,
-            ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                    let (left, right) = match search_pfx.addr() {
-                        std::net::IpAddr::V4(addr) => {
-                            (
-                                Some(self
-                                    .v4
-                                    .more_specifics_iter_from(
-                                        PrefixId::<IPv4>::new(
-                                            <IPv4 as AddressFamily>::
-                                                from_ipaddr(addr),
-                                            search_pfx.len(),
-                                        ),
-                                        mui,
-                                        include_withdrawn,
-                                        guard
-                                    )
-                                    .map(|p| PrefixRecord::from(p))
-                                ),
-                                None
-                            )
-                        }
-                        std::net::IpAddr::V6(addr) => {
-                            (
-                                None,
-                                Some(self
-                                    .v6
-                                    .more_specifics_iter_from(
-                                        PrefixId::<IPv6>::new(
-                                            <IPv6 as AddressFamily>::
-                                                from_ipaddr(addr),
-                                            search_pfx.len(),
-                                        ),
-                                        mui,
-                                        include_withdrawn,
-                                        guard
-                                    )
-                                    .map(|p| PrefixRecord::from(p))
-                                )
-                            )
-                        }
-                    };
-
-                    left
-                        .into_iter()
-                        .flatten()
-                        .chain(right.into_iter().flatten())
-            }
-
-            pub fn iter_records_for_mui_v4(
-                &'a self,
-                mui: u32,
-                include_withdrawn: bool,
-                guard: &'a Guard
-            ) -> impl Iterator<Item=PrefixRecord<M>> +'a {
-
-                if self.v4.mui_is_withdrawn(mui, guard)
-                    && !include_withdrawn {
-                    None
-                } else {
-                    Some(
-                        self.v4
-                            .more_specifics_iter_from(
-                                PrefixId::<IPv4>::new(
-                                    <IPv4 as AddressFamily>::zero(),
-                                    0,
-                                ),
-                                Some(mui),
-                                include_withdrawn,
-                                guard
-                            ).map(|p| PrefixRecord::from(p))
-                    )}
-                .into_iter().flatten()
-            }
-
-            pub fn iter_records_for_mui_v6(
-                &'a self,
-                mui: u32,
-                include_withdrawn: bool,
-                guard: &'a Guard
-            ) -> impl Iterator<Item=PrefixRecord<M>> +'a {
-
-                if self.v6.mui_is_withdrawn(mui, guard)
-                    && !include_withdrawn {
-                    None
-                } else {
-                    Some(
-                        self.v6
-                            .more_specifics_iter_from(
-                                PrefixId::<IPv6>::new(
-                                    <IPv6 as AddressFamily>::zero(),
-                                    0,
-                                ),
-                                Some(mui),
-                                include_withdrawn,
-                                guard
-                            ).map(|p| PrefixRecord::from(p))
-                    )}
-                .into_iter().flatten()
-            }
-
-            /// Insert or replace a Record into the Store
-            ///
-            /// The specified Record will replace an existing record in the
-            /// store if the multi-map for the specified prefix already has an
-            /// entry for the `multi_uniq_id`, otherwise it will be added to
-            /// the multi-map.
-            ///
-            /// If the `update_path_sections` argument is used the best path
-            /// selection will be run on the resulting multi-map after insert
-            /// and stored for the specified prefix.
-            ///
-            /// Returns some metrics about the resulting insert.
-            pub fn insert(
-                &self,
-                prefix: &Prefix,
-                record: Record<M>,
-                update_path_selections: Option<M::TBI>
-            ) -> Result<UpsertReport, PrefixStoreError> {
-                match prefix.addr() {
-                    std::net::IpAddr::V4(addr) => {
-                        self.v4.insert(
-                            PrefixId::<IPv4>::from(*prefix),
-                            record,
-                            update_path_selections,
-                        )
-                    }
-                    std::net::IpAddr::V6(addr) => {
-                        self.v6.insert(
-                            PrefixId::<IPv6>::from(*prefix),
-                            record,
-                            update_path_selections,
-                        )
-                    }
-                }
-            }
-
-            /// Returns an unordered iterator over all prefixes, with any
-            /// status (including Withdrawn), for both IPv4 and IPv6,
-            /// currently in the store, including meta-data.
-            ///
-            /// Although the iterator is unordered within an address-family,
-            /// it first iterates over all IPv4 addresses and then over all
-            /// IPv6 addresses.
-            ///
-            /// The `guard` should be a `&epoch::pin()`. It allows the
-            /// iterator to create and return references to the meta-data
-            /// objects to the caller (instead of cloning them).
-            ///
-            /// # Example
-            /// ```
-            /// use std::net::Ipv4Addr;
-            ///
-            /// use rotonda_store::prelude::*;
-            /// use rotonda_store::prelude::multi::*;
-            /// use rotonda_store::meta_examples::PrefixAs;
-            ///
-            /// let store = MultiThreadedStore::<PrefixAs>::new().unwrap();
-            /// let guard = epoch::pin();
-            ///
-            /// let pfx_addr = "185.49.140.0".parse::<Ipv4Addr>()
-            ///         .unwrap()
-            ///         .into();
-            /// let our_asn = Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321));
-            ///
-            /// store.insert(&Prefix::new(pfx_addr, 22).unwrap(), our_asn.clone(), None);
-            /// store.insert(&Prefix::new(pfx_addr, 23).unwrap(), our_asn.clone(), None);
-            /// store.insert(&Prefix::new(pfx_addr, 24).unwrap(), our_asn.clone(), None);
-            /// store.insert(&Prefix::new(pfx_addr, 25).unwrap(), our_asn, None);
-            ///
-            /// let mut iter = store.prefixes_iter();
-            ///
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 22).unwrap());
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 23).unwrap());
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 24).unwrap());
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 25).unwrap());
-            /// ```
-            pub fn prefixes_iter(
-                &'a self,
-                guard: &'a Guard
-            ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v4.prefixes_iter(guard)
-                    .map(|p| PrefixRecord::from(p))
-                    .chain(
-                        self.v6.prefixes_iter(guard)
-                        .map(|p| PrefixRecord::from(p))
-                    )
-            }
-
-            /// Returns an unordered iterator over all IPv4 prefixes in the
-            /// currently in the store, with any status (including Withdrawn),
-            /// including meta-data.
-            ///
-            /// The `guard` should be a `&epoch::pin()`. It allows the
-            /// iterator to create and return references to the meta-data
-            /// objects to the caller (instead of cloning them).
-            ///
-            /// # Example
-            /// ```
-            /// use std::net::Ipv4Addr;
-            ///
-            /// use rotonda_store::prelude::*;
-            /// use rotonda_store::prelude::multi::*;
-            /// use rotonda_store::meta_examples::PrefixAs;
-            ///
-            /// let store = MultiThreadedStore::<PrefixAs>::new().unwrap();
-            /// let guard = epoch::pin();
-            ///
-            /// let pfx_addr = "185.49.140.0".parse::<Ipv4Addr>()
-            ///         .unwrap()
-            ///         .into();
-            /// let our_asn = Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321));
-            ///
-            /// store.insert(&Prefix::new(pfx_addr, 22).unwrap(), our_asn.clone(), None);
-            /// store.insert(&Prefix::new(pfx_addr, 23).unwrap(), our_asn.clone(), None);
-            /// store.insert(&Prefix::new(pfx_addr, 24).unwrap(), our_asn.clone(), None);
-            /// store.insert(&Prefix::new(pfx_addr, 25).unwrap(), our_asn, None);
-            ///
-            /// let mut iter = store.prefixes_iter();
-            ///
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 22).unwrap());
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 23).unwrap());
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 24).unwrap());
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 25).unwrap());
-            /// ```
-            pub fn prefixes_iter_v4(
-                &'a self,
-                guard: &'a Guard
-            ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v4.prefixes_iter(guard)
-                    .map(|p| PrefixRecord::from(p))
-            }
-
-            /// Returns an unordered iterator over all IPv6 prefixes in the
-            /// currently in the store, with any status (including Withdrawn),
-            /// including meta-data.
-            ///
-            /// The `guard` should be a `&epoch::pin()`. It allows the
-            /// iterator to create and return references to the meta-data
-            /// objects to the caller (instead of cloning them).
-            ///
-            /// # Example
-            /// ```
-            /// use std::net::Ipv6Addr;
-            ///
-            /// use rotonda_store::prelude::*;
-            /// use rotonda_store::prelude::multi::*;
-            /// use rotonda_store::meta_examples::PrefixAs;
-            ///
-            /// let store = MultiThreadedStore::<PrefixAs>::new().unwrap();
-            /// let guard = epoch::pin();
-            ///
-            /// let pfx_addr = "2a04:b900::".parse::<Ipv6Addr>()
-            ///         .unwrap()
-            ///         .into();
-            /// let our_asn = Record::new(0, 0, RouteStatus::Active, PrefixAs::new_from_u32(211321));
-            ///
-            /// store.insert(&Prefix::new(pfx_addr, 29).unwrap(), our_asn.clone(), None);
-            /// store.insert(&Prefix::new(pfx_addr, 48).unwrap(), our_asn.clone(), None);
-            /// store.insert(&Prefix::new(pfx_addr, 56).unwrap(), our_asn.clone(), None);
-            /// store.insert(&Prefix::new(pfx_addr, 64).unwrap(), our_asn, None);
-            ///
-            /// let mut iter = store.prefixes_iter();
-            ///
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 29).unwrap());
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 48).unwrap());
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 56).unwrap());
-            /// assert_eq!(iter.next().unwrap().prefix,
-            ///     Prefix::new(pfx_addr, 64).unwrap());
-            /// ```
-            pub fn prefixes_iter_v6(
-                &'a self,
-                guard: &'a Guard
-            ) -> impl Iterator<Item=PrefixRecord<M>> + 'a {
-                self.v6.prefixes_iter(guard)
-                    .map(|p| PrefixRecord::from(p))
-            }
-
-            /// Change the local status of the record for the combination of
-            /// (prefix, multi_uniq_id) to Withdrawn. Note that by default the
-            /// global `Withdrawn` status for a mui overrides the local status
-            /// of a record.
-            pub fn mark_mui_as_withdrawn_for_prefix(
-                &self,
-                prefix: &Prefix,
-                mui: u32,
-                ltime: u64
-            ) -> Result<(), PrefixStoreError> {
-                let guard = &epoch::pin();
-                match prefix.addr() {
-                    std::net::IpAddr::V4(addr) => {
-                        self.v4.mark_mui_as_withdrawn_for_prefix(
-                            PrefixId::<IPv4>::from(*prefix),
-                            mui,
-                            ltime
-                        )
-                    }
-                    std::net::IpAddr::V6(addr) => {
-                        self.v6.mark_mui_as_withdrawn_for_prefix(
-                            PrefixId::<IPv6>::from(*prefix),
-                            mui,
-                            ltime
-                        )
-                    }
-                }
-            }
-
-            /// Change the local status of the record for the combination of
-            /// (prefix, multi_uniq_id) to Active. Note that by default the
-            /// global `Withdrawn` status for a mui overrides the local status
-            /// of a record.
-            pub fn mark_mui_as_active_for_prefix(
-                &self,
-                prefix: &Prefix,
-                mui: u32,
-                ltime: u64
-            ) -> Result<(), PrefixStoreError> {
-                let guard = &epoch::pin();
-                match prefix.addr() {
-                    std::net::IpAddr::V4(addr) => {
-                        self.v4.mark_mui_as_active_for_prefix(
-                            PrefixId::<IPv4>::from(*prefix),
-                            mui,
-                            ltime
-                        )
-                    }
-                    std::net::IpAddr::V6(addr) => {
-                        self.v6.mark_mui_as_active_for_prefix(
-                            PrefixId::<IPv6>::from(*prefix),
-                            mui,
-                            ltime
-                        )
-                    }
-                }
-            }
-
-            /// Change the status of all records for IPv4 prefixes for this
-            /// `multi_uniq_id` globally to Active.  Note that the global
-            /// `Active` status will be overridden by the local status of the
-            /// record.
-            pub fn mark_mui_as_active_v4(
-                &self,
-                mui: u32
-            ) -> Result<(), PrefixStoreError> {
-                let guard = &epoch::pin();
-
-                self.v4.mark_mui_as_active(
-                    mui,
-                    &guard
-                )
-            }
-
-            /// Change the status of all records for IPv4 prefixes for this
-            /// `multi_uniq_id` globally to Withdrawn. A global `Withdrawn`
-            /// status for a `multi_uniq_id` overrides the local status of
-            /// prefixes for this mui. However the local status can still be
-            /// modified. This modification will take effect if the global
-            /// status is changed to `Active`.
-            pub fn mark_mui_as_withdrawn_v4(
-                &self,
-                mui: u32
-            ) -> Result<(), PrefixStoreError> {
-                let guard = &epoch::pin();
-
-                self.v4.mark_mui_as_withdrawn(
-                    mui,
-                    &guard
-                )
-            }
-
-            /// Change the status of all records for IPv6 prefixes for this
-            /// `multi_uniq_id` globally to Active.  Note that the global
-            /// `Active` status will be overridden by the local status of the
-            /// record.
-            pub fn mark_mui_as_active_v6(
-                &self,
-                mui: u32
-            ) -> Result<(), PrefixStoreError> {
-                let guard = &epoch::pin();
-
-                self.v6.mark_mui_as_active(
-                    mui,
-                    &guard
-                )
-            }
-
-            /// Change the status of all records for IPv6 prefixes for this
-            /// `multi_uniq_id` globally to Withdrawn. A global `Withdrawn`
-            /// status for a `multi_uniq_id` overrides the local status of
-            /// prefixes for this mui. However the local status can still be
-            /// modified. This modification will take effect if the global
-            /// status is changed to `Active`.
-            pub fn mark_mui_as_withdrawn_v6(
-                &self,
-                mui: u32
-            ) -> Result<(), PrefixStoreError> {
-                let guard = &epoch::pin();
-
-                self.v6.mark_mui_as_withdrawn(
-                    mui,
-                    &guard
-                )
-            }
-
-            /// Change the status of all records for this `multi_uniq_id` to
-            /// Withdrawn.
-            ///
-            /// This method tries to mark all records: first the IPv4 records,
-            /// then the IPv6 records. If marking of the IPv4 records fails,
-            /// the method continues and tries to mark the IPv6 records. If
-            /// either or both fail, an error is returned.
-            pub fn mark_mui_as_withdrawn(
-                &self,
-                mui: u32
-            ) -> Result<(), PrefixStoreError> {
-                let guard = &epoch::pin();
-
-                let res_v4 = self.v4.mark_mui_as_withdrawn(
-                    mui,
-                    &guard
-                );
-                let res_v6 = self.v6.mark_mui_as_withdrawn(
-                    mui,
-                    &guard
-                );
-
-                res_v4.and(res_v6)
-            }
-
-            // Whether the global status for IPv4 prefixes and the specified
-            // `multi_uniq_id` is set to `Withdrawn`.
-            pub fn mui_is_withdrawn_v4(
-                &self,
-                mui: u32
-            ) -> bool {
-                let guard = &epoch::pin();
-
-                self.v4.mui_is_withdrawn(mui, guard)
-            }
-
-            // Whether the global status for IPv6 prefixes and the specified
-            // `multi_uniq_id` is set to `Active`.
-            pub fn mui_is_withdrawn_v6(
-                &self,
-                mui: u32
-            ) -> bool {
-                let guard = &epoch::pin();
-
-                self.v6.mui_is_withdrawn(mui, guard)
-            }
-
-            /// Returns the number of all prefixes in the store.
-            ///
-            /// Note that this method will actually traverse the complete
-            /// tree.
-            pub fn prefixes_count(&self) -> UpsertCounters {
-                self.v4.get_prefixes_count()
-                + self.v6.get_prefixes_count()
-            }
-
-            /// Returns the number of all IPv4 prefixes in the store.
-            ///
-            /// Note that this counter may be lower than the actual
-            /// number in the store, due to contention at the time of
-            /// reading the value.
-            pub fn prefixes_v4_count(&self) -> UpsertCounters {
-                self.v4.get_prefixes_count()
-            }
-
-            /// Returns the number of all IPv4 prefixes with the
-            /// supplied prefix length in the store.
-            ///
-            /// Note that this counter may be lower than the actual
-            /// number in the store, due to contention at the time of
-            /// reading the value.
-            pub fn prefixes_v4_count_for_len(&self, len: u8)
-                -> UpsertCounters {
-                    self.v4.get_prefixes_count_for_len(len)
-                }
-
-            /// Returns the number of all IPv6 prefixes in the store.
-            ///
-            /// Note that this counter may be lower than the actual
-            /// number in the store, due to contention at the time of
-            /// reading the value.
-            pub fn prefixes_v6_count(&self) -> UpsertCounters {
-                self.v6.get_prefixes_count()
-            }
-
-            /// Returns the number of all IPv6 prefixes with the
-            /// supplied prefix length in the store.
-            ///
-            /// Note that this counter may be lower than the actual
-            /// number in the store, due to contention at the time of
-            /// reading the value.
-            pub fn prefixes_v6_count_for_len(&self, len: u8)
-                -> UpsertCounters {
-                    self.v6.get_prefixes_count_for_len(len)
-                }
-
-            /// Returns the number of nodes in the store.
-            ///
-            /// Note that this counter may be lower than the actual
-            /// number in the store, due to contention at the time of
-            /// reading the value.
-            pub fn nodes_count(&self) -> usize {
-                self.v4.get_nodes_count()
-                + self.v6.get_nodes_count()
-            }
-
-            /// Returns the number of IPv4 nodes in the store.
-            ///
-            /// Note that this counter may be lower than the actual
-            /// number in the store, due to contention at the time of
-            /// reading the value.
-            pub fn nodes_v4_count(&self) -> usize {
-                self.v4.get_nodes_count()
-            }
-
-            /// Returns the number of IPv6 nodes in the store.
-            ///
-            /// Note that this counter may be lower than the actual
-            /// number in the store, due to contention at the time of
-            /// reading the value.
-            pub fn nodes_v6_count(&self) -> usize {
-                self.v6.get_nodes_count()
-            }
-
-            /// Print the store statistics to the standard output.
-            #[cfg(feature = "cli")]
-            pub fn print_funky_stats(&self) {
-                println!("");
-                println!("Stats for IPv4 multi-threaded store\n");
-                println!("{}", self.v4.in_memory_tree);
-                println!("Stats for IPv6 multi-threaded store\n");
-                println!("{}", self.v6.in_memory_tree);
-            }
-
-            // The Store statistics.
-            pub fn stats(&self) -> StoreStats {
-                StoreStats {
-                    v4: self.v4.counters.get_prefix_stats(),
-                    v6: self.v6.counters.get_prefix_stats(),
-                }
-            }
-
-            // Disk Persistence
-
-            pub fn persist_strategy(&self) -> PersistStrategy {
-                self.config.persist_strategy()
-            }
-
-            pub fn get_records_for_prefix(
-                &self,
-                prefix: &Prefix,
-                mui: Option<u32>,
-                include_withdrawn: bool
-            ) -> Option<Vec<Record<M>>> {
-                let guard = &epoch::pin();
-
-                match prefix.is_v4() {
-                    true => self.v4.get_value(
-                        PrefixId::<IPv4>::from(*prefix),
-                        mui,
-                        include_withdrawn,
-                        guard
-                    ),
-                    false => self.v6.get_value(
-                        PrefixId::<IPv6>::from(*prefix),
-                        mui,
-                        include_withdrawn,
-                        guard
-                    )
-                }
-            }
-
-            /// Persist all the non-unique (prefix, mui, ltime) tuples
-            /// with their values to disk
-            pub fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
-                self.v4.flush_to_disk()?;
-                self.v6.flush_to_disk()?;
-
-                Ok(())
-            }
-
-            /// Return the approximate number of items that are persisted
-            /// to disk, for IPv4 and IPv6 respectively.
-            pub fn approx_persisted_items(&self) -> (usize, usize) {
-                (
-                    self.v4.approx_persisted_items(),
-                    self.v6.approx_persisted_items()
-                )
-            }
-
-            /// Return an estimation of the disk space currently used by the
-            /// store in bytes.
-            pub fn disk_space(&self) -> u64 {
-                self.v4.disk_space() + self.v6.disk_space()
-            }
-        }
-    };
-
-    let result = quote! {
-        #create_strides
-        #store
-    };
-
-    TokenStream::from(result)
-}
diff --git a/proc_macros/src/maps.rs b/proc_macros/src/maps.rs
deleted file mode 100644
index 7696a4f0..00000000
--- a/proc_macros/src/maps.rs
+++ /dev/null
@@ -1,547 +0,0 @@
-// Mappings between the Prefix lengths and storage level
-//
-// Each field in the CustomAllocStorage stores prefixes of the same length
-// and consists of a start bucket, which in turn contains StoredPrefixes
-// Each StoredPrefix contains a reference to another bucket, the next storage
-// level. Each length field in the CustomAllocStorage can have its own
-// arrangement for the sequence of bucket sizes. This file describes these
-// arrangements in the form of a mapping between prefix-length and bucket
-// sizes for each storage level.
-//
-// The mapping is a two-dimensional array, where the first dimension is the
-// prefix-length and the second dimension is the index of the end bit that is
-// stored in the storage level. Hence, an prefix-length array in this mapping
-// should only have increasing numbers as elements, and the values of the
-// elements summed MUST be the same as the prefix-length. A last requirement
-// is that THE ARAY SHOULD END IN AT LEAST ONE ZERO. This is because a zero
-// is used by the callers as a sentinel value to indicate that the last bucket
-// has been reached.
-//
-// So an array for a length might look like this:
-// [12, 24, 0, 0, 0, 0, 0, 0, 0, 0].
-// This is an array for a prefix-length of 24 and stores all prefixes in two
-// levels maximum.
-
-use quote::quote;
-
-pub fn node_buckets_map_v4() -> quote::__private::TokenStream {
-    quote! {
-
-    fn len_to_store_bits(len: u8, lvl: u8) -> u8 {
-        let res = 4 * (lvl + 1);
-        if res < len {
-            res
-        } else {
-           if res >= len + 4 {
-                0
-            } else {
-                len
-            }
-        }
-        // match len {
-        //     l if l <= 12 => {
-        //         if lvl > 0 {
-        //             0
-        //         } else {
-        //             len
-        //         }
-        //     }
-        //     l if l < 16 => {
-        //         if lvl > 1 {
-        //             0
-        //         } else {
-        //             12 * if lvl == 0 { 1 } else { lvl } + lvl * (len - 12)
-        //         }
-        //     }
-        //     _ => {
-        //         let res = 4 * (lvl + 1);
-        //         if res < len {
-        //             res
-        //         } else {
-        //             if res >= len + 4 {
-        //                  0
-        //             } else {
-        //                 len
-        //             }
-        //         }
-        //     }
-        // }
-    }
-
-    // fn len_to_store_bits_old(len: u8, level: u8) -> u8 {
-    //     // (vert x hor) = len x level -> number of bits
-    //     [
-    //         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 0
-    //         [1, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 1 - never exists
-    //         [2, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 2 - never exists
-    //         [3, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 3
-    //         [4, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 4
-    //         [5, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 5
-    //         [6, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 6
-    //         [7, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 7
-    //         [8, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 8
-    //         [9, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 9
-    //         [10, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 10
-    //         [11, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 11
-    //         [12, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 12
-    //         [12, 13, 0, 0, 0, 0, 0, 0, 0, 0],     // 13
-    //         [12, 14, 0, 0, 0, 0, 0, 0, 0, 0],     // 14
-    //         [12, 15, 0, 0, 0, 0, 0, 0, 0, 0],     // 15
-    //         [12, 16, 0, 0, 0, 0, 0, 0, 0, 0],     // 16
-    //         [12, 17, 0, 0, 0, 0, 0, 0, 0, 0],     // 17
-    //         [12, 18, 0, 0, 0, 0, 0, 0, 0, 0],     // 18
-    //         [12, 19, 0, 0, 0, 0, 0, 0, 0, 0],     // 19
-    //         [12, 20, 0, 0, 0, 0, 0, 0, 0, 0],     // 20
-    //         [12, 21, 0, 0, 0, 0, 0, 0, 0, 0],     // 21
-    //         [12, 22, 0, 0, 0, 0, 0, 0, 0, 0],     // 22
-    //         [12, 23, 0, 0, 0, 0, 0, 0, 0, 0],     // 23
-    //         [12, 24, 0, 0, 0, 0, 0, 0, 0, 0],     // 24
-    //         [12, 24, 25, 0, 0, 0, 0, 0, 0, 0],    // 25
-    //         [4, 8, 12, 16, 20, 24, 26, 0, 0, 0],  // 26
-    //         [4, 8, 12, 16, 20, 24, 27, 0, 0, 0],  // 27
-    //         [4, 8, 12, 16, 20, 24, 28, 0, 0, 0],  // 28
-    //         [4, 8, 12, 16, 20, 24, 28, 29, 0, 0], // 29
-    //         [4, 8, 12, 16, 20, 24, 28, 30, 0, 0], // 30
-    //         [4, 8, 12, 16, 20, 24, 28, 31, 0, 0],  // 31
-    //         [4, 8 , 12, 16, 20, 24, 28, 32, 0, 0], // 32
-    //     ][len as usize][level as usize]
-    // }
-
-    }
-}
-
-pub fn prefix_buckets_map_v4() -> quote::__private::TokenStream {
-    quote! {
-
-        fn get_bits_for_len(len: u8, lvl: u8) -> u8 {
-            let res = 4 * (lvl + 1);
-            if res < len {
-                res
-            } else {
-                if res >= len + 4 {
-                    0
-                } else {
-                    len
-                }
-            }
-            // match len {
-            //     l if l <= 12 => {
-            //         if lvl > 0 {
-            //             0
-            //         } else {
-            //             len
-            //         }
-            //     }
-            //     l if l < 16 => {
-            //         if lvl > 1 {
-            //             0
-            //         } else {
-            //             12 * if lvl == 0 { 1 } else { lvl } + lvl * (len - 12)
-            //         }
-            //     }
-            //     _ => {
-            //         let res = 4 * (lvl + 1);
-            //         if res < len {
-            //             res
-            //         } else {
-            //             if res >= len + 4 {
-            //                  0
-            //             } else {
-            //                 len
-            //             }
-            //         }
-            //     }
-            // }
-        }
-
-        // fn get_bits_for_len_old(len: u8, level: u8) -> u8 {
-        //     // (vert x hor) = len x level -> number of bits
-        //     [
-        //         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 0
-        //         [1, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 1 - never exists
-        //         [2, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 2 - never exists
-        //         [3, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 3
-        //         [4, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 4
-        //         [5, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 5
-        //         [6, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 6
-        //         [7, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 7
-        //         [8, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 8
-        //         [9, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 9
-        //         [10, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 10
-        //         [11, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 11
-        //         [12, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 12
-        //         [12, 13, 0, 0, 0, 0, 0, 0, 0, 0],     // 13
-        //         [12, 14, 0, 0, 0, 0, 0, 0, 0, 0],     // 14
-        //         [12, 15, 0, 0, 0, 0, 0, 0, 0, 0],     // 15
-        //         [4, 8, 12, 16, 0, 0, 0, 0, 0, 0],     // 16
-        //         [4, 8, 12, 16, 17, 0, 0, 0, 0, 0],     // 17
-        //         [4, 8, 12, 16, 18, 0, 0, 0, 0, 0],     // 18
-        //         [4, 8, 12, 16, 19, 0, 0, 0, 0, 0],     // 19
-        //         [4, 8, 12, 16, 20, 0, 0, 0, 0, 0],     // 20
-        //         [4, 8, 12, 16, 20, 21, 0, 0, 0, 0],     // 21
-        //         [4, 8, 12, 16, 20, 22, 0, 0, 0, 0],     // 22
-        //         [4, 8, 12, 16, 20, 23, 0, 0, 0, 0],     // 23
-        //         [4, 8, 12, 16, 20, 24, 0, 0, 0, 0],     // 24
-        //         [4, 8, 12, 16, 20, 24, 25, 0, 0, 0],    // 25
-        //         [4, 8, 12, 16, 20, 24, 26, 0, 0, 0],  // 26
-        //         [4, 8, 12, 16, 20, 24, 27, 0, 0, 0],  // 27
-        //         [4, 8, 12, 16, 20, 24, 28, 0, 0, 0],  // 28
-        //         [4, 8, 12, 16, 20, 24, 28, 29, 0, 0], // 29
-        //         [4, 8, 12, 16, 20, 24, 28, 30, 0, 0], // 30
-        //         [4, 8, 12, 16, 20, 24, 28, 31, 0, 0], // 31
-        //         [4, 8, 12, 16, 20, 24, 28, 32, 0, 0], // 32
-        //     ][len as usize][level as usize]
-        // }
-
-    }
-}
-
-pub fn node_buckets_map_v6() -> quote::__private::TokenStream {
-    quote! {
-
-        fn len_to_store_bits(len: u8, lvl: u8) -> u8 {
-            let res = 4 * (lvl + 1);
-            if res < len {
-                res
-            } else {
-                if res >= len + 4 { 0 } else { len }
-            }
-
-            // match len {
-            //     l if l <= 12 => if lvl > 0 { 0 } else { len },
-            //     l if l <= 24 => if lvl > 1 { 0 } else {
-            //         12 * if lvl == 0 { 1 } else { lvl } + lvl * (len - 12)
-            //         },
-            //     l if l <= 36 => { if lvl > 2 { 0 } else {
-            //         12 * if lvl <= 1 { lvl + 1 } else { 2 }
-            //             + if lvl == 0 { 0 } else { lvl - 1 } * (len - 24)
-            //         }
-            //     }
-            //     l if l <= 48 => { if lvl > 3 { 0 } else {
-            //             12 * if lvl <= 2 { lvl + 1 } else { 3 }
-            //             + if lvl < 2 { 0 } else { lvl - 2 } * (len - 36)
-            //         }
-            //     }
-            //     _ => {
-            //         let res = 8 * (lvl + 1);
-            //         if res < len {
-            //             res
-            //         } else {
-            //             if res >= len + 8 { 0 } else { len }
-            //         }
-            //     }
-            // }
-        }
-
-        // fn len_to_store_bits_old(len: u8, level: u8) -> u8 {
-        //     // (vert x hor) = len x level -> number of bits
-        //     [
-        //         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 0
-        //         [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 1 - never exists
-        //         [2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 2
-        //         [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 3
-        //         [4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 4
-        //         [5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 5
-        //         [6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 6
-        //         [7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 7
-        //         [8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 8
-        //         [9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // len 9
-        //         [10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // len 10
-        //         [11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // len 11
-        //         [12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // len 12
-        //         [12, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 13
-        //         [12, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 14
-        //         [12, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 15
-        //         [12, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 16
-        //         [12, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 17
-        //         [12, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 18
-        //         [12, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 19
-        //         [12, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 20
-        //         [12, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 21
-        //         [12, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 22
-        //         [12, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 23
-        //         [12, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 24
-        //         [12, 24, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],    // 25
-        //         [12, 24, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],    // 26
-        //         [12, 24, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],    // 27
-        //         [12, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],    // 28
-        //         [12, 24, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 29
-        //         [12, 24, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 30
-        //         [12, 24, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 31
-        //         [12, 24, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 32
-        //         [12, 24, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 33
-        //         [12, 24, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 34
-        //         [12, 24, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 35
-        //         [12, 24, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 36
-        //         [12, 24, 36, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 37
-        //         [12, 24, 36, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 38
-        //         [12, 24, 36, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 39
-        //         [12, 24, 36, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 40
-        //         [12, 24, 36, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 41
-        //         [12, 24, 36, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 42
-        //         [12, 24, 36, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 43
-        //         [12, 24, 36, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 44
-        //         [12, 24, 36, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 45
-        //         [12, 24, 36, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 46
-        //         [12, 24, 36, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 47
-        //         [12, 24, 36, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 48
-        //         [4, 8, 12, 24, 28, 48, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 49
-        //         [4, 8, 12, 24, 28, 48, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 50
-        //         [4, 8, 12, 24, 28, 48, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 51
-        //         [4, 8, 12, 24, 28, 48, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 52
-        //         [4, 8, 12, 24, 28, 48, 52, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 53
-        //         [4, 8, 12, 24, 28, 48, 52, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 54
-        //         [4, 8, 12, 24, 28, 48, 52, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 55
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 56
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 57
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 58
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 59
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 60
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 61
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 62
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 63
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 64
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],    // 65
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],    // 66
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],    // 67
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],    // 68
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 69
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 70
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 71
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 72
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 73
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],   // 74
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 75
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 76
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],  // 77
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 78
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 79
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 80
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 81
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 82
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],        // 83
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],        // 84
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 85
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 86
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 87
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],       // 88
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 89
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 90
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 91
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],      // 92
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 93
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 94
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 95
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0],     // 96
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 97, 0, 0, 0, 0, 0, 0, 0, 0],    // 97
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 98, 0, 0, 0, 0, 0, 0, 0, 0],    // 98
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 99, 0, 0, 0, 0, 0, 0, 0, 0],        // 99
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 0, 0, 0, 0, 0, 0, 0, 0],       // 100
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 101, 0, 0, 0, 0, 0, 0, 0],     // 101
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 102, 0, 0, 0, 0, 0, 0, 0],     // 102
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 103, 0, 0, 0, 0, 0, 0, 0],     // 103
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 0, 0, 0, 0, 0, 0, 0],     // 104
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 105, 0, 0, 0, 0, 0, 0],   // 105
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 106, 0, 0, 0, 0, 0, 0],       // 106
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 107, 0, 0, 0, 0, 0, 0],       // 107
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 0, 0, 0, 0, 0, 0],       // 108
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 109, 0, 0, 0, 0, 0],     // 109
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 110, 0, 0, 0, 0, 0],     // 110
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 111, 0, 0, 0, 0, 0],           // 111
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 0, 0, 0, 0, 0],           // 112
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 113, 0, 0, 0, 0],         // 113
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 114, 0, 0, 0, 0],         // 114
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 115, 0, 0, 0, 0],         // 115
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 0, 0, 0, 0],         // 116
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 117, 0, 0, 0],       // 117
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 118, 0, 0, 0],       // 118
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 119, 0, 0, 0],       // 119
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 0, 0, 0],       // 120
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 121, 0, 0],     // 121
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 122, 0, 0],     // 122
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 123, 0, 0],     // 123
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 0, 0],     // 124
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 125, 0],   // 125
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 126, 0],   // 126
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 127, 0],   // 127
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 0],   // 128
-        //     ][len as usize][level as usize]
-        // }
-
-    }
-}
-
-pub fn prefix_buckets_map_v6() -> quote::__private::TokenStream {
-    quote! {
-
-        fn get_bits_for_len(len: u8, lvl: u8) -> u8 {
-            let res = 4 * (lvl + 1);
-            if res <= len {
-                res
-            } else {
-                if res >= len + 4 { 0 } else { len }
-            }
-
-            // match len {
-            //     l if l <= 12 => if lvl > 0 { 0 } else { len },
-            //     l if l <= 24 => if lvl > 1 { 0 } else {
-            //         12 * if lvl == 0 { 1 } else { lvl } + lvl * (len - 12)
-            //         },
-            //     l if l <= 36 => { if lvl > 2 { 0 } else {
-            //         12 * if lvl <= 1 { lvl + 1 } else { 2 }
-            //             + if lvl == 0 { 0 } else { lvl - 1 } * (len - 24)
-            //         }
-            //     }
-            //     l if l <= 48 => { if lvl > 3 { 0 } else {
-            //             12 * if lvl <= 2 { lvl + 1 } else { 3 }
-            //             + if lvl < 2 { 0 } else { lvl - 2 } * (len - 36)
-            //         }
-            //     }
-            //     _ => {
-            //         let res = 8 * (lvl + 1);
-            //         if res < len {
-            //             res
-            //         } else {
-            //             if res >= len + 8 { 0 } else { len }
-            //         }
-            //     }
-            // }
-        }
-
-        // fn get_bits_for_len_old(len: u8, level: u8) -> u8 {
-        //     // (vert x hor) = len x level -> number of bits
-        //     [
-        //         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 0
-        //         [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 1 - never exists
-        //         [2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 2
-        //         [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 3
-        //         [4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 4
-        //         [5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 5
-        //         [6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 6
-        //         [7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 7
-        //         [8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 8
-        //         [9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 9
-        //         [10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 10
-        //         [11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 11
-        //         [12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // len 12
-        //         [12, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 13
-        //         [12, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 14
-        //         [12, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 15
-        //         [12, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 16
-        //         [12, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 17
-        //         [12, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 18
-        //         [12, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 19
-        //         [12, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 20
-        //         [12, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 21
-        //         [12, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 22
-        //         [12, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 23
-        //         [12, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 24
-        //         [12, 24, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 25
-        //         [12, 24, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 26
-        //         [12, 24, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 27
-        //         [12, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 28
-        //         [12, 24, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 29
-        //         [12, 24, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 30
-        //         [12, 24, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 31
-        //         [12, 24, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 32
-        //         [12, 24, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 33
-        //         [12, 24, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 34
-        //         [12, 24, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 35
-        //         [12, 24, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 36
-        //         [12, 24, 36, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 37
-        //         [12, 24, 36, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 38
-        //         [12, 24, 36, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 39
-        //         [12, 24, 36, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 40
-        //         [12, 24, 36, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 41
-        //         [12, 24, 36, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 42
-        //         [12, 24, 36, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 43
-        //         [12, 24, 36, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 44
-        //         [12, 24, 36, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 45
-        //         [12, 24, 36, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 46
-        //         [12, 24, 36, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 47
-        //         [12, 24, 36, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 48
-        //         [4, 8, 12, 24, 28, 48, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 49
-        //         [4, 8, 12, 24, 28, 48, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 50
-        //         [4, 8, 12, 24, 28, 48, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 51
-        //         [4, 8, 12, 24, 28, 48, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 52
-        //         [4, 8, 12, 24, 28, 48, 52, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 53
-        //         [4, 8, 12, 24, 28, 48, 52, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 54
-        //         [4, 8, 12, 24, 28, 48, 52, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 55
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 56
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 57
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 58
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 59
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 60
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 61
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 62
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 63
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 64
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 65
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 66
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 67
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 68
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 69
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 70
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 71
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 72
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 73
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 74
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 75
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 76
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 77
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 78
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 79
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 80
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 81
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 82
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 83
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 84
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 85
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 86
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 87
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 88
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 89
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 90
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 91
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 92
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 93
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 94
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 95
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 96
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 97, 0, 0, 0, 0, 0, 0, 0, 0], // 97
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 98, 0, 0, 0, 0, 0, 0, 0, 0], // 98
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 99, 0, 0, 0, 0, 0, 0, 0, 0], // 99
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 0, 0, 0, 0, 0, 0, 0, 0], // 100
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 101, 0, 0, 0, 0, 0, 0, 0], // 101
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 102, 0, 0, 0, 0, 0, 0, 0], // 102
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 103, 0, 0, 0, 0, 0, 0, 0], // 103
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 0, 0, 0, 0, 0, 0, 0], // 104
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 105, 0, 0, 0, 0, 0, 0], // 105
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 106, 0, 0, 0, 0, 0, 0], // 106
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 107, 0, 0, 0, 0, 0, 0], // 107
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 0, 0, 0, 0, 0, 0], // 108
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 109, 0, 0, 0, 0, 0], // 109
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 110, 0, 0, 0, 0, 0], // 110
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 111, 0, 0, 0, 0, 0], // 111
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 0, 0, 0, 0, 0], // 112
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 113, 0, 0, 0, 0], // 113
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 114, 0, 0, 0, 0], // 114
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 115, 0, 0, 0, 0], // 115
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 0, 0, 0, 0], // 116
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 117, 0, 0, 0], // 117
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 118, 0, 0, 0], // 118
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 119, 0, 0, 0], // 119
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 0, 0, 0], // 120
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 121, 0, 0], // 121
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 122, 0, 0], // 122
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 123, 0, 0], // 123
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 0, 0], // 124
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 125, 0], // 125
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 126, 0], // 126
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 127, 0], // 127
-        //         [4, 8, 12, 24, 28, 48, 52, 56, 60, 64, 68, 74, 78, 82, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 0], // 128
-        //     ][len as usize][level as usize]
-        // }
-
-    }
-}

From 02821052640ad453a9305e5e41138d65348c2618 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Mar 2025 15:41:19 +0100
Subject: [PATCH 105/147] remove persist as features

---
 Cargo.toml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index e9e1d1bd..ad584beb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -12,9 +12,6 @@ authors.workspace = true
 rust-version.workspace = true
 license.workspace = true
 
-[workspace]
-members   = ["proc_macros"]
-
 [workspace.package]
 version = "0.4.1"
 edition = "2021"
@@ -30,7 +27,6 @@ parking_lot_core = "0.9.10"
 inetnum   = "0.1"
 log       = "^0.4"
 roaring 	= "0.10.3"
-# rotonda-macros = { path = "proc_macros", version = "0.4.0" }
 routecore       = { git = "https://github.com/nlnetlabs/routecore", branch = "dev", version = "0.5.2-dev", features = ["bgp", "bmp", "fsm", "serde", "mrt"] }
 ansi_term   = { version = "0.12", optional = true }
 csv         = { version = "1", optional = true }
@@ -39,7 +35,7 @@ clap        = { version = "4.4", optional = true, features = ["derive"] }
 rayon = { version = "1.10", optional = true }
 memmap2 = { version = "0.9", optional = true }
 rand = { version = "0.9" }
-lsm-tree = { version = "2.6.3", optional = true }
+lsm-tree = { version = "2.6.3" }
 serde = "1.0.216"
 serde_derive = "1.0.216"
 serde_json = "1.0.133"
@@ -53,7 +49,6 @@ env_logger  = { version = "0.10" }
 [features]
 cli = ["ansi_term", "rustyline", "csv"]
 mrt = ["clap", "rayon"]
-persist = ["lsm-tree"]
 default = []
 
 [[bin]]
@@ -62,4 +57,4 @@ required-features = ["cli"]
 
 [[bin]]
 name = "load_mrt"
-required-features = ["mrt","persist"]
+required-features = ["mrt"]

From 83ef0678b3cdf56371ef72de853b80255439a84c Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Tue, 4 Mar 2025 17:58:49 +0100
Subject: [PATCH 106/147] remove local_array and prelude modules

---
 .../in_memory/atomic_stride.rs                |   0
 .../in_memory/atomic_types.rs                 |   0
 .../in_memory/deprecated_query.rs             |   0
 src/{local_array => }/in_memory/iterators.rs  |   0
 src/{local_array => }/in_memory/mod.rs        |   0
 src/{local_array => }/in_memory/node.rs       |   0
 src/{local_array => }/in_memory/oncebox.rs    |   0
 src/{local_array => }/in_memory/query.rs      |   0
 src/{local_array => }/in_memory/tree.rs       |   0
 src/local_array/rib/mod.rs                    |   6 -
 src/{local_array => }/mod.rs                  |   0
 src/{local_array => }/persist/lsm_tree.rs     |   0
 src/{local_array => }/persist/mod.rs          |   0
 src/{local_array => }/prefix_cht/cht.rs       |   0
 src/{local_array => }/prefix_cht/iterators.rs |   0
 .../prefix_cht/iterators_cp.rs                |   0
 src/{local_array => }/prefix_cht/mod.rs       |   0
 src/{local_array => }/query.rs                |   0
 src/{local_array => }/rib/init                |   0
 src/rib/mod.rs                                |   8 +
 src/rib/query.rs                              | 395 ++++++++++++++++++
 src/{local_array => }/rib/rib.rs              |  23 +-
 .../rib/default_store.rs => rib/starcast.rs}  |  17 +-
 src/{local_array => }/tests.rs                |   0
 src/{ => types}/af.rs                         |   0
 src/{local_array => types}/bit_span.rs        |   0
 src/{local_array => types}/errors.rs          |   0
 .../types.rs => types/prefix_id.rs}           |   0
 src/{ => types}/prefix_record.rs              |   0
 src/{ => types}/test_types.rs                 |   0
 30 files changed, 428 insertions(+), 21 deletions(-)
 rename src/{local_array => }/in_memory/atomic_stride.rs (100%)
 rename src/{local_array => }/in_memory/atomic_types.rs (100%)
 rename src/{local_array => }/in_memory/deprecated_query.rs (100%)
 rename src/{local_array => }/in_memory/iterators.rs (100%)
 rename src/{local_array => }/in_memory/mod.rs (100%)
 rename src/{local_array => }/in_memory/node.rs (100%)
 rename src/{local_array => }/in_memory/oncebox.rs (100%)
 rename src/{local_array => }/in_memory/query.rs (100%)
 rename src/{local_array => }/in_memory/tree.rs (100%)
 delete mode 100644 src/local_array/rib/mod.rs
 rename src/{local_array => }/mod.rs (100%)
 rename src/{local_array => }/persist/lsm_tree.rs (100%)
 rename src/{local_array => }/persist/mod.rs (100%)
 rename src/{local_array => }/prefix_cht/cht.rs (100%)
 rename src/{local_array => }/prefix_cht/iterators.rs (100%)
 rename src/{local_array => }/prefix_cht/iterators_cp.rs (100%)
 rename src/{local_array => }/prefix_cht/mod.rs (100%)
 rename src/{local_array => }/query.rs (100%)
 rename src/{local_array => }/rib/init (100%)
 create mode 100644 src/rib/mod.rs
 create mode 100644 src/rib/query.rs
 rename src/{local_array => }/rib/rib.rs (97%)
 rename src/{local_array/rib/default_store.rs => rib/starcast.rs} (98%)
 rename src/{local_array => }/tests.rs (100%)
 rename src/{ => types}/af.rs (100%)
 rename src/{local_array => types}/bit_span.rs (100%)
 rename src/{local_array => types}/errors.rs (100%)
 rename src/{local_array/types.rs => types/prefix_id.rs} (100%)
 rename src/{ => types}/prefix_record.rs (100%)
 rename src/{ => types}/test_types.rs (100%)

diff --git a/src/local_array/in_memory/atomic_stride.rs b/src/in_memory/atomic_stride.rs
similarity index 100%
rename from src/local_array/in_memory/atomic_stride.rs
rename to src/in_memory/atomic_stride.rs
diff --git a/src/local_array/in_memory/atomic_types.rs b/src/in_memory/atomic_types.rs
similarity index 100%
rename from src/local_array/in_memory/atomic_types.rs
rename to src/in_memory/atomic_types.rs
diff --git a/src/local_array/in_memory/deprecated_query.rs b/src/in_memory/deprecated_query.rs
similarity index 100%
rename from src/local_array/in_memory/deprecated_query.rs
rename to src/in_memory/deprecated_query.rs
diff --git a/src/local_array/in_memory/iterators.rs b/src/in_memory/iterators.rs
similarity index 100%
rename from src/local_array/in_memory/iterators.rs
rename to src/in_memory/iterators.rs
diff --git a/src/local_array/in_memory/mod.rs b/src/in_memory/mod.rs
similarity index 100%
rename from src/local_array/in_memory/mod.rs
rename to src/in_memory/mod.rs
diff --git a/src/local_array/in_memory/node.rs b/src/in_memory/node.rs
similarity index 100%
rename from src/local_array/in_memory/node.rs
rename to src/in_memory/node.rs
diff --git a/src/local_array/in_memory/oncebox.rs b/src/in_memory/oncebox.rs
similarity index 100%
rename from src/local_array/in_memory/oncebox.rs
rename to src/in_memory/oncebox.rs
diff --git a/src/local_array/in_memory/query.rs b/src/in_memory/query.rs
similarity index 100%
rename from src/local_array/in_memory/query.rs
rename to src/in_memory/query.rs
diff --git a/src/local_array/in_memory/tree.rs b/src/in_memory/tree.rs
similarity index 100%
rename from src/local_array/in_memory/tree.rs
rename to src/in_memory/tree.rs
diff --git a/src/local_array/rib/mod.rs b/src/local_array/rib/mod.rs
deleted file mode 100644
index b163e60f..00000000
--- a/src/local_array/rib/mod.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-#[allow(clippy::module_inception)]
-pub mod rib;
-
-pub(crate) mod default_store;
-
-pub use default_store::StarCastDb;
diff --git a/src/local_array/mod.rs b/src/mod.rs
similarity index 100%
rename from src/local_array/mod.rs
rename to src/mod.rs
diff --git a/src/local_array/persist/lsm_tree.rs b/src/persist/lsm_tree.rs
similarity index 100%
rename from src/local_array/persist/lsm_tree.rs
rename to src/persist/lsm_tree.rs
diff --git a/src/local_array/persist/mod.rs b/src/persist/mod.rs
similarity index 100%
rename from src/local_array/persist/mod.rs
rename to src/persist/mod.rs
diff --git a/src/local_array/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
similarity index 100%
rename from src/local_array/prefix_cht/cht.rs
rename to src/prefix_cht/cht.rs
diff --git a/src/local_array/prefix_cht/iterators.rs b/src/prefix_cht/iterators.rs
similarity index 100%
rename from src/local_array/prefix_cht/iterators.rs
rename to src/prefix_cht/iterators.rs
diff --git a/src/local_array/prefix_cht/iterators_cp.rs b/src/prefix_cht/iterators_cp.rs
similarity index 100%
rename from src/local_array/prefix_cht/iterators_cp.rs
rename to src/prefix_cht/iterators_cp.rs
diff --git a/src/local_array/prefix_cht/mod.rs b/src/prefix_cht/mod.rs
similarity index 100%
rename from src/local_array/prefix_cht/mod.rs
rename to src/prefix_cht/mod.rs
diff --git a/src/local_array/query.rs b/src/query.rs
similarity index 100%
rename from src/local_array/query.rs
rename to src/query.rs
diff --git a/src/local_array/rib/init b/src/rib/init
similarity index 100%
rename from src/local_array/rib/init
rename to src/rib/init
diff --git a/src/rib/mod.rs b/src/rib/mod.rs
new file mode 100644
index 00000000..3c1ef1ff
--- /dev/null
+++ b/src/rib/mod.rs
@@ -0,0 +1,8 @@
+pub mod query;
+pub mod rib;
+pub mod starcast;
+
+pub use rib::Counters;
+pub use starcast::StarCastDb;
+pub use starcast::BIT_SPAN_SIZE;
+pub use starcast::STRIDE_SIZE;
diff --git a/src/rib/query.rs b/src/rib/query.rs
new file mode 100644
index 00000000..a00b57e4
--- /dev/null
+++ b/src/rib/query.rs
@@ -0,0 +1,395 @@
+use crossbeam_epoch::{self as epoch};
+use epoch::Guard;
+use log::trace;
+use zerocopy::TryFromBytes;
+
+use crate::rib::rib::{Config, PersistStrategy, Rib};
+use crate::types::prefix_record::ZeroCopyRecord;
+use crate::types::AddressFamily;
+use crate::types::PublicRecord;
+use inetnum::addr::Prefix;
+
+use crate::{Meta, QueryResult};
+
+use crate::{MatchOptions, MatchType};
+
+use crate::types::errors::PrefixStoreError;
+use crate::types::PrefixId;
+
+//------------ Prefix Matching ----------------------------------------------
+
+impl<
+        'a,
+        AF: AddressFamily,
+        M: Meta,
+        const N_ROOT_SIZE: usize,
+        const P_ROOT_SIZE: usize,
+        C: Config,
+        const KEY_SIZE: usize,
+    > Rib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
+{
+    pub(crate) fn get_value(
+        &'a self,
+        prefix_id: PrefixId<AF>,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> Option<Vec<PublicRecord<M>>> {
+        match self.persist_strategy() {
+            PersistStrategy::PersistOnly => {
+                trace!("get value from persist_store for {:?}", prefix_id);
+                self.persist_tree.as_ref().and_then(|tree| {
+                    tree.get_records_for_prefix(
+                        prefix_id,
+                        mui,
+                        include_withdrawn,
+                        self.in_memory_tree.withdrawn_muis_bmin(guard),
+                    )
+                    .map(|v| {
+                        v.iter()
+                            .map(|bytes| {
+                                let record: &ZeroCopyRecord<AF> =
+                                    ZeroCopyRecord::try_ref_from_bytes(bytes)
+                                        .unwrap();
+                                PublicRecord::<M> {
+                                    multi_uniq_id: record.multi_uniq_id,
+                                    ltime: record.ltime,
+                                    status: record.status,
+                                    meta: <Vec<u8>>::from(
+                                        record.meta.as_ref(),
+                                    )
+                                    .into(),
+                                }
+                            })
+                            .collect::<Vec<_>>()
+                    })
+                })
+            }
+            _ => self.prefix_cht.get_records_for_prefix(
+                prefix_id,
+                mui,
+                include_withdrawn,
+                self.in_memory_tree.withdrawn_muis_bmin(guard),
+            ),
+        }
+    }
+
+    pub(crate) fn more_specifics_from(
+        &'a self,
+        prefix_id: PrefixId<AF>,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> QueryResult<M> {
+        let prefix = if !self.contains(prefix_id, mui) {
+            Some(Prefix::from(prefix_id))
+        } else {
+            None
+        };
+        let more_specifics = self
+            .in_memory_tree
+            .more_specific_prefix_iter_from(prefix_id)
+            .map(|p| {
+                self.get_value(prefix_id, mui, include_withdrawn, guard)
+                    .map(|v| (p, v))
+            })
+            .collect();
+
+        QueryResult {
+            prefix,
+            prefix_meta: prefix
+                .map(|_pfx| {
+                    self.get_value(prefix_id, mui, include_withdrawn, guard)
+                        .unwrap_or_default()
+                })
+                .unwrap_or(vec![]),
+            match_type: MatchType::EmptyMatch,
+            less_specifics: None,
+            more_specifics,
+        }
+    }
+
+    pub(crate) fn less_specifics_from(
+        &'a self,
+        prefix_id: PrefixId<AF>,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> QueryResult<M> {
+        let prefix = if !self.contains(prefix_id, mui) {
+            Some(Prefix::from(prefix_id))
+        } else {
+            None
+        };
+
+        let less_specifics = self
+            .in_memory_tree
+            .less_specific_prefix_iter(prefix_id)
+            .map(|p| {
+                self.get_value(prefix_id, mui, include_withdrawn, guard)
+                    .map(|v| (p, v))
+            })
+            .collect();
+
+        QueryResult {
+            prefix,
+            prefix_meta: self
+                .get_value(prefix_id, mui, include_withdrawn, guard)
+                .unwrap_or_default(),
+            match_type: MatchType::EmptyMatch,
+            less_specifics,
+            more_specifics: None,
+        }
+    }
+
+    pub(crate) fn more_specifics_iter_from(
+        &'a self,
+        prefix_id: PrefixId<AF>,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        println!("more_specifics_iter_from fn");
+        // If the user wanted a specific mui and not withdrawn prefixes, we
+        // may return early if the mui is globally withdrawn.
+        (if mui.is_some_and(|m| {
+            !include_withdrawn && self.mui_is_withdrawn(m, guard)
+        }) {
+            None
+        } else {
+            Some(
+                self.in_memory_tree
+                    .more_specific_prefix_iter_from(prefix_id)
+                    .filter_map(move |p| {
+                        self.get_value(p, mui, include_withdrawn, guard)
+                            .map(|v| (p, v))
+                    }),
+            )
+        })
+        .into_iter()
+        .flatten()
+        // .chain(
+        //     (if mui.is_some_and(|m| {
+        //         self.config.persist_strategy == PersistStrategy::WriteAhead
+        //             || (!include_withdrawn && self.mui_is_withdrawn(m, guard))
+        //     }) {
+        //         None
+        //     } else {
+        //         let global_withdrawn_bmin =
+        //             self.in_memory_tree.withdrawn_muis_bmin(guard);
+        //         self.persist_tree.as_ref().map(|persist_tree| {
+        //             persist_tree.more_specific_prefix_iter_from(
+        //                 prefix_id,
+        //                 vec![],
+        //                 mui,
+        //                 global_withdrawn_bmin,
+        //                 include_withdrawn,
+        //             )
+        //         })
+        //     })
+        //     .into_iter()
+        //     .flatten(),
+        // )
+    }
+
+    pub(crate) fn less_specifics_iter_from(
+        &'a self,
+        prefix_id: PrefixId<AF>,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        guard: &'a Guard,
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+        self.in_memory_tree
+            .less_specific_prefix_iter(prefix_id)
+            .filter_map(move |p| {
+                self.get_value(p, mui, include_withdrawn, guard)
+                    .map(|v| (p, v))
+            })
+    }
+
+    pub(crate) fn match_prefix(
+        &'a self,
+        search_pfx: PrefixId<AF>,
+        options: &MatchOptions,
+        guard: &'a Guard,
+    ) -> QueryResult<M> {
+        trace!("match_prefix rib {:?} {:?}", search_pfx, options);
+        let res = self.in_memory_tree.match_prefix(search_pfx, options);
+
+        trace!("res {:?}", res);
+        let mut res = QueryResult::from(res);
+
+        if let Some(Some(m)) = res.prefix.map(|p| {
+            self.get_value(
+                p.into(),
+                options.mui,
+                options.include_withdrawn,
+                guard,
+            )
+            .and_then(|v| if v.is_empty() { None } else { Some(v) })
+        }) {
+            res.prefix_meta = m;
+        } else {
+            res.prefix = None;
+            res.match_type = MatchType::EmptyMatch;
+        }
+
+        if options.include_more_specifics {
+            res.more_specifics = res.more_specifics.map(|p| {
+                p.iter()
+                    .filter_map(|mut r| {
+                        if let Some(m) = self.get_value(
+                            r.prefix.into(),
+                            options.mui,
+                            options.include_withdrawn,
+                            guard,
+                        ) {
+                            r.meta = m;
+                            Some(r)
+                        } else {
+                            None
+                        }
+                    })
+                    .collect()
+            });
+        }
+        if options.include_less_specifics {
+            res.less_specifics = res.less_specifics.map(|p| {
+                p.iter()
+                    .filter_map(|mut r| {
+                        if let Some(m) = self.get_value(
+                            r.prefix.into(),
+                            options.mui,
+                            options.include_withdrawn,
+                            guard,
+                        ) {
+                            r.meta = m;
+                            Some(r)
+                        } else {
+                            None
+                        }
+                    })
+                    .collect()
+            });
+        }
+
+        res
+    }
+
+    pub(crate) fn best_path(
+        &'a self,
+        search_pfx: PrefixId<AF>,
+        guard: &Guard,
+    ) -> Option<Result<PublicRecord<M>, PrefixStoreError>> {
+        self.prefix_cht
+            .non_recursive_retrieve_prefix(search_pfx)
+            .0
+            .map(|p_rec| {
+                p_rec.get_path_selections(guard).best().map_or_else(
+                    || Err(PrefixStoreError::BestPathNotFound),
+                    |mui| {
+                        p_rec
+                            .record_map
+                            .get_record_for_mui(mui, false)
+                            .ok_or(PrefixStoreError::StoreNotReadyError)
+                    },
+                )
+            })
+    }
+
+    pub(crate) fn calculate_and_store_best_and_backup_path(
+        &self,
+        search_pfx: PrefixId<AF>,
+        tbi: &<M as Meta>::TBI,
+        guard: &Guard,
+    ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
+        self.prefix_cht
+            .non_recursive_retrieve_prefix(search_pfx)
+            .0
+            .map_or(Err(PrefixStoreError::StoreNotReadyError), |p_rec| {
+                p_rec.calculate_and_store_best_backup(tbi, guard)
+            })
+    }
+
+    pub(crate) fn is_ps_outdated(
+        &self,
+        search_pfx: PrefixId<AF>,
+        guard: &Guard,
+    ) -> Result<bool, PrefixStoreError> {
+        self.prefix_cht
+            .non_recursive_retrieve_prefix(search_pfx)
+            .0
+            .map_or(Err(PrefixStoreError::StoreNotReadyError), |p| {
+                Ok(p.is_ps_outdated(guard))
+            })
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct TreeQueryResult<AF: AddressFamily> {
+    pub match_type: MatchType,
+    pub prefix: Option<PrefixId<AF>>,
+    pub less_specifics: Option<Vec<PrefixId<AF>>>,
+    pub more_specifics: Option<Vec<PrefixId<AF>>>,
+}
+
+impl<AF: AddressFamily, M: Meta> From<TreeQueryResult<AF>>
+    for QueryResult<M>
+{
+    fn from(value: TreeQueryResult<AF>) -> Self {
+        Self {
+            match_type: value.match_type,
+            prefix: value.prefix.map(|p| p.into()),
+            prefix_meta: vec![],
+            less_specifics: value
+                .less_specifics
+                .map(|ls| ls.into_iter().map(|p| (p, vec![])).collect()),
+            more_specifics: value
+                .more_specifics
+                .map(|ms| ms.into_iter().map(|p| (p, vec![])).collect()),
+        }
+    }
+}
+
+impl<AF: AddressFamily, M: Meta> From<TreeQueryResult<AF>>
+    for FamilyQueryResult<AF, M>
+{
+    fn from(value: TreeQueryResult<AF>) -> Self {
+        Self {
+            match_type: value.match_type,
+            prefix: value.prefix,
+            prefix_meta: vec![],
+            less_specifics: None,
+            more_specifics: None,
+        }
+    }
+}
+
+pub(crate) type FamilyRecord<AF, M> =
+    Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>;
+
+pub(crate) struct FamilyQueryResult<AF: AddressFamily, M: Meta> {
+    pub match_type: MatchType,
+    pub prefix: Option<PrefixId<AF>>,
+    pub prefix_meta: Vec<PublicRecord<M>>,
+    pub less_specifics: Option<FamilyRecord<AF, M>>,
+    pub more_specifics: Option<FamilyRecord<AF, M>>,
+}
+
+impl<AF: AddressFamily, M: Meta> From<FamilyQueryResult<AF, M>>
+    for QueryResult<M>
+{
+    fn from(value: FamilyQueryResult<AF, M>) -> Self {
+        QueryResult {
+            match_type: value.match_type,
+            prefix: value.prefix.map(|p| p.into()),
+            prefix_meta: value.prefix_meta,
+            less_specifics: value
+                .less_specifics
+                .map(|ls| ls.into_iter().collect()),
+            more_specifics: value
+                .more_specifics
+                .map(|ms| ms.into_iter().collect()),
+        }
+    }
+}
diff --git a/src/local_array/rib/rib.rs b/src/rib/rib.rs
similarity index 97%
rename from src/local_array/rib/rib.rs
rename to src/rib/rib.rs
index 99d91bfb..b68f54c2 100644
--- a/src/local_array/rib/rib.rs
+++ b/src/rib/rib.rs
@@ -9,21 +9,19 @@ use crossbeam_epoch::{self as epoch};
 use epoch::{Guard, Owned};
 use zerocopy::TryFromBytes;
 
-use crate::local_array::in_memory::tree::TreeBitMap;
-use crate::local_array::persist::lsm_tree::LongKey;
-use crate::local_array::prefix_cht::cht::PrefixCht;
-use crate::local_array::types::PrefixId;
-use crate::prefix_record::{ValueHeader, ZeroCopyRecord};
-use crate::prelude::multi::RouteStatus;
+use crate::in_memory::tree::TreeBitMap;
+use crate::persist::lsm_tree::LongKey;
+use crate::prefix_cht::cht::PrefixCht;
+// use crate::prelude::multi::RouteStatus;
 use crate::stats::CreatedNodes;
-use crate::{
-    local_array::errors::PrefixStoreError, prefix_record::PublicRecord,
-};
+use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
+use crate::types::{PrefixId, RouteStatus};
+use crate::{types::errors::PrefixStoreError, types::PublicRecord};
 
 // Make sure to also import the other methods for the Rib, so the proc macro
 // create_store can use them.
-pub use crate::local_array::in_memory::iterators;
-pub use crate::local_array::query;
+pub use crate::in_memory::iterators;
+pub use crate::rib::query;
 
 use crate::{IPv4, IPv6, Meta};
 
@@ -334,8 +332,7 @@ pub struct Rib<
     pub config: C,
     pub(crate) in_memory_tree: TreeBitMap<AF, N_ROOT_SIZE>,
     pub(crate) prefix_cht: PrefixCht<AF, M, P_ROOT_SIZE>,
-    pub(in crate::local_array) persist_tree:
-        Option<PersistTree<AF, LongKey<AF>, KEY_SIZE>>,
+    pub(crate) persist_tree: Option<PersistTree<AF, LongKey<AF>, KEY_SIZE>>,
     pub counters: Counters,
 }
 
diff --git a/src/local_array/rib/default_store.rs b/src/rib/starcast.rs
similarity index 98%
rename from src/local_array/rib/default_store.rs
rename to src/rib/starcast.rs
index 4e011907..947154ce 100644
--- a/src/local_array/rib/default_store.rs
+++ b/src/rib/starcast.rs
@@ -1,7 +1,20 @@
-use crate::prelude::multi::*;
-use crate::prelude::*;
+use crossbeam_epoch::Guard;
+use inetnum::addr::Prefix;
+// use crate::prelude::multi::*;
+// use crate::prelude::*;
 use rand::prelude::*;
 
+use crate::{
+    epoch,
+    types::{errors::PrefixStoreError, PrefixId},
+    AddressFamily, IPv4, IPv6, MatchOptions, Meta, PrefixRecord, QueryResult,
+    Record,
+};
+
+use super::rib::{
+    Config, PersistStrategy, Rib, StoreStats, UpsertCounters, UpsertReport,
+};
+
 pub const STRIDE_SIZE: u8 = 4;
 pub const BIT_SPAN_SIZE: u8 = 32;
 
diff --git a/src/local_array/tests.rs b/src/tests.rs
similarity index 100%
rename from src/local_array/tests.rs
rename to src/tests.rs
diff --git a/src/af.rs b/src/types/af.rs
similarity index 100%
rename from src/af.rs
rename to src/types/af.rs
diff --git a/src/local_array/bit_span.rs b/src/types/bit_span.rs
similarity index 100%
rename from src/local_array/bit_span.rs
rename to src/types/bit_span.rs
diff --git a/src/local_array/errors.rs b/src/types/errors.rs
similarity index 100%
rename from src/local_array/errors.rs
rename to src/types/errors.rs
diff --git a/src/local_array/types.rs b/src/types/prefix_id.rs
similarity index 100%
rename from src/local_array/types.rs
rename to src/types/prefix_id.rs
diff --git a/src/prefix_record.rs b/src/types/prefix_record.rs
similarity index 100%
rename from src/prefix_record.rs
rename to src/types/prefix_record.rs
diff --git a/src/test_types.rs b/src/types/test_types.rs
similarity index 100%
rename from src/test_types.rs
rename to src/types/test_types.rs

From 6889b6c8b6fa43e18d3806d1f6e7bbd4da0beadc Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 5 Mar 2025 11:02:30 +0100
Subject: [PATCH 107/147] reshuffle modules, files & dirs

---
 src/mod.rs                                 | 10 ----
 src/prelude/mod.rs                         | 32 -------------
 src/rib/init                               |  1 -
 src/rib/mod.rs                             |  8 ++--
 src/rib/starcast.rs                        | 53 ++++++++++++++++++----
 src/rib/{rib.rs => starcast_af.rs}         | 26 ++++++-----
 src/rib/{query.rs => starcast_af_query.rs} |  4 +-
 src/{ => types}/stats.rs                   |  0
 8 files changed, 65 insertions(+), 69 deletions(-)
 delete mode 100644 src/mod.rs
 delete mode 100644 src/prelude/mod.rs
 delete mode 100644 src/rib/init
 rename src/rib/{rib.rs => starcast_af.rs} (97%)
 rename src/rib/{query.rs => starcast_af_query.rs} (98%)
 rename src/{ => types}/stats.rs (100%)

diff --git a/src/mod.rs b/src/mod.rs
deleted file mode 100644
index bc4bee30..00000000
--- a/src/mod.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-pub(crate) mod in_memory;
-pub(crate) mod persist;
-pub(crate) mod prefix_cht;
-mod tests;
-
-pub(crate) mod bit_span;
-pub mod errors;
-pub mod query;
-pub mod rib;
-pub mod types;
diff --git a/src/prelude/mod.rs b/src/prelude/mod.rs
deleted file mode 100644
index 243a62b3..00000000
--- a/src/prelude/mod.rs
+++ /dev/null
@@ -1,32 +0,0 @@
-pub use crate::{AddressFamily, IPv4, IPv6};
-
-pub use crate::af::IntoIpAddr;
-pub use crate::prefix_record::{Meta, PublicPrefixRecord as PrefixRecord};
-pub use crate::{IncludeHistory, MatchOptions, MatchType, QueryResult};
-pub use inetnum::addr::Prefix;
-
-pub mod multi {
-    pub use crate::MultiThreadedStore;
-
-    pub use std::sync::atomic::Ordering;
-
-    pub use crossbeam_epoch::{self as epoch, Guard};
-
-    pub use crate::local_array::errors::PrefixStoreError;
-    pub use crate::local_array::in_memory::iterators;
-    pub use crate::local_array::in_memory::node::StrideNodeId;
-    pub use crate::local_array::persist::lsm_tree::KeySize;
-    pub use crate::local_array::rib::rib::{
-        MemoryOnlyConfig, PersistHistoryConfig, PersistOnlyConfig,
-        WriteAheadConfig,
-    };
-    pub use crate::local_array::types::{PrefixId, RouteStatus};
-    pub use crate::prefix_record::PublicRecord as Record;
-
-    pub use crate::rib::Rib;
-    pub use crate::rib::{
-        Config, PersistStrategy, StoreStats, UpsertCounters, UpsertReport,
-    };
-
-    pub use routecore::bgp::path_selection::TiebreakerInfo;
-}
diff --git a/src/rib/init b/src/rib/init
deleted file mode 100644
index 8b137891..00000000
--- a/src/rib/init
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/src/rib/mod.rs b/src/rib/mod.rs
index 3c1ef1ff..e982b458 100644
--- a/src/rib/mod.rs
+++ b/src/rib/mod.rs
@@ -1,8 +1,8 @@
-pub mod query;
-pub mod rib;
 pub mod starcast;
+pub mod starcast_af;
+pub mod starcast_af_query;
 
-pub use rib::Counters;
-pub use starcast::StarCastDb;
+pub use starcast::StarCastRib;
 pub use starcast::BIT_SPAN_SIZE;
 pub use starcast::STRIDE_SIZE;
+pub use starcast_af::Counters;
diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index 947154ce..b1dd2b1f 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -11,20 +11,21 @@ use crate::{
     Record,
 };
 
-use super::rib::{
-    Config, PersistStrategy, Rib, StoreStats, UpsertCounters, UpsertReport,
+use super::starcast_af::{
+    Config, PersistStrategy, StarCastAfRib, StoreStats, UpsertCounters,
+    UpsertReport,
 };
 
 pub const STRIDE_SIZE: u8 = 4;
 pub const BIT_SPAN_SIZE: u8 = 32;
 
-pub struct StarCastDb<M: Meta, C: Config> {
-    v4: Rib<IPv4, M, 9, 33, C, 18>,
-    v6: Rib<IPv6, M, 33, 129, C, 30>,
+pub struct StarCastRib<M: Meta, C: Config> {
+    v4: StarCastAfRib<IPv4, M, 9, 33, C, 18>,
+    v6: StarCastAfRib<IPv6, M, 33, 129, C, 30>,
     config: C,
 }
 
-impl<'a, M: Meta, C: Config> StarCastDb<M, C> {
+impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     pub fn try_default() -> Result<Self, PrefixStoreError> {
         let config = C::default();
         Self::new_with_config(config)
@@ -53,8 +54,8 @@ impl<'a, M: Meta, C: Config> StarCastDb<M, C> {
         }
 
         Ok(Self {
-            v4: Rib::new(config_v4)?,
-            v6: Rib::new(config_v6)?,
+            v4: StarCastAfRib::new(config_v4)?,
+            v6: StarCastAfRib::new(config_v6)?,
             config,
         })
     }
@@ -422,6 +423,42 @@ impl<'a, M: Meta, C: Config> StarCastDb<M, C> {
         self.v6.prefixes_iter(guard).map(PrefixRecord::from)
     }
 
+    pub fn persist_prefixes_iter(
+        &'a self,
+    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+        self.v4
+            .persist_prefixes_iter()
+            .map(PrefixRecord::from)
+            .chain(self.v6.persist_prefixes_iter().map(PrefixRecord::from))
+    }
+
+    pub fn persist_prefixes_iter_v4(
+        &'a self,
+    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+        self.v4.persist_prefixes_iter().map(PrefixRecord::from)
+    }
+
+    pub fn persist_prefixes_iter_v6(
+        &'a self,
+    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+        self.v6.persist_prefixes_iter().map(PrefixRecord::from)
+    }
+
+    pub fn is_mui_active(&self, mui: u32) -> bool {
+        let guard = &epoch::pin();
+        self.v4.is_mui_active(mui, guard) || self.v6.is_mui_active(mui, guard)
+    }
+
+    pub fn is_mui_active_v4(&self, mui: u32) -> bool {
+        let guard = &epoch::pin();
+        self.v4.is_mui_active(mui, guard)
+    }
+
+    pub fn is_mui_active_v6(&self, mui: u32) -> bool {
+        let guard = &epoch::pin();
+        self.v6.is_mui_active(mui, guard)
+    }
+
     /// Change the local status of the record for the combination of
     /// (prefix, multi_uniq_id) to Withdrawn. Note that by default the
     /// global `Withdrawn` status for a mui overrides the local status
diff --git a/src/rib/rib.rs b/src/rib/starcast_af.rs
similarity index 97%
rename from src/rib/rib.rs
rename to src/rib/starcast_af.rs
index b68f54c2..5a18a530 100644
--- a/src/rib/rib.rs
+++ b/src/rib/starcast_af.rs
@@ -21,7 +21,7 @@ use crate::{types::errors::PrefixStoreError, types::PublicRecord};
 // Make sure to also import the other methods for the Rib, so the proc macro
 // create_store can use them.
 pub use crate::in_memory::iterators;
-pub use crate::rib::query;
+pub use crate::rib::starcast_af_query;
 
 use crate::{IPv4, IPv6, Meta};
 
@@ -321,7 +321,7 @@ pub struct UpsertReport {
 // A Routing Information Base that consists of multiple different trees for
 // in-memory and on-disk (persisted storage).
 #[derive(Debug)]
-pub struct Rib<
+pub(crate) struct StarCastAfRib<
     AF: AddressFamily,
     M: Meta,
     const N_ROOT_SIZE: usize,
@@ -343,15 +343,17 @@ impl<
         const N_ROOT_SIZE: usize,
         C: Config,
         const KEY_SIZE: usize,
-    > Rib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
+    > StarCastAfRib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
 {
     pub(crate) fn new(
         config: C,
     ) -> Result<
-        Rib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>,
+        StarCastAfRib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>,
         Box<dyn std::error::Error>,
     > {
-        Rib::<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>::init(config)
+        StarCastAfRib::<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>::init(
+            config,
+        )
     }
 
     fn init(config: C) -> Result<Self, Box<dyn std::error::Error>> {
@@ -366,7 +368,7 @@ impl<
             }
         };
 
-        let store = Rib {
+        let store = StarCastAfRib {
             config,
             in_memory_tree: TreeBitMap::<AF, N_ROOT_SIZE>::new()?,
             persist_tree,
@@ -696,7 +698,7 @@ impl<
     // Whether this mui is globally active. Note that the local statuses of
     // records (prefix, mui) may be set to withdrawn in iterators and match
     // functions.
-    pub fn mui_is_active(&self, mui: u32, guard: &Guard) -> bool {
+    pub(crate) fn is_mui_active(&self, mui: u32, guard: &Guard) -> bool {
         !unsafe {
             self.in_memory_tree
                 .withdrawn_muis_bmin
@@ -707,7 +709,7 @@ impl<
         .contains(mui)
     }
 
-    pub fn get_prefixes_count(&self) -> UpsertCounters {
+    pub(crate) fn get_prefixes_count(&self) -> UpsertCounters {
         UpsertCounters {
             in_memory_count: self.in_memory_tree.get_prefixes_count(),
             persisted_count: self
@@ -750,7 +752,7 @@ impl<
         self.config.persist_strategy()
     }
 
-    pub fn persist_prefixes_iter(
+    pub(crate) fn persist_prefixes_iter(
         &self,
     ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + '_ {
         self.persist_tree
@@ -787,7 +789,7 @@ impl<
             .flatten()
     }
 
-    pub fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
+    pub(crate) fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
         if let Some(p) = &self.persist_tree {
             p.flush_to_disk()
                 .map_err(|_| PrefixStoreError::PersistFailed)
@@ -820,7 +822,7 @@ impl<
         C: Config,
         const KEY_SIZE: usize,
     > std::fmt::Display
-    for Rib<IPv4, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
+    for StarCastAfRib<IPv4, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
 {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "Rib<IPv4, {}>", std::any::type_name::<M>())
@@ -834,7 +836,7 @@ impl<
         C: Config,
         const KEY_SIZE: usize,
     > std::fmt::Display
-    for Rib<IPv6, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
+    for StarCastAfRib<IPv6, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
 {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "Rib<IPv6, {}>", std::any::type_name::<M>())
diff --git a/src/rib/query.rs b/src/rib/starcast_af_query.rs
similarity index 98%
rename from src/rib/query.rs
rename to src/rib/starcast_af_query.rs
index a00b57e4..a2ca5926 100644
--- a/src/rib/query.rs
+++ b/src/rib/starcast_af_query.rs
@@ -3,7 +3,7 @@ use epoch::Guard;
 use log::trace;
 use zerocopy::TryFromBytes;
 
-use crate::rib::rib::{Config, PersistStrategy, Rib};
+use crate::rib::starcast_af::{Config, PersistStrategy, StarCastAfRib};
 use crate::types::prefix_record::ZeroCopyRecord;
 use crate::types::AddressFamily;
 use crate::types::PublicRecord;
@@ -26,7 +26,7 @@ impl<
         const P_ROOT_SIZE: usize,
         C: Config,
         const KEY_SIZE: usize,
-    > Rib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
+    > StarCastAfRib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
 {
     pub(crate) fn get_value(
         &'a self,
diff --git a/src/stats.rs b/src/types/stats.rs
similarity index 100%
rename from src/stats.rs
rename to src/types/stats.rs

From f99e5045285a75acedce1b2377ceb64dfa289b11 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 5 Mar 2025 13:14:28 +0100
Subject: [PATCH 108/147] more reshuffling

---
 .../{atomic_stride.rs => atomic_bitmap.rs}    |   0
 src/in_memory/deprecated_query.rs             | 701 ------------------
 .../{atomic_types.rs => node_cht.rs}          |   0
 src/in_memory/{tree.rs => tree_bitmap.rs}     |   0
 ...{iterators.rs => tree_bitmap_iterators.rs} |   0
 .../{node.rs => tree_bitmap_node.rs}          |   0
 .../{query.rs => tree_bitmap_query.rs}        |   0
 src/{in_memory => stride_table}/oncebox.rs    |   0
 src/types/af.rs                               |   2 +-
 src/types/bit_span.rs                         |   2 +-
 .../match_options.rs}                         |  53 +-
 src/types/mod.rs                              |  15 +
 src/types/prefix_record.rs                    |   8 +-
 src/types/stats.rs                            |   6 +-
 14 files changed, 26 insertions(+), 761 deletions(-)
 rename src/in_memory/{atomic_stride.rs => atomic_bitmap.rs} (100%)
 delete mode 100644 src/in_memory/deprecated_query.rs
 rename src/in_memory/{atomic_types.rs => node_cht.rs} (100%)
 rename src/in_memory/{tree.rs => tree_bitmap.rs} (100%)
 rename src/in_memory/{iterators.rs => tree_bitmap_iterators.rs} (100%)
 rename src/in_memory/{node.rs => tree_bitmap_node.rs} (100%)
 rename src/in_memory/{query.rs => tree_bitmap_query.rs} (100%)
 rename src/{in_memory => stride_table}/oncebox.rs (100%)
 rename src/{rotonda_store.rs => types/match_options.rs} (72%)
 create mode 100644 src/types/mod.rs

diff --git a/src/in_memory/atomic_stride.rs b/src/in_memory/atomic_bitmap.rs
similarity index 100%
rename from src/in_memory/atomic_stride.rs
rename to src/in_memory/atomic_bitmap.rs
diff --git a/src/in_memory/deprecated_query.rs b/src/in_memory/deprecated_query.rs
deleted file mode 100644
index 0a912c2b..00000000
--- a/src/in_memory/deprecated_query.rs
+++ /dev/null
@@ -1,701 +0,0 @@
-use log::trace;
-
-use crate::af::AddressFamily;
-use crate::prelude::multi::PrefixSet;
-use inetnum::addr::Prefix;
-
-use crate::{Meta, QueryResult};
-
-use crate::local_array::in_memory::node::{SizedStrideRef, TreeBitMapNode};
-use crate::{MatchOptions, MatchType};
-
-use super::super::in_memory::atomic_types::StoredPrefix;
-use super::super::in_memory::tree::Stride;
-use super::super::types::PrefixId;
-use super::atomic_types::{NodeBuckets, PrefixBuckets};
-use super::node::StrideNodeId;
-use super::tree::TreeBitMap;
-
-#[allow(dead_code)]
-impl<'a, AF, M, NB, PB> TreeBitMap<AF, M, NB, PB>
-where
-    AF: AddressFamily,
-    M: Meta,
-    NB: NodeBuckets<AF>,
-    PB: PrefixBuckets<AF, M>,
-{
-    #[allow(clippy::type_complexity)]
-    fn retrieve_prefix(
-        &'a self,
-        prefix_id: PrefixId<AF>,
-    ) -> Option<(&'a StoredPrefix<AF, M>, usize)> {
-        struct SearchLevel<
-            's,
-            AF: AddressFamily,
-            M: crate::prefix_record::Meta,
-        > {
-            f: &'s dyn for<'a> Fn(
-                &SearchLevel<AF, M>,
-                &'a PrefixSet<AF, M>,
-                u8,
-            )
-                -> Option<(&'a StoredPrefix<AF, M>, usize)>,
-        }
-
-        let search_level = SearchLevel {
-            f: &|search_level: &SearchLevel<AF, M>,
-                 prefix_set: &PrefixSet<AF, M>,
-                 mut level: u8| {
-                // HASHING FUNCTION
-                let index =
-                    crate::local_array::in_memory::tree::TreeBitMap::<
-                        AF,
-                        M,
-                        NB,
-                        PB,
-                    >::hash_prefix_id(prefix_id, level);
-
-                if let Some(stored_prefix) = prefix_set.0.get(index) {
-                    if prefix_id == stored_prefix.prefix {
-                        trace!("found requested prefix {:?}", prefix_id,);
-                        return Some((stored_prefix, 0));
-                    };
-                    level += 1;
-
-                    (search_level.f)(
-                        search_level,
-                        &stored_prefix.next_bucket,
-                        level,
-                    );
-                }
-                None
-            },
-        };
-
-        (search_level.f)(
-            &search_level,
-            self.prefix_buckets.get_root_prefix_set(prefix_id.get_len()),
-            0,
-        )
-    }
-
-    // This function assembles all entries in the `pfx_vec` of all child nodes
-    // of the `start_node` into one vec, starting from itself and then
-    // recursively assembling adding all `pfx_vec`s of its children.
-    fn _get_all_more_specifics_for_node(
-        &self,
-        start_node_id: StrideNodeId<AF>,
-        found_pfx_vec: &mut Vec<PrefixId<AF>>,
-    ) {
-        trace!("{:?}", self.retrieve_node(start_node_id));
-        match self.retrieve_node(start_node_id) {
-            Some(SizedStrideRef::Stride3(n)) => {
-                found_pfx_vec.extend(
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
-                );
-
-                for child_node in n.ptr_iter(start_node_id) {
-                    self._get_all_more_specifics_for_node(
-                        child_node,
-                        found_pfx_vec,
-                    );
-                }
-            }
-            Some(SizedStrideRef::Stride4(n)) => {
-                found_pfx_vec.extend(
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
-                );
-
-                for child_node in n.ptr_iter(start_node_id) {
-                    self._get_all_more_specifics_for_node(
-                        child_node,
-                        found_pfx_vec,
-                    );
-                }
-            }
-            Some(SizedStrideRef::Stride5(n)) => {
-                found_pfx_vec.extend(
-                    n.pfx_iter(start_node_id).collect::<Vec<PrefixId<AF>>>(),
-                );
-
-                for child_node in n.ptr_iter(start_node_id) {
-                    self._get_all_more_specifics_for_node(
-                        child_node,
-                        found_pfx_vec,
-                    );
-                }
-            }
-            _ => {
-                panic!("can't find node {}", start_node_id);
-            }
-        }
-    }
-
-    // This function assembles the prefixes of a child node starting on a
-    // specified bit position in a ptr_vec of `current_node` into a vec,
-    // then adds all prefixes of these children recursively into a vec and
-    // returns that.
-    fn _get_all_more_specifics_from_nibble<S: Stride>(
-        &self,
-        current_node: &TreeBitMapNode<AF, S>,
-        nibble: u32,
-        nibble_len: u8,
-        base_prefix: StrideNodeId<AF>,
-    ) -> Option<Vec<PrefixId<AF>>> {
-        let (cnvec, mut msvec) = current_node.add_more_specifics_at(
-            nibble,
-            nibble_len,
-            base_prefix,
-        );
-
-        for child_node in cnvec.iter() {
-            self._get_all_more_specifics_for_node(*child_node, &mut msvec);
-        }
-        Some(msvec)
-    }
-
-    // In a LMP search we have to go over all the nibble lengths in the
-    // stride up until the value of the actual nibble length were looking for
-    // (until we reach stride length for all strides that aren't the last)
-    // and see if the prefix bit in that position is set. Note that this does
-    // not search for prefixes with length 0 (which would always match).
-    // So for matching a nibble 1010, we have to search for 1, 10, 101 and
-    // 1010 on resp. position 1, 5, 12 and 25:
-    //                       ↓          ↓                         ↓
-    // nibble              * 0 1 00 01 10 11 000 001 010 011 100 101 110 111
-    // nibble len offset   0 1    2            3
-    //
-    // (contd.)
-    // pfx bit arr (u32)     15   16   17   18   19   20   21   22   23   24
-    // nibble              0000 0001 0010 0011 0100 0101 0110 0111 1000 1001
-    // nibble len offset      4
-    //
-    // (contd.)               ↓
-    // pfx bit arr (u32)     25   26   27   28   29   30   31
-    // nibble              1010 1011 1100 1101 1110 1111    x
-    // nibble len offset      4(contd.)
-
-    fn _match_prefix_by_tree_traversal(
-        &'a self,
-        search_pfx: PrefixId<AF>,
-        options: &MatchOptions,
-        // guard: &'a Guard,
-    ) -> QueryResult<M> {
-        // --- The Default Route Prefix -------------------------------------
-
-        // The Default Route Prefix unfortunately does not fit in tree as we
-        // have it. There's no room for it in the pfxbitarr of the root node,
-        // since that can only contain serial numbers for prefixes that are
-        // children of the root node. We, however, want the default prefix
-        // which lives on the root node itself! We are *not* going to return
-        // all of the prefixes in the tree as more-specifics.
-        if search_pfx.get_len() == 0 {
-            // match self.load_default_route_prefix_serial() {
-            //     0 => {
-            //         return QueryResult {
-            //             prefix: None,
-            //             prefix_meta: vec![],
-            //             match_type: MatchType::EmptyMatch,
-            //             less_specifics: None,
-            //             more_specifics: None,
-            //         };
-            //     }
-
-            //     _serial => {
-            return QueryResult {
-                prefix: None,
-                prefix_meta: vec![],
-                match_type: MatchType::EmptyMatch,
-                less_specifics: None,
-                more_specifics: None,
-            };
-            // }
-            // }
-        }
-
-        let mut stride_end = 0;
-
-        let root_node_id = self.get_root_node_id();
-        let mut node = match self.get_stride_for_id(root_node_id) {
-            3 => self.retrieve_node(root_node_id).unwrap(),
-            4 => self.retrieve_node(root_node_id).unwrap(),
-            _ => self.retrieve_node(root_node_id).unwrap(),
-        };
-
-        let mut nibble;
-        let mut nibble_len;
-
-        //---- result values ------------------------------------------------
-
-        // These result values are kept in mutable variables, and assembled
-        // at the end into a QueryResult struct. This proved to result in the
-        // most efficient code, where we don't have to match on
-        // SizedStrideNode over and over. The `match_type` field in the
-        // QueryResult is computed at the end.
-
-        // The final prefix
-        let mut match_prefix_idx: Option<PrefixId<AF>> = None;
-
-        // The indexes of the less-specifics
-        let mut less_specifics_vec = if options.include_less_specifics {
-            Some(Vec::<PrefixId<AF>>::new())
-        } else {
-            None
-        };
-
-        // The indexes of the more-specifics.
-        let mut more_specifics_vec = if options.include_more_specifics {
-            Some(Vec::<PrefixId<AF>>::new())
-        } else {
-            None
-        };
-
-        //---- Stride Processing --------------------------------------------
-
-        // We're going to iterate over all the strides in the treebitmap (so
-        // up to the last bit in the max prefix length for that tree). When
-        // a final prefix is found or we get to the end of the strides,
-        // depending on the options.match_type (the type requested by the
-        // user). we ALWAYS break out of the loop. WE ALWAYS BREAK OUT OF THE
-        // LOOP. Just before breaking some processing is done inside the loop
-        // before the break (looking up more-specifics mainly), which looks a
-        // bit repetitious, but again it's been done like that to avoid
-        // having to match over a SizedStrideNode again in the
-        // `post-processing` section.
-
-        for stride in self.get_stride_sizes() {
-            stride_end += stride;
-
-            let last_stride = search_pfx.get_len() < stride_end;
-
-            nibble_len = if last_stride {
-                stride + search_pfx.get_len() - stride_end
-            } else {
-                *stride
-            };
-
-            // Shift left and right to set the bits to zero that are not
-            // in the nibble we're handling here.
-            nibble = AddressFamily::get_nibble(
-                search_pfx.get_net(),
-                stride_end - stride,
-                nibble_len,
-            );
-
-            match node {
-                SizedStrideRef::Stride3(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-
-                    // This whole match assumes that:
-                    // - if the first value in the return tuple of
-                    //   `search_fn` holds a value, then we need to continue
-                    //   searching by following the node contained in the
-                    //   value.
-                    // - The second value in the tuple holds the prefix that
-                    //   was found.
-                    // The less_specifics_vec is mutated by `search_fn` to
-                    // hold the prefixes found along the way, in the cases
-                    // where `include_less_specifics` was requested by the
-                    // user.
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        // This and the next match will handle all
-                        // intermediary nodes, but they might also handle
-                        // exit nodes.
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx);
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        ._get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        ._get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        // This handles exact and longest matches: there are
-                        // no more children, but there is a prefix on this
-                        // node.
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    ._get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                        StrideNodeId::new_with_cleaned_id(
-                                            search_pfx.get_net(),
-                                            stride_end - stride,
-                                        ),
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx);
-                            break;
-                        }
-                        // This handles cases where there's no prefix (and no
-                        // child) for exact match or longest match, the empty
-                        // match - which doesn't care about actually finding
-                        // a prefix - just continues in search of
-                        // more-specifics.
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    // To make sure we don't process this
-                                    // match arm more then once, we return
-                                    // early here.
-                                    more_specifics_vec = self
-                                        ._get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                //---- From here only repetitions for all strides -----------
-                // For comments see the code above for the Stride3 arm.
-                SizedStrideRef::Stride4(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx);
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        ._get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        ._get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    ._get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                        StrideNodeId::new_with_cleaned_id(
-                                            search_pfx.get_net(),
-                                            stride_end - stride,
-                                        ),
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx);
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    // To make sure we don't process this match arm more then once, we
-                                    // return early here.
-                                    more_specifics_vec = self
-                                        ._get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-                SizedStrideRef::Stride5(current_node) => {
-                    let search_fn = match options.match_type {
-                        MatchType::ExactMatch => {
-                            if options.include_less_specifics {
-                                TreeBitMapNode::search_stride_for_exact_match_with_less_specifics_at
-                            } else {
-                                TreeBitMapNode::search_stride_for_exact_match_at
-                            }
-                        }
-                        MatchType::LongestMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                        MatchType::EmptyMatch => {
-                            TreeBitMapNode::search_stride_for_longest_match_at
-                        }
-                    };
-                    match search_fn(
-                        current_node,
-                        search_pfx,
-                        nibble,
-                        nibble_len,
-                        stride_end - stride,
-                        &mut less_specifics_vec,
-                    ) {
-                        (Some(n), Some(pfx_idx)) => {
-                            match_prefix_idx = Some(pfx_idx);
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        ._get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (Some(n), None) => {
-                            node = self.retrieve_node(n).unwrap();
-
-                            if last_stride {
-                                if options.include_more_specifics {
-                                    more_specifics_vec = self
-                                        ._get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-                                }
-                                break;
-                            }
-                        }
-                        (None, Some(pfx_idx)) => {
-                            if options.include_more_specifics {
-                                more_specifics_vec = self
-                                    ._get_all_more_specifics_from_nibble(
-                                        current_node,
-                                        nibble,
-                                        nibble_len,
-                                        StrideNodeId::new_with_cleaned_id(
-                                            search_pfx.get_net(),
-                                            stride_end - stride,
-                                        ),
-                                    );
-                            }
-                            match_prefix_idx = Some(pfx_idx);
-                            break;
-                        }
-                        (None, None) => {
-                            match options.match_type {
-                                MatchType::EmptyMatch => {
-                                    more_specifics_vec = self
-                                        ._get_all_more_specifics_from_nibble(
-                                            current_node,
-                                            nibble,
-                                            nibble_len,
-                                            StrideNodeId::new_with_cleaned_id(
-                                                search_pfx.get_net(),
-                                                stride_end - stride,
-                                            ),
-                                        );
-
-                                    match_prefix_idx = None;
-                                    break;
-                                }
-                                MatchType::LongestMatch => {}
-                                MatchType::ExactMatch => {
-                                    match_prefix_idx = None;
-                                }
-                            }
-                            break;
-                        }
-                    }
-                }
-            }
-        }
-        //------------------ end of Stride branch arm repetition ------------
-
-        //------------------ post-processing --------------------------------
-
-        // If the above loop finishes (so not hitting a break) we have
-        // processed all strides and have found a child node and maybe a
-        // prefix. Now we will look up more-specifics for longest-matching
-        // prefixes that were found in the last stride only. Note that still
-        // any of the match_types (as specified by the user, not the return
-        // type) may end up here.
-
-        let mut match_type: MatchType = MatchType::EmptyMatch;
-        let prefix = None;
-        if let Some(pfx_idx) = match_prefix_idx {
-            match_type = match self.retrieve_prefix(pfx_idx) {
-                Some(prefix) => {
-                    if prefix.0.prefix.get_len() == search_pfx.get_len() {
-                        MatchType::ExactMatch
-                    } else {
-                        MatchType::LongestMatch
-                    }
-                }
-                None => MatchType::EmptyMatch,
-            };
-        };
-
-        QueryResult {
-            prefix: prefix.map(|pfx: (&StoredPrefix<AF, M>, usize)| {
-                pfx.0.prefix.into_pub()
-            }),
-            prefix_meta: prefix
-                .map(|pfx| pfx.0.record_map.as_records())
-                .unwrap_or_default(),
-            match_type,
-            less_specifics: if options.include_less_specifics {
-                less_specifics_vec
-                    .unwrap()
-                    .iter()
-                    .filter_map(move |p| {
-                        self.retrieve_prefix(*p).map(|p| {
-                            Some((p.0.prefix, p.0.record_map.as_records()))
-                        })
-                    })
-                    .collect()
-            } else {
-                None
-            },
-            more_specifics: if options.include_more_specifics {
-                more_specifics_vec.map(|vec| {
-                    vec.into_iter()
-                        .map(|p| {
-                            self.retrieve_prefix(p)
-                                .unwrap_or_else(|| {
-                                    panic!(
-                                        "more specific {:?} does not exist",
-                                        p
-                                    )
-                                })
-                                .0
-                        })
-                        .map(|sp| (sp.prefix, sp.record_map.as_records()))
-                        .collect()
-                })
-            } else {
-                None
-            },
-        }
-    }
-}
diff --git a/src/in_memory/atomic_types.rs b/src/in_memory/node_cht.rs
similarity index 100%
rename from src/in_memory/atomic_types.rs
rename to src/in_memory/node_cht.rs
diff --git a/src/in_memory/tree.rs b/src/in_memory/tree_bitmap.rs
similarity index 100%
rename from src/in_memory/tree.rs
rename to src/in_memory/tree_bitmap.rs
diff --git a/src/in_memory/iterators.rs b/src/in_memory/tree_bitmap_iterators.rs
similarity index 100%
rename from src/in_memory/iterators.rs
rename to src/in_memory/tree_bitmap_iterators.rs
diff --git a/src/in_memory/node.rs b/src/in_memory/tree_bitmap_node.rs
similarity index 100%
rename from src/in_memory/node.rs
rename to src/in_memory/tree_bitmap_node.rs
diff --git a/src/in_memory/query.rs b/src/in_memory/tree_bitmap_query.rs
similarity index 100%
rename from src/in_memory/query.rs
rename to src/in_memory/tree_bitmap_query.rs
diff --git a/src/in_memory/oncebox.rs b/src/stride_table/oncebox.rs
similarity index 100%
rename from src/in_memory/oncebox.rs
rename to src/stride_table/oncebox.rs
diff --git a/src/types/af.rs b/src/types/af.rs
index 79ffeeb6..f3875472 100644
--- a/src/types/af.rs
+++ b/src/types/af.rs
@@ -1,7 +1,7 @@
 use log::trace;
 use zerocopy::{IntoBytes, NetworkEndian, U128, U32};
 
-use crate::local_array::bit_span::BitSpan;
+use crate::types::BitSpan;
 
 //------------ AddressFamily (trait) ----------------------------------------
 /// The address family of an IP address as a Trait.
diff --git a/src/types/bit_span.rs b/src/types/bit_span.rs
index a23a6681..483d66b8 100644
--- a/src/types/bit_span.rs
+++ b/src/types/bit_span.rs
@@ -1,4 +1,4 @@
-use super::rib::default_store::BIT_SPAN_SIZE;
+use crate::rib::BIT_SPAN_SIZE;
 
 #[derive(Copy, Clone, Debug)]
 pub struct BitSpan {
diff --git a/src/rotonda_store.rs b/src/types/match_options.rs
similarity index 72%
rename from src/rotonda_store.rs
rename to src/types/match_options.rs
index 21a7e14d..fa768e09 100644
--- a/src/rotonda_store.rs
+++ b/src/types/match_options.rs
@@ -1,23 +1,9 @@
-use std::{fmt, slice};
+use std::fmt;
 
-use crate::prefix_record::InternalPrefixRecord;
-pub use crate::prefix_record::{
-    Meta, PublicPrefixSingleRecord, RecordSingleSet,
-};
-use crate::prefix_record::{PublicRecord, RecordSet};
+use crate::types::{prefix_record::RecordSet, Meta, PublicRecord};
 
 use inetnum::addr::Prefix;
 
-pub use crate::af::{AddressFamily, IPv4, IPv6};
-
-pub use crate::local_array::rib::rib;
-
-pub const RECORDS_MAX_NUM: usize = 3;
-
-//------------ The publicly available Rotonda Stores ------------------------
-
-pub use crate::local_array::rib::StarCastDb as MultiThreadedStore;
-
 //------------ MatchOptions / MatchType -------------------------------------
 
 /// Options for the `match_prefix` method
@@ -81,41 +67,6 @@ pub enum IncludeHistory {
     All,
 }
 
-//------------ PrefixRecordIter ---------------------------------------------
-
-// Converts from the InternalPrefixRecord to the (public) PrefixRecord
-// while iterating.
-#[derive(Clone, Debug)]
-pub struct PrefixSingleRecordIter<'a, M: Meta> {
-    pub(crate) v4: Option<slice::Iter<'a, InternalPrefixRecord<IPv4, M>>>,
-    pub(crate) v6: slice::Iter<'a, InternalPrefixRecord<IPv6, M>>,
-}
-
-impl<M: Meta> Iterator for PrefixSingleRecordIter<'_, M> {
-    type Item = PublicPrefixSingleRecord<M>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        // V4 is already done.
-        if self.v4.is_none() {
-            return self.v6.next().map(|res| {
-                PublicPrefixSingleRecord::new(
-                    Prefix::new(res.net.into_ipaddr(), res.len).unwrap(),
-                    res.meta.clone(),
-                )
-            });
-        }
-
-        if let Some(res) = self.v4.as_mut().and_then(|v4| v4.next()) {
-            return Some(PublicPrefixSingleRecord::new(
-                Prefix::new(res.net.into_ipaddr(), res.len).unwrap(),
-                res.meta.clone(),
-            ));
-        }
-        self.v4 = None;
-        self.next()
-    }
-}
-
 //------------- QueryResult -------------------------------------------------
 
 /// The type that is returned by a query.
diff --git a/src/types/mod.rs b/src/types/mod.rs
new file mode 100644
index 00000000..ebd7193b
--- /dev/null
+++ b/src/types/mod.rs
@@ -0,0 +1,15 @@
+pub(crate) mod af;
+pub(crate) mod bit_span;
+pub(crate) mod match_options;
+pub(crate) mod prefix_id;
+pub(crate) mod prefix_record;
+
+pub use af::AddressFamily;
+pub(crate) use bit_span::BitSpan;
+pub(crate) use prefix_id::{PrefixId, RouteStatus};
+
+pub use prefix_record::Meta;
+pub mod errors;
+pub mod stats;
+pub mod test_types;
+pub(crate) use prefix_record::PublicRecord;
diff --git a/src/types/prefix_record.rs b/src/types/prefix_record.rs
index 33169ebc..28789051 100644
--- a/src/types/prefix_record.rs
+++ b/src/types/prefix_record.rs
@@ -2,15 +2,17 @@ use std::fmt;
 use std::fmt::Debug;
 use std::{cmp::Ordering, sync::Arc};
 
-use crate::af::AddressFamily;
-use crate::local_array::types::RouteStatus;
-use crate::prelude::multi::PrefixId;
+// use crate::prelude::multi::PrefixId;
+use crate::types::AddressFamily;
+use crate::types::RouteStatus;
 use inetnum::addr::Prefix;
 use zerocopy::{
     Immutable, IntoBytes, KnownLayout, NetworkEndian, TryFromBytes,
     Unaligned, U128, U32,
 };
 
+use super::PrefixId;
+
 //------------ InternalPrefixRecord -----------------------------------------
 
 // This struct is used for the SingleThreadedStore only.
diff --git a/src/types/stats.rs b/src/types/stats.rs
index e8fe5d2d..97b370c0 100644
--- a/src/types/stats.rs
+++ b/src/types/stats.rs
@@ -7,10 +7,8 @@ use std::{
 };
 
 use crate::{
-    local_array::{
-        in_memory::node::TreeBitMapNode, rib::default_store::STRIDE_SIZE,
-    },
-    AddressFamily,
+    in_memory::tree_bitmap_node::TreeBitMapNode, rib::STRIDE_SIZE,
+    types::AddressFamily,
 };
 
 pub struct StrideStats<AF: AddressFamily> {

From 0876c97a5a3da25f06d94e937cfb5beaa28cafe8 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 5 Mar 2025 13:15:11 +0100
Subject: [PATCH 109/147] more reshuffling

---
 examples/exact_matches.rs                  |  12 +-
 examples/exact_matches_single.rs           |  14 +-
 examples/full_table_multiple_trees_json.rs |  14 +-
 examples/more_specifics.rs                 |  12 +-
 examples/multi_no_thread.rs                |  14 +-
 examples/multi_single_thread.rs            |  10 +-
 examples/multi_thread_1.rs                 |  14 +-
 examples/multi_thread_2.rs                 |  19 +-
 examples/multi_thread_3.rs                 |  16 +-
 examples/multi_thread_4.rs                 |  16 +-
 examples/multi_thread_multi_prefix.rs      |   8 +-
 examples/multi_thread_single_prefix.rs     |  16 +-
 examples/numbers_treebitmap.rs             |   9 +-
 examples/real_single_thread_24.rs          |   7 +-
 examples/single_thread_24.rs               |  10 +-
 examples/treebitmap.rs                     |  11 +-
 src/bin/cli.rs                             |   8 +-
 src/bin/load_mrt.rs                        |  49 +-
 src/cht/mod.rs                             |  42 ++
 src/{stride_table => cht}/oncebox.rs       |   2 +-
 src/in_memory/atomic_bitmap.rs             | 177 +------
 src/in_memory/mod.rs                       |  20 +-
 src/in_memory/node_cht.rs                  | 576 +--------------------
 src/in_memory/tree_bitmap.rs               |  37 +-
 src/in_memory/tree_bitmap_iterators.rs     |  12 +-
 src/in_memory/tree_bitmap_node.rs          |  52 +-
 src/in_memory/tree_bitmap_query.rs         |   8 +-
 src/lib.rs                                 |  40 +-
 src/macros.rs                              |  16 +-
 src/persist/lsm_tree.rs                    |   9 +-
 src/prefix_cht/cht.rs                      | 506 +++++++++++++++++-
 src/prefix_cht/iterators.rs                |  13 +-
 src/query.rs                               | 395 --------------
 src/rib/starcast_af.rs                     |   6 +-
 src/tests.rs                               |   7 +-
 tests/best-path.rs                         |  12 +-
 tests/concurrency.rs                       |  14 +-
 tests/full-table.rs                        |   8 +-
 tests/more-more-specifics.rs               |  11 +-
 tests/more-specifics.rs                    |  10 +-
 tests/treebitmap.rs                        |  16 +-
 tests/treebitmap_v6.rs                     |  90 +++-
 42 files changed, 933 insertions(+), 1405 deletions(-)
 create mode 100644 src/cht/mod.rs
 rename src/{stride_table => cht}/oncebox.rs (99%)
 delete mode 100644 src/query.rs

diff --git a/examples/exact_matches.rs b/examples/exact_matches.rs
index f69b6183..abb60652 100644
--- a/examples/exact_matches.rs
+++ b/examples/exact_matches.rs
@@ -1,12 +1,14 @@
+use inetnum::addr::Prefix;
 use rotonda_store::meta_examples::NoMeta;
-use rotonda_store::prelude::multi::*;
-use rotonda_store::rib::MemoryOnlyConfig;
-use rotonda_store::{prelude::*, IncludeHistory};
+// use rotonda_store::prelude::multi::*;
+use rotonda_store::{
+    epoch, IncludeHistory, IntoIpAddr, MatchOptions, MatchType,
+    MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
+};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let guard = &epoch::pin();
-    let tree_bitmap =
-        MultiThreadedStore::<NoMeta, MemoryOnlyConfig>::try_default()?;
+    let tree_bitmap = StarCastRib::<NoMeta, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new_relaxed(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
diff --git a/examples/exact_matches_single.rs b/examples/exact_matches_single.rs
index 25fe86ee..34df1b92 100644
--- a/examples/exact_matches_single.rs
+++ b/examples/exact_matches_single.rs
@@ -1,12 +1,12 @@
-use multi::MemoryOnlyConfig;
-use multi::Record;
+use inetnum::addr::Prefix;
 use rotonda_store::meta_examples::NoMeta;
-use rotonda_store::prelude::*;
-use rotonda_store::MultiThreadedStore;
+use rotonda_store::{
+    IncludeHistory, IntoIpAddr, MatchOptions, MatchType, MemoryOnlyConfig,
+    Record, RouteStatus, StarCastRib,
+};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let tree_bitmap =
-        MultiThreadedStore::<NoMeta, MemoryOnlyConfig>::try_default()?;
+    let tree_bitmap = StarCastRib::<NoMeta, MemoryOnlyConfig>::try_default()?;
 
     let pfxs = vec![
         Prefix::new_relaxed(
@@ -263,7 +263,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         // let p : rotonda_store::Prefix<u32, PrefixAs> = pfx.into();
         tree_bitmap.insert(
             &pfx.unwrap(),
-            Record::new(1, 0, multi::RouteStatus::Active, NoMeta::Empty),
+            Record::new(1, 0, RouteStatus::Active, NoMeta::Empty),
             None,
         )?;
     }
diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index b8ba41ce..4577f542 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -1,8 +1,10 @@
-// extern crate self as roto;
+use inetnum::addr::Prefix;
+use rotonda_store::epoch;
 use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::{
+    IncludeHistory, MatchOptions, MatchType, MemoryOnlyConfig, PrefixRecord,
+    Record, RouteStatus, StarCastRib,
+};
 
 use std::error::Error;
 use std::fs::File;
@@ -54,7 +56,9 @@ fn main() -> Result<(), Box<dyn Error>> {
             let mut rec_vec: Vec<PrefixRecord<PrefixAs>> = vec![];
             let config = MemoryOnlyConfig;
             let tree_bitmap =
-                MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::new_with_config(config)?;
+                StarCastRib::<PrefixAs, MemoryOnlyConfig>::new_with_config(
+                    config,
+                )?;
 
             if let Err(err) = load_prefixes(&mut rec_vec) {
                 println!("error running example: {}", err);
diff --git a/examples/more_specifics.rs b/examples/more_specifics.rs
index a3f893d0..2985f47c 100644
--- a/examples/more_specifics.rs
+++ b/examples/more_specifics.rs
@@ -1,14 +1,14 @@
-use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
+use rotonda_store::{
+    epoch, meta_examples::PrefixAs, IncludeHistory, IntoIpAddr, MatchOptions,
+    MatchType, Record, RouteStatus, StarCastRib,
+};
 
 use inetnum::addr::Prefix;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::MemoryOnlyConfig;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // type StoreType = InMemStorage<u32, PrefixAs>;
     let tree_bitmap =
-        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
+        StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new_relaxed(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
diff --git a/examples/multi_no_thread.rs b/examples/multi_no_thread.rs
index a5924214..32a10caa 100644
--- a/examples/multi_no_thread.rs
+++ b/examples/multi_no_thread.rs
@@ -1,9 +1,15 @@
+use inetnum::addr::Prefix;
 use log::trace;
 
+use rotonda_store::epoch;
 use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::IncludeHistory;
+use rotonda_store::IntoIpAddr;
+use rotonda_store::MatchOptions;
+use rotonda_store::MemoryOnlyConfig;
+use rotonda_store::Record;
+use rotonda_store::RouteStatus;
+use rotonda_store::StarCastRib;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
@@ -11,7 +17,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     trace!("Starting multi-threaded yolo testing....");
     let tree_bitmap =
-        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
+        StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     // let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_single_thread.rs b/examples/multi_single_thread.rs
index 5de76bdb..5187bfae 100644
--- a/examples/multi_single_thread.rs
+++ b/examples/multi_single_thread.rs
@@ -1,13 +1,13 @@
+use inetnum::addr::Prefix;
 use log::trace;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::{
+    IntoIpAddr, MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
+};
 use std::thread;
 use std::time::Duration;
 
 use rand::Rng;
 
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
-
 use rotonda_store::meta_examples::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -16,7 +16,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     trace!("Starting multi-threaded yolo testing....");
     let tree_bitmap =
-        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
+        StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     // let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_thread_1.rs b/examples/multi_thread_1.rs
index 7672cff3..5f869298 100644
--- a/examples/multi_thread_1.rs
+++ b/examples/multi_thread_1.rs
@@ -1,14 +1,14 @@
 use std::{sync::Arc, thread};
 
-use rotonda_store::meta_examples::NoMeta;
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
-use rotonda_store::rib::MemoryOnlyConfig;
+use inetnum::addr::Prefix;
+use rotonda_store::{
+    epoch, meta_examples::NoMeta, IncludeHistory, IntoIpAddr, MatchOptions,
+    MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
+};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let tree_bitmap = Arc::new(
-        MultiThreadedStore::<NoMeta, MemoryOnlyConfig>::try_default()?,
-    );
+    let tree_bitmap =
+        Arc::new(StarCastRib::<NoMeta, MemoryOnlyConfig>::try_default()?);
 
     let _: Vec<_> = (0..16)
         .map(|i: i32| {
diff --git a/examples/multi_thread_2.rs b/examples/multi_thread_2.rs
index 9175e471..5ef0c9cd 100644
--- a/examples/multi_thread_2.rs
+++ b/examples/multi_thread_2.rs
@@ -1,22 +1,21 @@
+use inetnum::addr::Prefix;
 use log::trace;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::epoch;
+use rotonda_store::meta_examples::PrefixAs;
+use rotonda_store::{
+    IncludeHistory, IntoIpAddr, MatchOptions, MemoryOnlyConfig, Record,
+    RouteStatus, StarCastRib,
+};
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
-
-use rotonda_store::meta_examples::PrefixAs;
-
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = Arc::new(MultiThreadedStore::<
-        PrefixAs,
-        MemoryOnlyConfig,
-    >::try_default()?);
+    let tree_bitmap =
+        Arc::new(StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?);
     let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_thread_3.rs b/examples/multi_thread_3.rs
index 035d6f18..f60763cd 100644
--- a/examples/multi_thread_3.rs
+++ b/examples/multi_thread_3.rs
@@ -1,11 +1,13 @@
+use inetnum::addr::Prefix;
 use log::trace;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::{epoch, IncludeHistory};
+use rotonda_store::{
+    IntoIpAddr, MatchOptions, MemoryOnlyConfig, Record, RouteStatus,
+    StarCastRib,
+};
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
-
 use rotonda_store::meta_examples::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -13,10 +15,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = Arc::new(MultiThreadedStore::<
-        PrefixAs,
-        MemoryOnlyConfig,
-    >::try_default()?);
+    let tree_bitmap =
+        Arc::new(StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?);
     let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_thread_4.rs b/examples/multi_thread_4.rs
index ecca4f12..3643805c 100644
--- a/examples/multi_thread_4.rs
+++ b/examples/multi_thread_4.rs
@@ -1,12 +1,13 @@
+use inetnum::addr::Prefix;
 use inetnum::asn::Asn;
 use log::trace;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::{
+    epoch, IncludeHistory, IntoIpAddr, MatchOptions, MemoryOnlyConfig, Meta,
+    Record, RouteStatus, StarCastRib,
+};
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
-
 #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
 pub struct BytesPrefixAs(pub [u8; 4]);
 
@@ -42,10 +43,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = Arc::new(MultiThreadedStore::<
-        BytesPrefixAs,
-        MemoryOnlyConfig,
-    >::try_default()?);
+    let tree_bitmap = Arc::new(
+        StarCastRib::<BytesPrefixAs, MemoryOnlyConfig>::try_default()?,
+    );
     let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let pfx = Prefix::new_relaxed(
diff --git a/examples/multi_thread_multi_prefix.rs b/examples/multi_thread_multi_prefix.rs
index 446c0a38..7015aeae 100644
--- a/examples/multi_thread_multi_prefix.rs
+++ b/examples/multi_thread_multi_prefix.rs
@@ -1,7 +1,6 @@
+use inetnum::addr::Prefix;
 use log::trace;
 
-use rotonda_store::prelude::multi::*;
-use rotonda_store::rib::MemoryOnlyConfig;
 use std::sync::atomic::AtomicU32;
 use std::sync::Arc;
 use std::thread;
@@ -10,8 +9,7 @@ use std::time::Duration;
 use rand::Rng;
 
 use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::prelude::*;
-use rotonda_store::MultiThreadedStore;
+use rotonda_store::{IncludeHistory, IntoIpAddr, MatchOptions, MemoryOnlyConfig, StarCastRib, Record, RouteStatus};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
@@ -19,7 +17,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     trace!("Starting multi-threaded yolo testing....");
     let tree_bitmap =
-        Arc::new(MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?);
+        Arc::new(StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?);
     // let pfx = Prefix::new_relaxed(
     //     0b1111_1111_1111_1111_1111_1111_1111_1111_u32.into_ipaddr(),
     //     32,
diff --git a/examples/multi_thread_single_prefix.rs b/examples/multi_thread_single_prefix.rs
index 304e6408..660a2173 100644
--- a/examples/multi_thread_single_prefix.rs
+++ b/examples/multi_thread_single_prefix.rs
@@ -1,7 +1,5 @@
 use log::trace;
 
-use rotonda_store::prelude::multi::*;
-use rotonda_store::rib::MemoryOnlyConfig;
 use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
@@ -9,20 +7,20 @@ use std::time::Duration;
 use rand::Rng;
 
 use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::prelude::*;
-use rotonda_store::MultiThreadedStore;
+use rotonda_store::{
+    IncludeHistory, IntoIpAddr, MatchOptions, MemoryOnlyConfig, Record,
+    RouteStatus, StarCastRib,
+};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
     env_logger::init();
 
     trace!("Starting multi-threaded yolo testing....");
-    let tree_bitmap = Arc::new(MultiThreadedStore::<
-        PrefixAs,
-        MemoryOnlyConfig,
-    >::try_default()?);
+    let tree_bitmap =
+        Arc::new(StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?);
 
-    let pfx = Prefix::new_relaxed(
+    let pfx = inetnum::addr::Prefix::new_relaxed(
         0b1111_1111_1111_1111_1111_1111_1111_1111_u32.into_ipaddr(),
         32,
     );
diff --git a/examples/numbers_treebitmap.rs b/examples/numbers_treebitmap.rs
index fee74030..8e667284 100644
--- a/examples/numbers_treebitmap.rs
+++ b/examples/numbers_treebitmap.rs
@@ -1,7 +1,8 @@
+use inetnum::addr::Prefix;
 use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::{
+    MemoryOnlyConfig, PrefixRecord, Record, RouteStatus, StarCastRib,
+};
 
 use std::env;
 use std::error::Error;
@@ -64,7 +65,7 @@ fn main() -> Result<(), Box<dyn Error>> {
     for _strides in strides_vec.iter() {
         let mut pfxs: Vec<PrefixRecord<PrefixAs>> = vec![];
         let tree_bitmap =
-            MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
+            StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?;
 
         if let Err(err) = load_prefixes(&mut pfxs) {
             println!("error running example: {}", err);
diff --git a/examples/real_single_thread_24.rs b/examples/real_single_thread_24.rs
index 996ec206..718540ba 100644
--- a/examples/real_single_thread_24.rs
+++ b/examples/real_single_thread_24.rs
@@ -1,12 +1,11 @@
+use inetnum::addr::Prefix;
 use log::trace;
-use multi::{MemoryOnlyConfig, Record, RouteStatus};
+use rotonda_store::{IntoIpAddr, MemoryOnlyConfig, Record, RouteStatus};
 use std::thread;
 use std::time::Duration;
 
 use rand::Rng;
 
-use rotonda_store::prelude::*;
-
 use rotonda_store::meta_examples::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -14,7 +13,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     trace!("Starting one-threaded yolo testing....");
-    let tree_bitmap = rotonda_store::MultiThreadedStore::<
+    let tree_bitmap = rotonda_store::StarCastRib::<
         PrefixAs,
         MemoryOnlyConfig,
     >::try_default()?;
diff --git a/examples/single_thread_24.rs b/examples/single_thread_24.rs
index abde1c1f..32b70e38 100644
--- a/examples/single_thread_24.rs
+++ b/examples/single_thread_24.rs
@@ -1,13 +1,13 @@
+use inetnum::addr::Prefix;
 use log::trace;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::{
+    IntoIpAddr, MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
+};
 use std::thread;
 use std::time::Duration;
 
 use rand::Rng;
 
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
-
 use rotonda_store::meta_examples::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -16,7 +16,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     trace!("Starting multi-threaded yolo testing....");
     let tree_bitmap =
-        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
+        StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     // let f = Arc::new(std::sync::atomic::AtomicBool::new(false));
 
     let mut pfx_int = 0_u32;
diff --git a/examples/treebitmap.rs b/examples/treebitmap.rs
index 741e7067..80103f8a 100644
--- a/examples/treebitmap.rs
+++ b/examples/treebitmap.rs
@@ -1,14 +1,13 @@
 use inetnum::addr::Prefix;
-use rotonda_store::meta_examples::NoMeta;
-use rotonda_store::prelude::multi::*;
-use rotonda_store::prelude::*;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::{
+    epoch, meta_examples::NoMeta, IncludeHistory, IntoIpAddr, MatchOptions,
+    MatchType, MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
+};
 
 type Prefix4<'a> = Prefix;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let tree_bitmap =
-        MultiThreadedStore::<_, MemoryOnlyConfig>::try_default()?;
+    let tree_bitmap = StarCastRib::<_, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
diff --git a/src/bin/cli.rs b/src/bin/cli.rs
index b78b3041..0b35eac0 100644
--- a/src/bin/cli.rs
+++ b/src/bin/cli.rs
@@ -1,12 +1,14 @@
 #![cfg(feature = "cli")]
 use ansi_term::Colour;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::{
+    epoch, IncludeHistory, MatchOptions, MatchType, MemoryOnlyConfig,
+    PrefixRecord, Record, RouteStatus, StarCastRib,
+};
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 
 use inetnum::addr::Prefix;
 use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::prelude::{multi::*, *};
 use rustyline::history::DefaultHistory;
 
 use std::env;
@@ -68,7 +70,7 @@ fn load_prefixes(
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut pfxs: Vec<PrefixRecord<PrefixAs>> = vec![];
     let tree_bitmap =
-        MultiThreadedStore::<PrefixAs, MemoryOnlyConfig>::try_default()?;
+        StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?;
 
     if let Err(err) = load_prefixes(&mut pfxs) {
         println!("error running example: {}", err);
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index 8c544317..1d6aa538 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -1,7 +1,13 @@
-use rotonda_store::rib::MemoryOnlyConfig;
-use rotonda_store::rib::PersistHistoryConfig;
-use rotonda_store::rib::PersistOnlyConfig;
-use rotonda_store::rib::WriteAheadConfig;
+use rotonda_store::rib::starcast_af::Config;
+use rotonda_store::rib::starcast_af::PersistStrategy;
+use rotonda_store::rib::starcast_af::UpsertReport;
+use rotonda_store::MemoryOnlyConfig;
+use rotonda_store::PersistHistoryConfig;
+use rotonda_store::PersistOnlyConfig;
+use rotonda_store::Record;
+use rotonda_store::RouteStatus;
+use rotonda_store::StarCastRib;
+use rotonda_store::WriteAheadConfig;
 use routecore::bgp::aspath::HopPath;
 use routecore::bgp::message::update_builder::StandardCommunitiesList;
 use std::collections::BTreeSet;
@@ -16,12 +22,6 @@ use memmap2::Mmap;
 use rayon::iter::ParallelBridge;
 use rayon::iter::ParallelIterator;
 use rayon::prelude::*;
-use rotonda_store::prelude::multi::PrefixStoreError;
-use rotonda_store::prelude::multi::{MultiThreadedStore, RouteStatus};
-use rotonda_store::rib::Config;
-use rotonda_store::rib::PersistStrategy;
-use rotonda_store::rib::UpsertReport;
-use rotonda_store::PublicRecord;
 use routecore::bgp::message::PduParseInfo;
 use routecore::bgp::path_attributes::OwnedPathAttributes;
 use routecore::mrt::MrtFile;
@@ -174,15 +174,17 @@ struct Cli {
     persist_strategy: Option<String>,
 }
 
+type Type = rotonda_store::errors::PrefixStoreError;
+
 fn insert<C: Config, T: rotonda_store::Meta>(
-    store: &MultiThreadedStore<T, C>,
+    store: &StarCastRib<T, C>,
     prefix: &Prefix,
     mui: u32,
     ltime: u64,
     route_status: RouteStatus,
     value: T,
-) -> Result<UpsertReport, PrefixStoreError> {
-    let record = PublicRecord::new(mui, ltime, route_status, value);
+) -> Result<UpsertReport, Type> {
+    let record = Record::new(mui, ltime, route_status, value);
     store
         .insert(prefix, record, None)
         .inspect_err(|e| eprintln!("Error in test_store: {e}"))
@@ -222,7 +224,7 @@ fn par_load_prefixes(
 
 fn mt_parse_and_insert_table<C: Config + Sync>(
     tables: TableDumpIterator<&[u8]>,
-    store: Option<&MultiThreadedStore<PaBytes, C>>,
+    store: Option<&StarCastRib<PaBytes, C>>,
     ltime: u64,
 ) -> (UpsertCounters, Vec<Prefix>) {
     let persist_strategy =
@@ -334,7 +336,7 @@ fn mt_parse_and_insert_table<C: Config + Sync>(
 
 fn st_parse_and_insert_table<C: Config>(
     entries: RibEntryIterator<&[u8]>,
-    store: Option<&MultiThreadedStore<PaBytes, C>>,
+    store: Option<&StarCastRib<PaBytes, C>>,
     ltime: u64,
 ) -> UpsertCounters {
     let mut counters = UpsertCounters::default();
@@ -365,7 +367,7 @@ fn st_parse_and_insert_table<C: Config>(
 
 fn mt_prime_store<C: Config + Sync>(
     prefixes: &Vec<(Prefix, u16)>,
-    store: &MultiThreadedStore<PaBytes, C>,
+    store: &StarCastRib<PaBytes, C>,
 ) -> UpsertCounters {
     let t0 = std::time::Instant::now();
 
@@ -398,7 +400,7 @@ fn mt_prime_store<C: Config + Sync>(
 
 fn st_prime_store<C: Config>(
     prefixes: &Vec<(Prefix, u16)>,
-    store: &MultiThreadedStore<PaBytes, C>,
+    store: &StarCastRib<PaBytes, C>,
 ) -> UpsertCounters {
     let mut counters = UpsertCounters::default();
 
@@ -418,7 +420,7 @@ fn st_prime_store<C: Config>(
     counters
 }
 
-type Stores<C> = Vec<MultiThreadedStore<PaBytes, C>>;
+type Stores<C> = Vec<StarCastRib<PaBytes, C>>;
 
 // Create all the stores necessary, and if at least one is created, create
 // a reference to the first one.
@@ -426,7 +428,7 @@ fn create_stores<'a, C: Config + Sync>(
     stores: &'a mut Stores<C>,
     args: &'a Cli,
     store_config: C,
-) -> Option<&'a MultiThreadedStore<PaBytes, C>> {
+) -> Option<&'a StarCastRib<PaBytes, C>> {
     match &args {
         a if a.single_store && a.parse_only => {
             eprintln!(
@@ -437,7 +439,7 @@ fn create_stores<'a, C: Config + Sync>(
         }
         a if a.single_store => {
             stores.push(
-                MultiThreadedStore::<PaBytes, C>::new_with_config(
+                StarCastRib::<PaBytes, C>::new_with_config(
                     store_config.clone(),
                 )
                 .unwrap(),
@@ -456,9 +458,8 @@ fn create_stores<'a, C: Config + Sync>(
         }
         _ => {
             for _ in &args.mrt_files {
-                stores.push(
-                    MultiThreadedStore::<PaBytes, C>::try_default().unwrap(),
-                );
+                stores
+                    .push(StarCastRib::<PaBytes, C>::try_default().unwrap());
             }
             println!("Number of created stores: {}", stores.len());
             println!("store config: {:?}", store_config);
@@ -469,7 +470,7 @@ fn create_stores<'a, C: Config + Sync>(
 }
 
 fn exec_for_store<'a, C: Config + Sync>(
-    mut store: Option<&'a MultiThreadedStore<PaBytes, C>>,
+    mut store: Option<&'a StarCastRib<PaBytes, C>>,
     inner_stores: &'a Stores<C>,
     args: &'a Cli,
 ) {
diff --git a/src/cht/mod.rs b/src/cht/mod.rs
new file mode 100644
index 00000000..4c1acf1e
--- /dev/null
+++ b/src/cht/mod.rs
@@ -0,0 +1,42 @@
+mod oncebox;
+
+pub(crate) use oncebox::OnceBox;
+pub(crate) use oncebox::OnceBoxSlice;
+
+use crate::rib::STRIDE_SIZE;
+
+pub(crate) trait Value {
+    fn init(size: usize) -> Self;
+}
+
+#[derive(Debug)]
+pub(crate) struct Cht<
+    V,
+    const ROOT_SIZE: usize,
+    const STRIDES_PER_BUCKET: usize,
+>([V; ROOT_SIZE]);
+
+impl<V: Value, const ROOT_SIZE: usize, const STRIDES_PER_BUCKET: usize>
+    Cht<V, ROOT_SIZE, STRIDES_PER_BUCKET>
+{
+    pub(crate) fn init() -> Self {
+        Self(std::array::from_fn::<_, ROOT_SIZE, _>(|_| {
+            V::init(STRIDE_SIZE as usize)
+        }))
+    }
+
+    pub(crate) fn root_for_len(&self, len: u8) -> &V {
+        &self.0[len as usize / STRIDES_PER_BUCKET]
+    }
+}
+
+pub(crate) fn bits_for_len(len: u8, lvl: u8) -> u8 {
+    let res = STRIDE_SIZE * (lvl + 1);
+    if res < len {
+        res
+    } else if res >= len + STRIDE_SIZE {
+        0
+    } else {
+        len
+    }
+}
diff --git a/src/stride_table/oncebox.rs b/src/cht/oncebox.rs
similarity index 99%
rename from src/stride_table/oncebox.rs
rename to src/cht/oncebox.rs
index 8234883e..891378dc 100644
--- a/src/stride_table/oncebox.rs
+++ b/src/cht/oncebox.rs
@@ -75,7 +75,7 @@ impl<T> OnceBoxSlice<T> {
         }
     }
 
-    pub fn is_null(&self) -> bool {
+    pub fn _is_null(&self) -> bool {
         self.ptr.load(Ordering::Relaxed).is_null()
     }
 
diff --git a/src/in_memory/atomic_bitmap.rs b/src/in_memory/atomic_bitmap.rs
index 98529109..1514f253 100644
--- a/src/in_memory/atomic_bitmap.rs
+++ b/src/in_memory/atomic_bitmap.rs
@@ -1,10 +1,10 @@
-use log::{log_enabled, trace};
 use parking_lot_core::SpinWait;
 use std::fmt::{Binary, Debug};
 use std::sync::atomic::{fence, AtomicU16, AtomicU32, Ordering};
 
-use crate::local_array;
-use crate::local_array::bit_span::BitSpan;
+use crate::types::bit_span::BitSpan;
+
+use super::tree_bitmap_node;
 
 pub struct AtomicPtrBitArr(pub AtomicU16);
 pub struct AtomicPfxBitArr(pub AtomicU32);
@@ -55,20 +55,7 @@ where
 impl AtomicPtrBitArr {
     pub(crate) fn ptr_range(&self, bs: BitSpan) -> (u16, u8) {
         let ptrbitarr = self.load();
-        let start: u8 = (bs.bits << (4 - bs.len)) as u8;
-        let stop: u8 = start + (1 << (4 - bs.len));
-        let mask: u16 = (((1_u32 << (stop as u32 - start as u32)) - 1)
-            .rotate_right(stop as u32)
-            >> 16)
-            .try_into()
-            .unwrap();
-        if log_enabled!(log::Level::Trace) {
-            trace!("- mask      {:032b}", mask);
-            trace!("- ptrbitarr {:032b}", ptrbitarr);
-            trace!("- shl bitar {:032b}", ptrbitarr & mask);
-        }
-
-        (ptrbitarr & mask, start)
+        tree_bitmap_node::ptr_range(ptrbitarr, bs)
     }
 
     pub(crate) fn as_stride_size(&self) -> u32 {
@@ -109,29 +96,10 @@ impl From<u16> for AtomicPtrBitArr {
 impl AtomicPfxBitArr {
     pub(crate) fn ms_pfx_mask(&self, bs: BitSpan) -> u32 {
         let pfxbitarr = self.load();
-        local_array::in_memory::node::ms_prefix_mask_arr(bs) & pfxbitarr
+        tree_bitmap_node::ms_prefix_mask_arr(bs) & pfxbitarr
     }
 }
 
-pub(crate) fn into_ptrbitarr(bitmap: u32) -> u16 {
-    (bitmap >> 1) as u16
-}
-
-pub(crate) fn into_pfxbitarr(bitmap: u16) -> u32 {
-    (bitmap as u32) << 1
-}
-
-pub(crate) fn bit_pos_from_index(i: u8) -> u32 {
-    1_u32.rotate_right(1) >> i
-}
-
-pub(crate) fn ptr_bit_pos_from_index(i: u8) -> u16 {
-    // trace!("pfx {} ptr {} strlen {}",
-    // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
-    trace!("PTR_BIT_POS_FROM_INDEX {i}");
-    1_u16.rotate_right(i as u32 + 2)
-}
-
 impl AtomicBitmap for AtomicPfxBitArr {
     type InnerType = u32;
 
@@ -160,138 +128,3 @@ impl From<u32> for AtomicPfxBitArr {
         Self(AtomicU32::new(value))
     }
 }
-
-// pub(crate) trait Stride {
-// Get the bit position of the start of the given nibble.
-// The nibble is defined as a `len` number of bits set from the right.
-// bit_pos always has only one bit set in the complete array.
-// e.g.:
-// len: 4
-// nibble: u16  =  0b0000 0000 0000 0111
-// bit_pos: u16 =  0b0000 0000 0000 1000
-
-// `<Self as Stride>::BITS`
-// is the whole length of the bitmap, since we are shifting to the left,
-// we have to start at the end of the bitmap.
-// `((1 << len) - 1)`
-// is the offset for this nibble length in the bitmap.
-// `nibble`
-// shifts to the right position withing the bit range for this nibble
-// length, this follows from the fact that the `nibble` value represents
-// *both* the bitmap part, we're considering here *and* the position
-// relative to the nibble length offset in the bitmap.
-// fn get_bit_pos(bit_span: BitSpan) -> u32;
-
-// fn bit_pos_from_index(i: u8) -> u32;
-
-// fn ptr_bit_pos_from_index(i: u8) -> u16;
-
-// fn cursor_from_bit_span(bs: BitSpan) -> u8;
-
-// fn ptr_range(ptrbitarr: u16, range: BitSpan) -> (u16, u8);
-
-// fn ms_pfx_mask(pfxbitarr: u32, range: BitSpan) -> u32;
-
-// Clear the bitmap to the right of the pointer and count the number of
-// ones. This number represents the index to the corresponding prefix in
-// the pfx_vec.
-
-// Clearing is performed by shifting to the right until we have the
-// nibble all the way at the right.
-
-// `(<Self as Stride>::BITS >> 1)`
-// The end of the bitmap (this bitmap is half the size of the pfx bitmap)
-
-// `nibble`
-// The bit position relative to the offset for the nibble length, this
-// index is only used at the last (relevant) stride, so the offset is
-// always 0.
-
-// get_pfx_index only needs nibble and len for fixed-layout bitarrays,
-// since the index can be deducted from them.
-
-// Clear the bitmap to the right of the pointer and count the number of
-// ones. This number represents the index to the corresponding child node
-// in the ptr_vec.
-
-// Clearing is performed by shifting to the right until we have the
-// nibble all the way at the right.
-
-// For ptrbitarr the only index we want is the one for a full-length
-// nibble (stride length) at the last stride, so we don't need the length
-//  of the nibble.
-
-// `(<Self as Stride>::BITS >> 1)`
-// The end of the bitmap (this bitmap is half the size of the pfx bitmap)
-// AF::BITS is the size of the pfx bitmap.
-
-// `nibble`
-// The bit position relative to the offset for the nibble length, this
-// index is only used at the last (relevant) stride, so the offset is
-// always 0.
-
-// Convert a ptrbitarr into a pfxbitarr sized bitmap,
-// so we can do bitwise operations with a pfxbitarr sized
-// bitmap on them.
-// Since the last bit in the pfxbitarr isn't used, but the
-// full ptrbitarr *is* used, the prtbitarr should be shifted
-// one bit to the left.
-// fn into_stride_size(bitmap: u16) -> u32;
-
-// Convert a pfxbitarr sized bitmap into a ptrbitarr sized
-// Note that bitwise operators align bits of unsigned types with
-// different sizes to the right, so we don't have to do anything to pad
-// the smaller sized type. We do have to shift one bit to the left, to
-// accommodate the unused pfxbitarr's last bit.
-// fn into_ptrbitarr_size(bitmap: u32) -> u16;
-// }
-
-// impl Stride for AtomicPfxBitArr {
-// fn get_bit_pos(bs: BitSpan) -> u32 {
-//     // trace!("nibble {}, len {}, BITS {}", nibble, len, <Self as Stride>::BITS);
-//     1 << (STRIDE_BITS - ((1 << bs.len) - 1) as u8 - bs.bits as u8 - 1)
-// }
-
-// fn bit_pos_from_index(i: u8) -> u32 {
-//     <u32>::try_from(1).unwrap().rotate_right(1) >> i
-// }
-
-// fn ptr_bit_pos_from_index(i: u8) -> u16 {
-//     // trace!("pfx {} ptr {} strlen {}",
-//     // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
-//     <u16>::try_from(1).unwrap().rotate_right(1) >> (i + 1)
-// }
-
-// fn cursor_from_bit_span(bs: BitSpan) -> u8 {
-//     Self::get_bit_pos(bs).leading_zeros() as u8
-// }
-
-// fn ptr_range(ptrbitarr: u16, bs: BitSpan) -> (u16, u8) {
-//     let start: u8 = (bs.bits << (4 - bs.len)) as u8;
-//     let stop: u8 = start + (1 << (4 - bs.len));
-//     let mask: u16 = (((1_u32 << (stop as u32 - start as u32)) - 1)
-//         .rotate_right(stop as u32)
-//         >> 16)
-//         .try_into()
-//         .unwrap();
-//     trace!("- mask      {:032b}", mask);
-//     trace!("- ptrbitarr {:032b}", ptrbitarr);
-//     trace!("- shl bitar {:032b}", ptrbitarr & mask);
-
-//     // if ptrbitarr & mask == <$ptrsize>::zero() { panic!("stop"); }
-
-//     (ptrbitarr & mask, start)
-// }
-
-// fn ms_pfx_mask(pfxbitarr: u32, bs: BitSpan) -> u32 {
-//     local_array::in_memory::node::ms_prefix_mask_arr(bs) & pfxbitarr
-// }
-
-// fn into_stride_size(bitmap: u16) -> u32 {
-//     (bitmap as u32) << 1
-// }
-
-// fn into_ptrbitarr_size(bitmap: u32) -> u16 {
-//     (bitmap >> 1) as u16
-// }
-// }
diff --git a/src/in_memory/mod.rs b/src/in_memory/mod.rs
index 42284fb8..4a031dd6 100644
--- a/src/in_memory/mod.rs
+++ b/src/in_memory/mod.rs
@@ -1,15 +1,9 @@
-pub(crate) mod atomic_stride;
-pub(crate) mod atomic_types;
-pub(crate) mod node;
-mod oncebox;
-pub(crate) mod tree;
+pub(crate) mod atomic_bitmap;
+pub(crate) mod node_cht;
 
-// #[deprecated]
-// mod deprecated_query;
+pub(crate) mod tree_bitmap;
+pub(crate) mod tree_bitmap_iterators;
+pub(crate) mod tree_bitmap_node;
+mod tree_bitmap_query;
 
-pub mod iterators;
-
-mod query;
-
-// #[macro_use]
-// mod macros;
+pub(crate) use tree_bitmap::TreeBitMap;
diff --git a/src/in_memory/node_cht.rs b/src/in_memory/node_cht.rs
index 97de4a23..edc01be9 100644
--- a/src/in_memory/node_cht.rs
+++ b/src/in_memory/node_cht.rs
@@ -1,29 +1,17 @@
-use std::collections::HashMap;
-use std::sync::{Arc, Mutex, MutexGuard, RwLock};
-use std::{
-    fmt::{Debug, Display},
-    sync::atomic::Ordering,
-};
+use std::sync::RwLock;
 
-use crossbeam_epoch::{self as epoch, Atomic};
+use log::{debug, log_enabled};
 
-use crossbeam_utils::Backoff;
-use log::{debug, log_enabled, trace};
-
-use epoch::{Guard, Owned};
 use roaring::RoaringBitmap;
 
-use crate::local_array::rib::default_store::STRIDE_SIZE;
-use crate::local_array::types::{PrefixId, RouteStatus};
-use crate::prefix_record::PublicRecord;
-use crate::prelude::Meta;
-use crate::AddressFamily;
+use crate::cht::{Cht, OnceBoxSlice, Value};
+use crate::types::AddressFamily;
 
-use super::super::errors::PrefixStoreError;
-use super::node::{StrideNodeId, TreeBitMapNode};
-use super::oncebox::OnceBoxSlice;
+use super::tree_bitmap_node::{StrideNodeId, TreeBitMapNode};
+use crate::types::errors::PrefixStoreError;
 
-// ----------- Node related structs -----------------------------------------
+pub(crate) type NodeCht<AF, const ROOT_SIZE: usize> =
+    Cht<NodeSet<AF>, ROOT_SIZE, 4>;
 
 #[derive(Debug)]
 pub(crate) struct StoredNode<AF>
@@ -66,9 +54,9 @@ impl<AF: AddressFamily> NodeSet<AF> {
     pub(crate) fn update_rbm_index(
         &self,
         multi_uniq_id: u32,
-    ) -> Result<(u32, bool), crate::prelude::multi::PrefixStoreError>
+    ) -> Result<(u32, bool), PrefixStoreError>
     where
-        AF: crate::AddressFamily,
+        AF: crate::types::AddressFamily,
     {
         let try_count = 0;
         let mut rbm = self.1.write().unwrap();
@@ -81,9 +69,9 @@ impl<AF: AddressFamily> NodeSet<AF> {
         &self,
         multi_uniq_id: u32,
         _guard: &crate::epoch::Guard,
-    ) -> Result<u32, crate::prelude::multi::PrefixStoreError>
+    ) -> Result<u32, PrefixStoreError>
     where
-        AF: crate::AddressFamily,
+        AF: crate::types::AddressFamily,
     {
         let try_count = 0;
 
@@ -98,511 +86,6 @@ impl<AF: AddressFamily> NodeSet<AF> {
     }
 }
 
-// ----------- Prefix related structs ---------------------------------------
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub struct PathSelections {
-    pub(crate) path_selection_muis: (Option<u32>, Option<u32>),
-}
-
-impl PathSelections {
-    pub fn best(&self) -> Option<u32> {
-        self.path_selection_muis.0
-    }
-
-    pub fn backup(&self) -> Option<u32> {
-        self.path_selection_muis.1
-    }
-}
-
-// ----------- StoredPrefix -------------------------------------------------
-// This is the top-level struct that's linked from the slots in the buckets.
-// It contains a super_agg_record that is supposed to hold counters for the
-// records that are stored inside it, so that iterators over its linked lists
-// don't have to go into them if there's nothing there and could stop early.
-#[derive(Debug)]
-pub struct StoredPrefix<AF: AddressFamily, M: Meta> {
-    // the serial number
-    // pub serial: usize,
-    // the prefix itself,
-    pub prefix: PrefixId<AF>,
-    // the aggregated data for this prefix
-    pub record_map: MultiMap<M>,
-    // (mui of best path entry, mui of backup path entry) from the record_map
-    path_selections: Atomic<PathSelections>,
-    // the reference to the next set of records for this prefix, if any.
-    pub next_bucket: PrefixSet<AF, M>,
-}
-
-impl<AF: AddressFamily, M: crate::prefix_record::Meta> StoredPrefix<AF, M> {
-    pub(crate) fn new(pfx_id: PrefixId<AF>, level: u8) -> Self {
-        // start calculation size of next set, it's dependent on the level
-        // we're in.
-        // let pfx_id = PrefixId::new(record.net, record.len);
-        let this_level = bits_for_len(pfx_id.get_len(), level);
-        let next_level = bits_for_len(pfx_id.get_len(), level + 1);
-
-        trace!("this level {} next level {}", this_level, next_level);
-        let next_bucket: PrefixSet<AF, M> = if next_level > 0 {
-            debug!(
-                "{} store: INSERT with new bucket of size {} at prefix len {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                1 << (next_level - this_level),
-                pfx_id.get_len()
-            );
-            PrefixSet::init(next_level.saturating_sub(this_level))
-        } else {
-            debug!(
-                "{} store: INSERT at LAST LEVEL with empty bucket at prefix len {}",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                pfx_id.get_len()
-            );
-            PrefixSet::init(next_level.saturating_sub(this_level))
-        };
-        // End of calculation
-
-        let rec_map = HashMap::new();
-
-        StoredPrefix {
-            // serial: 1,
-            prefix: pfx_id,
-            path_selections: Atomic::init(PathSelections {
-                path_selection_muis: (None, None),
-            }),
-            record_map: MultiMap::new(rec_map),
-            next_bucket,
-        }
-    }
-
-    pub(crate) fn get_prefix_id(&self) -> PrefixId<AF> {
-        self.prefix
-    }
-
-    pub fn get_path_selections(&self, guard: &Guard) -> PathSelections {
-        let path_selections =
-            self.path_selections.load(Ordering::Acquire, guard);
-
-        unsafe { path_selections.as_ref() }.map_or(
-            PathSelections {
-                path_selection_muis: (None, None),
-            },
-            |ps| *ps,
-        )
-    }
-
-    pub(crate) fn set_path_selections(
-        &self,
-        path_selections: PathSelections,
-        guard: &Guard,
-    ) -> Result<(), PrefixStoreError> {
-        let current = self.path_selections.load(Ordering::SeqCst, guard);
-
-        if unsafe { current.as_ref() } == Some(&path_selections) {
-            debug!("unchanged path_selections");
-            return Ok(());
-        }
-
-        self.path_selections
-            .compare_exchange(
-                current,
-                // Set the tag to indicate we're updated
-                Owned::new(path_selections).with_tag(0),
-                Ordering::AcqRel,
-                Ordering::Acquire,
-                guard,
-            )
-            .map_err(|_| PrefixStoreError::PathSelectionOutdated)?;
-        Ok(())
-    }
-
-    pub fn set_ps_outdated(
-        &self,
-        guard: &Guard,
-    ) -> Result<(), PrefixStoreError> {
-        self.path_selections
-            .fetch_update(Ordering::Acquire, Ordering::Acquire, guard, |p| {
-                Some(p.with_tag(1))
-            })
-            .map(|_| ())
-            .map_err(|_| PrefixStoreError::StoreNotReadyError)
-    }
-
-    pub fn is_ps_outdated(&self, guard: &Guard) -> bool {
-        self.path_selections.load(Ordering::Acquire, guard).tag() == 1
-    }
-
-    pub fn calculate_and_store_best_backup<'a>(
-        &'a self,
-        tbi: &M::TBI,
-        guard: &'a Guard,
-    ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
-        let path_selection_muis = self.record_map.best_backup(*tbi);
-
-        self.set_path_selections(
-            PathSelections {
-                path_selection_muis,
-            },
-            guard,
-        )?;
-
-        Ok(path_selection_muis)
-    }
-}
-
-#[derive(Clone, Debug)]
-pub(crate) struct MultiMapValue<M> {
-    meta: M,
-    ltime: u64,
-    route_status: RouteStatus,
-}
-
-impl<M: Meta> MultiMapValue<M> {
-    pub(crate) fn logical_time(&self) -> u64 {
-        self.ltime
-    }
-
-    pub(crate) fn set_logical_time(&mut self, ltime: u64) {
-        self.ltime = ltime;
-    }
-
-    pub(crate) fn meta(&self) -> &M {
-        &self.meta
-    }
-
-    pub(crate) fn route_status(&self) -> RouteStatus {
-        self.route_status
-    }
-
-    pub(crate) fn set_route_status(&mut self, status: RouteStatus) {
-        self.route_status = status;
-    }
-}
-
-impl<M: Meta> std::fmt::Display for MultiMapValue<M> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{} {}",
-            // self.meta(),
-            self.logical_time(),
-            self.route_status()
-        )
-    }
-}
-
-impl<M: Meta> From<PublicRecord<M>> for MultiMapValue<M> {
-    fn from(value: PublicRecord<M>) -> Self {
-        Self {
-            ltime: value.ltime,
-            route_status: value.status,
-            meta: value.meta,
-        }
-    }
-}
-
-impl<M: Meta> From<(u32, &MultiMapValue<M>)> for PublicRecord<M> {
-    fn from(value: (u32, &MultiMapValue<M>)) -> Self {
-        Self {
-            multi_uniq_id: value.0,
-            meta: value.1.meta().clone(),
-            ltime: value.1.ltime,
-            status: value.1.route_status,
-        }
-    }
-}
-
-// ----------- MultiMap ------------------------------------------------------
-// This is the record that holds the aggregates at the top-level for a given
-// prefix.
-
-#[derive(Debug)]
-pub struct MultiMap<M: Meta>(
-    Arc<Mutex<std::collections::HashMap<u32, MultiMapValue<M>>>>,
-);
-
-impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
-    pub(crate) fn new(record_map: HashMap<u32, MultiMapValue<M>>) -> Self {
-        Self(Arc::new(Mutex::new(record_map)))
-    }
-
-    fn guard_with_retry(
-        &self,
-        mut retry_count: usize,
-    ) -> (MutexGuard<HashMap<u32, MultiMapValue<M>>>, usize) {
-        let backoff = Backoff::new();
-
-        loop {
-            if let Ok(guard) = self.0.try_lock() {
-                return (guard, retry_count);
-            }
-
-            backoff.spin();
-            retry_count += 1;
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        record_map.len()
-    }
-
-    pub fn get_record_for_mui(
-        &self,
-        mui: u32,
-        include_withdrawn: bool,
-    ) -> Option<PublicRecord<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-
-        record_map
-            .get(&mui)
-            .and_then(|r| -> Option<PublicRecord<M>> {
-                if include_withdrawn
-                    || r.route_status() == RouteStatus::Active
-                {
-                    Some(PublicRecord::from((mui, r)))
-                } else {
-                    None
-                }
-            })
-    }
-
-    pub fn best_backup(&self, tbi: M::TBI) -> (Option<u32>, Option<u32>) {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        let ord_routes = record_map
-            .iter()
-            .map(|r| (r.1.meta().as_orderable(tbi), *r.0));
-        let (best, bckup) =
-            routecore::bgp::path_selection::best_backup_generic(ord_routes);
-        (best.map(|b| b.1), bckup.map(|b| b.1))
-    }
-
-    pub(crate) fn get_record_for_mui_with_rewritten_status(
-        &self,
-        mui: u32,
-        bmin: &RoaringBitmap,
-        rewrite_status: RouteStatus,
-    ) -> Option<PublicRecord<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        record_map.get(&mui).map(|r| {
-            // We'll return a cloned record: the record in the store remains
-            // untouched.
-            let mut r = r.clone();
-            if bmin.contains(mui) {
-                r.set_route_status(rewrite_status);
-            }
-            PublicRecord::from((mui, &r))
-        })
-    }
-
-    pub fn get_filtered_record_for_mui(
-        &self,
-        mui: u32,
-        include_withdrawn: bool,
-        bmin: &RoaringBitmap,
-    ) -> Option<PublicRecord<M>> {
-        match include_withdrawn {
-            false => self.get_record_for_mui(mui, include_withdrawn),
-            true => self.get_record_for_mui_with_rewritten_status(
-                mui,
-                bmin,
-                RouteStatus::Withdrawn,
-            ),
-        }
-    }
-
-    // Helper to filter out records that are not-active (Inactive or
-    // Withdrawn), or whose mui appears in the global withdrawn index.
-    pub fn get_filtered_records(
-        &self,
-        mui: Option<u32>,
-        include_withdrawn: bool,
-        bmin: &RoaringBitmap,
-    ) -> Option<Vec<PublicRecord<M>>> {
-        if let Some(mui) = mui {
-            self.get_filtered_record_for_mui(mui, include_withdrawn, bmin)
-                .map(|r| vec![r])
-        } else {
-            match include_withdrawn {
-                false => {
-                    let recs = self.as_active_records_not_in_bmin(bmin);
-                    if recs.is_empty() {
-                        None
-                    } else {
-                        Some(recs)
-                    }
-                }
-                true => {
-                    let recs = self.as_records_with_rewritten_status(
-                        bmin,
-                        RouteStatus::Withdrawn,
-                    );
-                    if recs.is_empty() {
-                        None
-                    } else {
-                        Some(recs)
-                    }
-                }
-            }
-        }
-    }
-
-    // return all records regardless of their local status, or any globally
-    // set status for the mui of the record. However, the local status for a
-    // record whose mui appears in the specified bitmap index, will be
-    // rewritten with the specified RouteStatus.
-    pub fn as_records_with_rewritten_status(
-        &self,
-        bmin: &RoaringBitmap,
-        rewrite_status: RouteStatus,
-    ) -> Vec<PublicRecord<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        record_map
-            .iter()
-            .map(move |r| {
-                let mut rec = r.1.clone();
-                if bmin.contains(*r.0) {
-                    rec.set_route_status(rewrite_status);
-                }
-                PublicRecord::from((*r.0, &rec))
-            })
-            .collect::<Vec<_>>()
-    }
-
-    pub fn as_records(&self) -> Vec<PublicRecord<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        record_map
-            .iter()
-            .map(|r| PublicRecord::from((*r.0, r.1)))
-            .collect::<Vec<_>>()
-    }
-
-    // Returns a vec of records whose keys are not in the supplied bitmap
-    // index, and whose local Status is set to Active. Used to filter out
-    // withdrawn routes.
-    pub fn as_active_records_not_in_bmin(
-        &self,
-        bmin: &RoaringBitmap,
-    ) -> Vec<PublicRecord<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
-        record_map
-            .iter()
-            .filter_map(|r| {
-                if r.1.route_status() == RouteStatus::Active
-                    && !bmin.contains(*r.0)
-                {
-                    Some(PublicRecord::from((*r.0, r.1)))
-                } else {
-                    None
-                }
-            })
-            .collect::<Vec<_>>()
-    }
-
-    // Change the local status of the record for this mui to Withdrawn.
-    pub fn mark_as_withdrawn_for_mui(&self, mui: u32, ltime: u64) {
-        let c_map = Arc::clone(&self.0);
-        let mut record_map = c_map.lock().unwrap();
-        if let Some(rec) = record_map.get_mut(&mui) {
-            rec.set_route_status(RouteStatus::Withdrawn);
-            rec.set_logical_time(ltime);
-        }
-    }
-
-    // Change the local status of the record for this mui to Active.
-    pub fn mark_as_active_for_mui(&self, mui: u32, ltime: u64) {
-        let record_map = Arc::clone(&self.0);
-        let mut r_map = record_map.lock().unwrap();
-        if let Some(rec) = r_map.get_mut(&mui) {
-            rec.set_route_status(RouteStatus::Active);
-            rec.set_logical_time(ltime);
-        }
-    }
-
-    // Insert or replace the PublicRecord in the HashMap for the key of
-    // record.multi_uniq_id. Returns the number of entries in the HashMap
-    // after updating it, if it's more than 1. Returns None if this is the
-    // first entry.
-    #[allow(clippy::type_complexity)]
-    pub(crate) fn upsert_record(
-        &self,
-        new_rec: PublicRecord<M>,
-    ) -> Result<(Option<(MultiMapValue<M>, usize)>, usize), PrefixStoreError>
-    {
-        let (mut record_map, retry_count) = self.guard_with_retry(0);
-        let key = new_rec.multi_uniq_id;
-
-        match record_map.contains_key(&key) {
-            true => {
-                let old_rec = record_map
-                    .insert(key, MultiMapValue::from(new_rec))
-                    .map(|r| (r, record_map.len()));
-                Ok((old_rec, retry_count))
-            }
-            false => {
-                let new_rec = MultiMapValue::from(new_rec);
-                let old_rec = record_map.insert(key, new_rec);
-                assert!(old_rec.is_none());
-                Ok((None, retry_count))
-            }
-        }
-    }
-}
-
-impl<M: Meta> Clone for MultiMap<M> {
-    fn clone(&self) -> Self {
-        Self(Arc::clone(&self.0))
-    }
-}
-
-// ----------- FamilyBuckets Trait ------------------------------------------
-
-// pub trait FamilyCHT<AF: AddressFamily, V> {
-//     fn init() -> Self;
-//     fn bits_for_len(len: u8, level: u8) -> u8;
-//     fn root_for_len(&self, len: u8) -> &V;
-// }
-
-//------------ PrefixSet ----------------------------------------------------
-
-// The PrefixSet is the ARRAY that holds all the child prefixes in a node.
-// Since we are storing these prefixes in the global store in a HashMap that
-// is keyed on the tuple (addr_bits, len, serial number) we can get away with
-// storing ONLY THE SERIAL NUMBER in the pfx_vec: The addr_bits and len are
-// implied in the position in the array a serial number has. A PrefixSet
-// doesn't know anything about the node it is contained in, so it needs a base
-// address to be able to calculate the complete prefix of a child prefix.
-
-#[derive(Debug)]
-#[repr(align(8))]
-pub struct PrefixSet<AF: AddressFamily, M: Meta>(
-    pub OnceBoxSlice<StoredPrefix<AF, M>>,
-);
-
-impl<AF: AddressFamily, M: Meta> PrefixSet<AF, M> {
-    pub fn init(p2_size: u8) -> Self {
-        PrefixSet(OnceBoxSlice::new(p2_size))
-    }
-}
-
-//-----
-//
-pub(crate) trait Value {
-    fn init(size: usize) -> Self;
-}
-
-impl<AF: AddressFamily, M: Meta> Value for PrefixSet<AF, M> {
-    fn init(p2_size: usize) -> Self {
-        PrefixSet(OnceBoxSlice::new(p2_size as u8))
-    }
-}
-
 impl<AF: AddressFamily> Value for NodeSet<AF> {
     fn init(p2_size: usize) -> Self {
         if log_enabled!(log::Level::Debug) {
@@ -619,38 +102,3 @@ impl<AF: AddressFamily> Value for NodeSet<AF> {
         )
     }
 }
-
-#[derive(Debug)]
-pub(crate) struct Cht<
-    V,
-    const ROOT_SIZE: usize,
-    const STRIDES_PER_BUCKET: usize,
->([V; ROOT_SIZE]);
-
-impl<V: Value, const ROOT_SIZE: usize, const STRIDES_PER_BUCKET: usize>
-    Cht<V, ROOT_SIZE, STRIDES_PER_BUCKET>
-{
-    pub(crate) fn init() -> Self {
-        Self(std::array::from_fn::<_, ROOT_SIZE, _>(|_| {
-            V::init(STRIDE_SIZE as usize)
-        }))
-    }
-
-    pub(crate) fn root_for_len(&self, len: u8) -> &V {
-        &self.0[len as usize / STRIDES_PER_BUCKET]
-    }
-}
-
-pub(crate) fn bits_for_len(len: u8, lvl: u8) -> u8 {
-    let res = STRIDE_SIZE * (lvl + 1);
-    if res < len {
-        res
-    } else if res >= len + STRIDE_SIZE {
-        0
-    } else {
-        len
-    }
-}
-
-pub(crate) type NodeCht<AF, const ROOT_SIZE: usize> =
-    Cht<NodeSet<AF>, ROOT_SIZE, 4>;
diff --git a/src/in_memory/tree_bitmap.rs b/src/in_memory/tree_bitmap.rs
index f0d194b2..0475dc37 100644
--- a/src/in_memory/tree_bitmap.rs
+++ b/src/in_memory/tree_bitmap.rs
@@ -183,12 +183,10 @@
 // an actual memory leak in the mt-prefix-store (and even more if they can
 // produce a fix for it).
 
-use crate::local_array::bit_span::BitSpan;
-use crate::local_array::in_memory::atomic_types::{
-    bits_for_len, NodeSet, StoredNode,
-};
-use crate::local_array::rib::default_store::STRIDE_SIZE;
-use crate::prelude::multi::PrefixId;
+use crate::cht::{bits_for_len, Cht};
+use crate::in_memory::node_cht::{NodeCht, NodeSet, StoredNode};
+use crate::rib::STRIDE_SIZE;
+use crate::types::{BitSpan, PrefixId};
 use crossbeam_epoch::{Atomic, Guard};
 use log::{debug, error, log_enabled, trace};
 use roaring::RoaringBitmap;
@@ -196,15 +194,14 @@ use roaring::RoaringBitmap;
 use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering};
 use std::{fmt::Debug, marker::PhantomData};
 
-use super::atomic_types::{Cht, NodeCht};
-use crate::af::AddressFamily;
+use super::atomic_bitmap::{AtomicBitmap, AtomicPfxBitArr, AtomicPtrBitArr};
 use crate::rib::Counters;
+use crate::types::AddressFamily;
 
-use super::super::errors::PrefixStoreError;
-pub(crate) use super::atomic_stride::*;
-use crate::local_array::in_memory::node::{NewNodeOrIndex, StrideNodeId};
+use crate::in_memory::tree_bitmap_node::{NewNodeOrIndex, StrideNodeId};
+use crate::types::errors::PrefixStoreError;
 
-use super::node::TreeBitMapNode;
+use super::tree_bitmap_node::TreeBitMapNode;
 
 #[cfg(feature = "cli")]
 use ansi_term::Colour;
@@ -213,8 +210,8 @@ use ansi_term::Colour;
 
 #[derive(Debug)]
 pub struct TreeBitMap<AF: AddressFamily, const ROOT_SIZE: usize> {
-    pub(crate) node_buckets: NodeCht<AF, ROOT_SIZE>,
-    pub(in crate::local_array) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
+    pub(crate) node_cht: NodeCht<AF, ROOT_SIZE>,
+    pub(crate) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     counters: Counters,
     default_route_exists: AtomicBool,
 }
@@ -222,7 +219,7 @@ pub struct TreeBitMap<AF: AddressFamily, const ROOT_SIZE: usize> {
 impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
     pub(crate) fn new() -> Result<Self, Box<dyn std::error::Error>> {
         let tree_bitmap = Self {
-            node_buckets: Cht::init(),
+            node_cht: Cht::init(),
             withdrawn_muis_bmin: RoaringBitmap::new().into(),
             counters: Counters::default(),
             default_route_exists: AtomicBool::new(false),
@@ -465,7 +462,7 @@ Giving up this node. This shouldn't happen!",
         if let Some(_root_node) =
             self.retrieve_node_mut(self.get_root_node_id(), mui)
         {
-            self.node_buckets
+            self.node_cht
                 .root_for_len(self.get_root_node_id().len())
                 .update_rbm_index(mui)
         } else {
@@ -501,7 +498,7 @@ Giving up this node. This shouldn't happen!",
         }
         self.counters.inc_nodes_count();
 
-        let mut nodes = self.node_buckets.root_for_len(id.len());
+        let mut nodes = self.node_cht.root_for_len(id.len());
         let new_node = next_node;
         let mut level = 0;
         let mut retry_count = 0;
@@ -665,7 +662,7 @@ lvl{}",
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
-        let mut nodes = self.node_buckets.root_for_len(id.len());
+        let mut nodes = self.node_cht.root_for_len(id.len());
 
         loop {
             let index = Self::hash_node_id(id, level);
@@ -745,7 +742,7 @@ lvl{}",
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
-        let mut nodes = self.node_buckets.root_for_len(id.len());
+        let mut nodes = self.node_cht.root_for_len(id.len());
 
         loop {
             let index = Self::hash_node_id(id, level);
@@ -823,7 +820,7 @@ lvl{}",
         // HASHING FUNCTION
         let mut level = 0;
         let mut node;
-        let mut nodes = self.node_buckets.root_for_len(id.len());
+        let mut nodes = self.node_cht.root_for_len(id.len());
 
         loop {
             let index = Self::hash_node_id(id, level);
diff --git a/src/in_memory/tree_bitmap_iterators.rs b/src/in_memory/tree_bitmap_iterators.rs
index b924664f..d10f6c30 100644
--- a/src/in_memory/tree_bitmap_iterators.rs
+++ b/src/in_memory/tree_bitmap_iterators.rs
@@ -19,16 +19,12 @@
 // contention, since every lookup has to go through the levels near the root
 // in the TreeBitMap.
 
-use crate::local_array::in_memory::tree::TreeBitMap;
+use crate::in_memory::TreeBitMap;
 use crate::{
-    af::AddressFamily,
-    local_array::{
-        bit_span::BitSpan,
-        in_memory::node::{
-            NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter,
-        },
-        types::PrefixId,
+    in_memory::tree_bitmap_node::{
+        NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter,
     },
+    types::{AddressFamily, BitSpan, PrefixId},
 };
 
 use inetnum::addr::Prefix;
diff --git a/src/in_memory/tree_bitmap_node.rs b/src/in_memory/tree_bitmap_node.rs
index 260f25fd..983d7f6b 100644
--- a/src/in_memory/tree_bitmap_node.rs
+++ b/src/in_memory/tree_bitmap_node.rs
@@ -4,18 +4,14 @@ use std::{fmt::Debug, marker::PhantomData};
 use log::{log_enabled, trace};
 use parking_lot_core::SpinWait;
 
-use super::super::bit_span::BitSpan;
-use super::tree::{
-    into_pfxbitarr, into_ptrbitarr, AtomicBitmap, AtomicPfxBitArr,
-    AtomicPtrBitArr, CasResult,
+use crate::in_memory::atomic_bitmap::{
+    AtomicBitmap, AtomicPfxBitArr, AtomicPtrBitArr, CasResult,
 };
+use crate::types::BitSpan;
 
-use crate::af::AddressFamily;
-use crate::local_array::in_memory::tree::{
-    bit_pos_from_index, ptr_bit_pos_from_index,
-};
-use crate::local_array::rib::default_store::{BIT_SPAN_SIZE, STRIDE_SIZE};
-use crate::local_array::types::PrefixId;
+use crate::rib::{BIT_SPAN_SIZE, STRIDE_SIZE};
+use crate::types::AddressFamily;
+use crate::types::PrefixId;
 
 //------------ TreeBitMap Node ----------------------------------------------
 
@@ -481,6 +477,42 @@ pub const fn ms_prefix_mask_arr(bs: BitSpan) -> u32 {
     ][(1 << bs.len) - 1 + bs.bits as usize]
 }
 
+fn into_ptrbitarr(bitmap: u32) -> u16 {
+    (bitmap >> 1) as u16
+}
+
+fn into_pfxbitarr(bitmap: u16) -> u32 {
+    (bitmap as u32) << 1
+}
+
+fn bit_pos_from_index(i: u8) -> u32 {
+    1_u32.rotate_right(1) >> i
+}
+
+fn ptr_bit_pos_from_index(i: u8) -> u16 {
+    // trace!("pfx {} ptr {} strlen {}",
+    // <$pfxsize>::BITS, <$ptrsize>::BITS, Self::STRIDE_LEN);
+    trace!("PTR_BIT_POS_FROM_INDEX {i}");
+    1_u16.rotate_right(i as u32 + 2)
+}
+
+pub(crate) fn ptr_range(ptrbitarr: u16, bs: BitSpan) -> (u16, u8) {
+    let start: u8 = (bs.bits << (4 - bs.len)) as u8;
+    let stop: u8 = start + (1 << (4 - bs.len));
+    let mask: u16 = (((1_u32 << (stop as u32 - start as u32)) - 1)
+        .rotate_right(stop as u32)
+        >> 16)
+        .try_into()
+        .unwrap();
+    if log_enabled!(log::Level::Trace) {
+        trace!("- mask      {:032b}", mask);
+        trace!("- ptrbitarr {:032b}", ptrbitarr);
+        trace!("- shl bitar {:032b}", ptrbitarr & mask);
+    }
+
+    (ptrbitarr & mask, start)
+}
+
 // Creates an Iterator that returns all prefixes that exist in a node that
 // are a more-specific prefix of the `base_prefix` + `start_bit_span`.
 //
diff --git a/src/in_memory/tree_bitmap_query.rs b/src/in_memory/tree_bitmap_query.rs
index 5cf7fcc1..a3240102 100644
--- a/src/in_memory/tree_bitmap_query.rs
+++ b/src/in_memory/tree_bitmap_query.rs
@@ -1,10 +1,10 @@
-use crate::af::AddressFamily;
+use crate::types::AddressFamily;
 
-use crate::rib::query::TreeQueryResult;
+use crate::rib::starcast_af_query::TreeQueryResult;
 use crate::{MatchOptions, MatchType};
 
-use super::super::types::PrefixId;
-use super::tree::TreeBitMap;
+use super::TreeBitMap;
+use crate::types::PrefixId;
 
 impl<AF, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE>
 where
diff --git a/src/lib.rs b/src/lib.rs
index 06783840..1c74eb79 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -10,31 +10,45 @@
 //! Part of the Rotonda modular BGP engine.
 
 //! Read more about the data-structure in this [blog post](https://blog.nlnetlabs.nl/donkeys-mules-horses/).
-mod af;
-mod local_array;
-mod prefix_record;
-
-pub mod test_types;
 
 #[macro_use]
 mod macros;
 
-mod rotonda_store;
+pub(crate) mod types;
 
 // Public Interfaces
 
-pub mod prelude;
-/// Statistics for the two trees (IPv4 and IPv6).
-pub mod stats;
-
 /// Some simple metadata implementations
 pub mod meta_examples;
 
-/// The publicly available devices
-pub use crate::rotonda_store::*;
+pub(crate) mod cht;
+pub(crate) mod in_memory;
+pub(crate) mod persist;
+pub(crate) mod prefix_cht;
+mod tests;
 
 // re-exports
 pub use crossbeam_epoch::{self as epoch, Guard};
 pub use inetnum::addr;
 
-pub use prefix_record::{PublicRecord, RecordSet};
+pub mod rib;
+
+pub use rib::starcast::StarCastRib;
+pub use rib::starcast_af::{
+    MemoryOnlyConfig, PersistHistoryConfig, PersistOnlyConfig,
+    WriteAheadConfig,
+};
+pub use types::af::IPv4;
+pub use types::af::IPv6;
+pub use types::af::IntoIpAddr;
+pub use types::errors;
+pub use types::match_options::{
+    IncludeHistory, MatchOptions, MatchType, QueryResult,
+};
+pub use types::prefix_id::RouteStatus;
+pub use types::prefix_record::PublicPrefixRecord as PrefixRecord;
+pub use types::prefix_record::PublicRecord as Record;
+pub use types::stats;
+pub use types::test_types;
+pub use types::AddressFamily;
+pub use types::Meta;
diff --git a/src/macros.rs b/src/macros.rs
index a033be94..12528227 100644
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -6,11 +6,15 @@ macro_rules! all_strategies {
         $(
             #[test]
             fn $fn_name() -> Result<(), Box<dyn std::error::Error>> {
-                use rotonda_store::prelude::multi::*;
+                use rotonda_store::{
+                        MemoryOnlyConfig,
+                        PersistOnlyConfig,
+                        PersistHistoryConfig, WriteAheadConfig
+                    };
+
                 //------- Default (MemoryOnly)
                 println!("MemoryOnly strategy starting...");
-                let tree_bitmap =
-                    MultiThreadedStore::<
+                let tree_bitmap = StarCastRib::<
                         $ty, MemoryOnlyConfig>::try_default()?;
 
                 $test_name(tree_bitmap)?;
@@ -23,7 +27,7 @@ macro_rules! all_strategies {
                     "/tmp/rotonda/".into()
                 );
 
-                let tree_bitmap = MultiThreadedStore::<
+                let tree_bitmap = StarCastRib::<
                     $ty, PersistOnlyConfig
                 >::new_with_config(
                     store_config
@@ -39,7 +43,7 @@ macro_rules! all_strategies {
                     "/tmp/rotonda/".into()
                 );
 
-                let tree_bitmap = MultiThreadedStore::<
+                let tree_bitmap = StarCastRib::<
                     $ty,
                     PersistHistoryConfig
                 >::new_with_config(
@@ -57,7 +61,7 @@ macro_rules! all_strategies {
                     "/tmp/rotonda/".into()
                 );
 
-                let tree_bitmap = MultiThreadedStore::<
+                let tree_bitmap = StarCastRib::<
                     $ty,
                     WriteAheadConfig
                 >::new_with_config(
diff --git a/src/persist/lsm_tree.rs b/src/persist/lsm_tree.rs
index b0c1d0bf..8bdfa4d4 100644
--- a/src/persist/lsm_tree.rs
+++ b/src/persist/lsm_tree.rs
@@ -12,10 +12,11 @@ use zerocopy::{
     Unaligned, U32, U64,
 };
 
-use crate::local_array::types::{PrefixId, RouteStatus};
-use crate::prefix_record::{ValueHeader, ZeroCopyRecord};
 use crate::rib::Counters;
-use crate::{AddressFamily, Meta, PublicRecord};
+use crate::types::prefix_record::PublicRecord;
+use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
+use crate::types::{AddressFamily, Meta};
+use crate::types::{PrefixId, RouteStatus};
 
 type ZeroCopyError<'a, T> = zerocopy::ConvertError<
     zerocopy::AlignmentError<&'a [u8], T>,
@@ -28,7 +29,7 @@ type ZeroCopyMutError<'a, T> = zerocopy::ConvertError<
     zerocopy::ValidityError<&'a mut [u8], T>,
 >;
 
-pub trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
+pub(crate) trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
     TryFromBytes + KnownLayout + IntoBytes + Unaligned + Immutable
 {
     fn mut_from_bytes(
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index 4d9add20..d17e1a6c 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -1,18 +1,512 @@
-use crossbeam_epoch::Guard;
+use std::collections::HashMap;
+use std::fmt::{Debug, Display};
+use std::sync::atomic::Ordering;
+use std::sync::{Arc, Mutex, MutexGuard};
+
+use crossbeam_epoch::{Atomic, Guard, Owned};
 use crossbeam_utils::Backoff;
 use inetnum::addr::Prefix;
 use log::{debug, log_enabled, trace};
 use roaring::RoaringBitmap;
 
+use crate::RouteStatus;
 use crate::{
-    local_array::in_memory::atomic_types::{
-        bits_for_len, Cht, MultiMapValue, PrefixSet, StoredPrefix,
+    cht::{bits_for_len, Cht, OnceBoxSlice, Value},
+    rib::starcast_af::UpsertReport,
+    types::{
+        errors::PrefixStoreError, AddressFamily, PrefixId, PublicRecord,
     },
-    prelude::multi::{PrefixId, PrefixStoreError},
-    rib::UpsertReport,
-    AddressFamily, Meta, PublicRecord,
+    Meta,
 };
 
+// ----------- MultiMap ------------------------------------------------------
+// This is the record that holds the aggregates at the top-level for a given
+// prefix.
+
+#[derive(Debug)]
+pub struct MultiMap<M: Meta>(
+    Arc<Mutex<std::collections::HashMap<u32, MultiMapValue<M>>>>,
+);
+
+impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
+    pub(crate) fn new(record_map: HashMap<u32, MultiMapValue<M>>) -> Self {
+        Self(Arc::new(Mutex::new(record_map)))
+    }
+
+    fn guard_with_retry(
+        &self,
+        mut retry_count: usize,
+    ) -> (MutexGuard<HashMap<u32, MultiMapValue<M>>>, usize) {
+        let backoff = Backoff::new();
+
+        loop {
+            if let Ok(guard) = self.0.try_lock() {
+                return (guard, retry_count);
+            }
+
+            backoff.spin();
+            retry_count += 1;
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        record_map.len()
+    }
+
+    pub fn get_record_for_mui(
+        &self,
+        mui: u32,
+        include_withdrawn: bool,
+    ) -> Option<PublicRecord<M>> {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+
+        record_map
+            .get(&mui)
+            .and_then(|r| -> Option<PublicRecord<M>> {
+                if include_withdrawn
+                    || r.route_status() == RouteStatus::Active
+                {
+                    Some(PublicRecord::from((mui, r)))
+                } else {
+                    None
+                }
+            })
+    }
+
+    pub fn best_backup(&self, tbi: M::TBI) -> (Option<u32>, Option<u32>) {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        let ord_routes = record_map
+            .iter()
+            .map(|r| (r.1.meta().as_orderable(tbi), *r.0));
+        let (best, bckup) =
+            routecore::bgp::path_selection::best_backup_generic(ord_routes);
+        (best.map(|b| b.1), bckup.map(|b| b.1))
+    }
+
+    pub(crate) fn get_record_for_mui_with_rewritten_status(
+        &self,
+        mui: u32,
+        bmin: &RoaringBitmap,
+        rewrite_status: RouteStatus,
+    ) -> Option<PublicRecord<M>> {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        record_map.get(&mui).map(|r| {
+            // We'll return a cloned record: the record in the store remains
+            // untouched.
+            let mut r = r.clone();
+            if bmin.contains(mui) {
+                r.set_route_status(rewrite_status);
+            }
+            PublicRecord::from((mui, &r))
+        })
+    }
+
+    pub fn get_filtered_record_for_mui(
+        &self,
+        mui: u32,
+        include_withdrawn: bool,
+        bmin: &RoaringBitmap,
+    ) -> Option<PublicRecord<M>> {
+        match include_withdrawn {
+            false => self.get_record_for_mui(mui, include_withdrawn),
+            true => self.get_record_for_mui_with_rewritten_status(
+                mui,
+                bmin,
+                RouteStatus::Withdrawn,
+            ),
+        }
+    }
+
+    // Helper to filter out records that are not-active (Inactive or
+    // Withdrawn), or whose mui appears in the global withdrawn index.
+    pub fn get_filtered_records(
+        &self,
+        mui: Option<u32>,
+        include_withdrawn: bool,
+        bmin: &RoaringBitmap,
+    ) -> Option<Vec<PublicRecord<M>>> {
+        if let Some(mui) = mui {
+            self.get_filtered_record_for_mui(mui, include_withdrawn, bmin)
+                .map(|r| vec![r])
+        } else {
+            match include_withdrawn {
+                false => {
+                    let recs = self.as_active_records_not_in_bmin(bmin);
+                    if recs.is_empty() {
+                        None
+                    } else {
+                        Some(recs)
+                    }
+                }
+                true => {
+                    let recs = self.as_records_with_rewritten_status(
+                        bmin,
+                        RouteStatus::Withdrawn,
+                    );
+                    if recs.is_empty() {
+                        None
+                    } else {
+                        Some(recs)
+                    }
+                }
+            }
+        }
+    }
+
+    // return all records regardless of their local status, or any globally
+    // set status for the mui of the record. However, the local status for a
+    // record whose mui appears in the specified bitmap index, will be
+    // rewritten with the specified RouteStatus.
+    pub fn as_records_with_rewritten_status(
+        &self,
+        bmin: &RoaringBitmap,
+        rewrite_status: RouteStatus,
+    ) -> Vec<PublicRecord<M>> {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        record_map
+            .iter()
+            .map(move |r| {
+                let mut rec = r.1.clone();
+                if bmin.contains(*r.0) {
+                    rec.set_route_status(rewrite_status);
+                }
+                PublicRecord::from((*r.0, &rec))
+            })
+            .collect::<Vec<_>>()
+    }
+
+    pub fn as_records(&self) -> Vec<PublicRecord<M>> {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        record_map
+            .iter()
+            .map(|r| PublicRecord::from((*r.0, r.1)))
+            .collect::<Vec<_>>()
+    }
+
+    // Returns a vec of records whose keys are not in the supplied bitmap
+    // index, and whose local Status is set to Active. Used to filter out
+    // withdrawn routes.
+    pub fn as_active_records_not_in_bmin(
+        &self,
+        bmin: &RoaringBitmap,
+    ) -> Vec<PublicRecord<M>> {
+        let c_map = Arc::clone(&self.0);
+        let record_map = c_map.lock().unwrap();
+        record_map
+            .iter()
+            .filter_map(|r| {
+                if r.1.route_status() == RouteStatus::Active
+                    && !bmin.contains(*r.0)
+                {
+                    Some(PublicRecord::from((*r.0, r.1)))
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>()
+    }
+
+    // Change the local status of the record for this mui to Withdrawn.
+    pub fn mark_as_withdrawn_for_mui(&self, mui: u32, ltime: u64) {
+        let c_map = Arc::clone(&self.0);
+        let mut record_map = c_map.lock().unwrap();
+        if let Some(rec) = record_map.get_mut(&mui) {
+            rec.set_route_status(RouteStatus::Withdrawn);
+            rec.set_logical_time(ltime);
+        }
+    }
+
+    // Change the local status of the record for this mui to Active.
+    pub fn mark_as_active_for_mui(&self, mui: u32, ltime: u64) {
+        let record_map = Arc::clone(&self.0);
+        let mut r_map = record_map.lock().unwrap();
+        if let Some(rec) = r_map.get_mut(&mui) {
+            rec.set_route_status(RouteStatus::Active);
+            rec.set_logical_time(ltime);
+        }
+    }
+
+    // Insert or replace the PublicRecord in the HashMap for the key of
+    // record.multi_uniq_id. Returns the number of entries in the HashMap
+    // after updating it, if it's more than 1. Returns None if this is the
+    // first entry.
+    #[allow(clippy::type_complexity)]
+    pub(crate) fn upsert_record(
+        &self,
+        new_rec: PublicRecord<M>,
+    ) -> Result<(Option<(MultiMapValue<M>, usize)>, usize), PrefixStoreError>
+    {
+        let (mut record_map, retry_count) = self.guard_with_retry(0);
+        let key = new_rec.multi_uniq_id;
+
+        match record_map.contains_key(&key) {
+            true => {
+                let old_rec = record_map
+                    .insert(key, MultiMapValue::from(new_rec))
+                    .map(|r| (r, record_map.len()));
+                Ok((old_rec, retry_count))
+            }
+            false => {
+                let new_rec = MultiMapValue::from(new_rec);
+                let old_rec = record_map.insert(key, new_rec);
+                assert!(old_rec.is_none());
+                Ok((None, retry_count))
+            }
+        }
+    }
+}
+#[derive(Clone, Debug)]
+pub(crate) struct MultiMapValue<M> {
+    meta: M,
+    ltime: u64,
+    route_status: RouteStatus,
+}
+
+impl<M: Meta> MultiMapValue<M> {
+    pub(crate) fn logical_time(&self) -> u64 {
+        self.ltime
+    }
+
+    pub(crate) fn set_logical_time(&mut self, ltime: u64) {
+        self.ltime = ltime;
+    }
+
+    pub(crate) fn meta(&self) -> &M {
+        &self.meta
+    }
+
+    pub(crate) fn route_status(&self) -> RouteStatus {
+        self.route_status
+    }
+
+    pub(crate) fn set_route_status(&mut self, status: RouteStatus) {
+        self.route_status = status;
+    }
+}
+
+impl<M: Meta> std::fmt::Display for MultiMapValue<M> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{} {}",
+            // self.meta(),
+            self.logical_time(),
+            self.route_status()
+        )
+    }
+}
+
+impl<M: Meta> From<PublicRecord<M>> for MultiMapValue<M> {
+    fn from(value: PublicRecord<M>) -> Self {
+        Self {
+            ltime: value.ltime,
+            route_status: value.status,
+            meta: value.meta,
+        }
+    }
+}
+
+impl<M: Meta> From<(u32, &MultiMapValue<M>)> for PublicRecord<M> {
+    fn from(value: (u32, &MultiMapValue<M>)) -> Self {
+        Self {
+            multi_uniq_id: value.0,
+            meta: value.1.meta().clone(),
+            ltime: value.1.ltime,
+            status: value.1.route_status,
+        }
+    }
+}
+
+impl<M: Meta> Clone for MultiMap<M> {
+    fn clone(&self) -> Self {
+        Self(Arc::clone(&self.0))
+    }
+}
+
+// ----------- Prefix related structs ---------------------------------------
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub struct PathSelections {
+    pub(crate) path_selection_muis: (Option<u32>, Option<u32>),
+}
+
+impl PathSelections {
+    pub fn best(&self) -> Option<u32> {
+        self.path_selection_muis.0
+    }
+
+    pub fn backup(&self) -> Option<u32> {
+        self.path_selection_muis.1
+    }
+}
+// ----------- StoredPrefix -------------------------------------------------
+// This is the top-level struct that's linked from the slots in the buckets.
+// It contains a super_agg_record that is supposed to hold counters for the
+// records that are stored inside it, so that iterators over its linked lists
+// don't have to go into them if there's nothing there and could stop early.
+#[derive(Debug)]
+pub struct StoredPrefix<AF: AddressFamily, M: Meta> {
+    // the serial number
+    // pub serial: usize,
+    // the prefix itself,
+    pub prefix: PrefixId<AF>,
+    // the aggregated data for this prefix
+    pub record_map: MultiMap<M>,
+    // (mui of best path entry, mui of backup path entry) from the record_map
+    path_selections: Atomic<PathSelections>,
+    // the reference to the next set of records for this prefix, if any.
+    pub next_bucket: PrefixSet<AF, M>,
+}
+
+impl<AF: AddressFamily, M: crate::types::Meta> StoredPrefix<AF, M> {
+    pub(crate) fn new(pfx_id: PrefixId<AF>, level: u8) -> Self {
+        // start calculation size of next set, it's dependent on the level
+        // we're in.
+        // let pfx_id = PrefixId::new(record.net, record.len);
+        let this_level = bits_for_len(pfx_id.get_len(), level);
+        let next_level = bits_for_len(pfx_id.get_len(), level + 1);
+
+        trace!("this level {} next level {}", this_level, next_level);
+        let next_bucket: PrefixSet<AF, M> = if next_level > 0 {
+            debug!(
+                "{} store: INSERT with new bucket of size {} at prefix len {}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                1 << (next_level - this_level),
+                pfx_id.get_len()
+            );
+            PrefixSet::init(next_level.saturating_sub(this_level))
+        } else {
+            debug!(
+                "{} store: INSERT at LAST LEVEL with empty bucket at prefix len {}",
+                std::thread::current().name().unwrap_or("unnamed-thread"),
+                pfx_id.get_len()
+            );
+            PrefixSet::init(next_level.saturating_sub(this_level))
+        };
+        // End of calculation
+
+        let rec_map = HashMap::new();
+
+        StoredPrefix {
+            // serial: 1,
+            prefix: pfx_id,
+            path_selections: Atomic::init(PathSelections {
+                path_selection_muis: (None, None),
+            }),
+            record_map: MultiMap::new(rec_map),
+            next_bucket,
+        }
+    }
+
+    pub(crate) fn get_prefix_id(&self) -> PrefixId<AF> {
+        self.prefix
+    }
+
+    pub fn get_path_selections(&self, guard: &Guard) -> PathSelections {
+        let path_selections =
+            self.path_selections.load(Ordering::Acquire, guard);
+
+        unsafe { path_selections.as_ref() }.map_or(
+            PathSelections {
+                path_selection_muis: (None, None),
+            },
+            |ps| *ps,
+        )
+    }
+
+    pub(crate) fn set_path_selections(
+        &self,
+        path_selections: PathSelections,
+        guard: &Guard,
+    ) -> Result<(), PrefixStoreError> {
+        let current = self.path_selections.load(Ordering::SeqCst, guard);
+
+        if unsafe { current.as_ref() } == Some(&path_selections) {
+            debug!("unchanged path_selections");
+            return Ok(());
+        }
+
+        self.path_selections
+            .compare_exchange(
+                current,
+                // Set the tag to indicate we're updated
+                Owned::new(path_selections).with_tag(0),
+                Ordering::AcqRel,
+                Ordering::Acquire,
+                guard,
+            )
+            .map_err(|_| PrefixStoreError::PathSelectionOutdated)?;
+        Ok(())
+    }
+
+    pub fn set_ps_outdated(
+        &self,
+        guard: &Guard,
+    ) -> Result<(), PrefixStoreError> {
+        self.path_selections
+            .fetch_update(Ordering::Acquire, Ordering::Acquire, guard, |p| {
+                Some(p.with_tag(1))
+            })
+            .map(|_| ())
+            .map_err(|_| PrefixStoreError::StoreNotReadyError)
+    }
+
+    pub fn is_ps_outdated(&self, guard: &Guard) -> bool {
+        self.path_selections.load(Ordering::Acquire, guard).tag() == 1
+    }
+
+    pub fn calculate_and_store_best_backup<'a>(
+        &'a self,
+        tbi: &M::TBI,
+        guard: &'a Guard,
+    ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
+        let path_selection_muis = self.record_map.best_backup(*tbi);
+
+        self.set_path_selections(
+            PathSelections {
+                path_selection_muis,
+            },
+            guard,
+        )?;
+
+        Ok(path_selection_muis)
+    }
+}
+//------------ PrefixSet ----------------------------------------------------
+
+// The PrefixSet is the ARRAY that holds all the child prefixes in a node.
+// Since we are storing these prefixes in the global store in a HashMap that
+// is keyed on the tuple (addr_bits, len, serial number) we can get away with
+// storing ONLY THE SERIAL NUMBER in the pfx_vec: The addr_bits and len are
+// implied in the position in the array a serial number has. A PrefixSet
+// doesn't know anything about the node it is contained in, so it needs a base
+// address to be able to calculate the complete prefix of a child prefix.
+
+#[derive(Debug)]
+#[repr(align(8))]
+pub struct PrefixSet<AF: AddressFamily, M: Meta>(
+    pub OnceBoxSlice<StoredPrefix<AF, M>>,
+);
+
+impl<AF: AddressFamily, M: Meta> PrefixSet<AF, M> {
+    pub fn init(p2_size: u8) -> Self {
+        PrefixSet(OnceBoxSlice::new(p2_size))
+    }
+}
+
+impl<AF: AddressFamily, M: Meta> Value for PrefixSet<AF, M> {
+    fn init(p2_size: usize) -> Self {
+        PrefixSet(OnceBoxSlice::new(p2_size as u8))
+    }
+}
+
 #[derive(Debug)]
 pub(crate) struct PrefixCht<
     AF: AddressFamily,
diff --git a/src/prefix_cht/iterators.rs b/src/prefix_cht/iterators.rs
index efbb7e22..0026d5b0 100644
--- a/src/prefix_cht/iterators.rs
+++ b/src/prefix_cht/iterators.rs
@@ -2,15 +2,14 @@ use log::trace;
 use roaring::RoaringBitmap;
 
 use crate::{
-    local_array::{
-        bit_span::BitSpan,
-        in_memory::{
-            node::{NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter},
-            tree::TreeBitMap,
+    in_memory::{
+        tree_bitmap_node::{
+            NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter,
         },
+        TreeBitMap,
     },
-    prelude::multi::PrefixId,
-    AddressFamily,
+    // prelude::multi::PrefixId,
+    types::{AddressFamily, BitSpan, PrefixId},
 };
 
 pub(crate) struct _MoreSpecificPrefixIter<
diff --git a/src/query.rs b/src/query.rs
deleted file mode 100644
index 17424d2f..00000000
--- a/src/query.rs
+++ /dev/null
@@ -1,395 +0,0 @@
-use crossbeam_epoch::{self as epoch};
-use epoch::Guard;
-use log::trace;
-use zerocopy::TryFromBytes;
-
-use crate::af::AddressFamily;
-use crate::prefix_record::ZeroCopyRecord;
-use crate::rib::{Config, PersistStrategy, Rib};
-use crate::PublicRecord;
-use inetnum::addr::Prefix;
-
-use crate::{Meta, QueryResult};
-
-use crate::{MatchOptions, MatchType};
-
-use super::errors::PrefixStoreError;
-use super::types::PrefixId;
-
-//------------ Prefix Matching ----------------------------------------------
-
-impl<
-        'a,
-        AF: AddressFamily,
-        M: Meta,
-        const N_ROOT_SIZE: usize,
-        const P_ROOT_SIZE: usize,
-        C: Config,
-        const KEY_SIZE: usize,
-    > Rib<AF, M, N_ROOT_SIZE, P_ROOT_SIZE, C, KEY_SIZE>
-{
-    pub(crate) fn get_value(
-        &'a self,
-        prefix_id: PrefixId<AF>,
-        mui: Option<u32>,
-        include_withdrawn: bool,
-        guard: &'a Guard,
-    ) -> Option<Vec<PublicRecord<M>>> {
-        match self.persist_strategy() {
-            PersistStrategy::PersistOnly => {
-                trace!("get value from persist_store for {:?}", prefix_id);
-                self.persist_tree.as_ref().and_then(|tree| {
-                    tree.get_records_for_prefix(
-                        prefix_id,
-                        mui,
-                        include_withdrawn,
-                        self.in_memory_tree.withdrawn_muis_bmin(guard),
-                    )
-                    .map(|v| {
-                        v.iter()
-                            .map(|bytes| {
-                                let record: &ZeroCopyRecord<AF> =
-                                    ZeroCopyRecord::try_ref_from_bytes(bytes)
-                                        .unwrap();
-                                PublicRecord::<M> {
-                                    multi_uniq_id: record.multi_uniq_id,
-                                    ltime: record.ltime,
-                                    status: record.status,
-                                    meta: <Vec<u8>>::from(
-                                        record.meta.as_ref(),
-                                    )
-                                    .into(),
-                                }
-                            })
-                            .collect::<Vec<_>>()
-                    })
-                })
-            }
-            _ => self.prefix_cht.get_records_for_prefix(
-                prefix_id,
-                mui,
-                include_withdrawn,
-                self.in_memory_tree.withdrawn_muis_bmin(guard),
-            ),
-        }
-    }
-
-    pub(crate) fn more_specifics_from(
-        &'a self,
-        prefix_id: PrefixId<AF>,
-        mui: Option<u32>,
-        include_withdrawn: bool,
-        guard: &'a Guard,
-    ) -> QueryResult<M> {
-        let prefix = if !self.contains(prefix_id, mui) {
-            Some(Prefix::from(prefix_id))
-        } else {
-            None
-        };
-        let more_specifics = self
-            .in_memory_tree
-            .more_specific_prefix_iter_from(prefix_id)
-            .map(|p| {
-                self.get_value(prefix_id, mui, include_withdrawn, guard)
-                    .map(|v| (p, v))
-            })
-            .collect();
-
-        QueryResult {
-            prefix,
-            prefix_meta: prefix
-                .map(|_pfx| {
-                    self.get_value(prefix_id, mui, include_withdrawn, guard)
-                        .unwrap_or_default()
-                })
-                .unwrap_or(vec![]),
-            match_type: MatchType::EmptyMatch,
-            less_specifics: None,
-            more_specifics,
-        }
-    }
-
-    pub(crate) fn less_specifics_from(
-        &'a self,
-        prefix_id: PrefixId<AF>,
-        mui: Option<u32>,
-        include_withdrawn: bool,
-        guard: &'a Guard,
-    ) -> QueryResult<M> {
-        let prefix = if !self.contains(prefix_id, mui) {
-            Some(Prefix::from(prefix_id))
-        } else {
-            None
-        };
-
-        let less_specifics = self
-            .in_memory_tree
-            .less_specific_prefix_iter(prefix_id)
-            .map(|p| {
-                self.get_value(prefix_id, mui, include_withdrawn, guard)
-                    .map(|v| (p, v))
-            })
-            .collect();
-
-        QueryResult {
-            prefix,
-            prefix_meta: self
-                .get_value(prefix_id, mui, include_withdrawn, guard)
-                .unwrap_or_default(),
-            match_type: MatchType::EmptyMatch,
-            less_specifics,
-            more_specifics: None,
-        }
-    }
-
-    pub(crate) fn more_specifics_iter_from(
-        &'a self,
-        prefix_id: PrefixId<AF>,
-        mui: Option<u32>,
-        include_withdrawn: bool,
-        guard: &'a Guard,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-        println!("more_specifics_iter_from fn");
-        // If the user wanted a specific mui and not withdrawn prefixes, we
-        // may return early if the mui is globally withdrawn.
-        (if mui.is_some_and(|m| {
-            !include_withdrawn && self.mui_is_withdrawn(m, guard)
-        }) {
-            None
-        } else {
-            Some(
-                self.in_memory_tree
-                    .more_specific_prefix_iter_from(prefix_id)
-                    .filter_map(move |p| {
-                        self.get_value(p, mui, include_withdrawn, guard)
-                            .map(|v| (p, v))
-                    }),
-            )
-        })
-        .into_iter()
-        .flatten()
-        // .chain(
-        //     (if mui.is_some_and(|m| {
-        //         self.config.persist_strategy == PersistStrategy::WriteAhead
-        //             || (!include_withdrawn && self.mui_is_withdrawn(m, guard))
-        //     }) {
-        //         None
-        //     } else {
-        //         let global_withdrawn_bmin =
-        //             self.in_memory_tree.withdrawn_muis_bmin(guard);
-        //         self.persist_tree.as_ref().map(|persist_tree| {
-        //             persist_tree.more_specific_prefix_iter_from(
-        //                 prefix_id,
-        //                 vec![],
-        //                 mui,
-        //                 global_withdrawn_bmin,
-        //                 include_withdrawn,
-        //             )
-        //         })
-        //     })
-        //     .into_iter()
-        //     .flatten(),
-        // )
-    }
-
-    pub(crate) fn less_specifics_iter_from(
-        &'a self,
-        prefix_id: PrefixId<AF>,
-        mui: Option<u32>,
-        include_withdrawn: bool,
-        guard: &'a Guard,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-        self.in_memory_tree
-            .less_specific_prefix_iter(prefix_id)
-            .filter_map(move |p| {
-                self.get_value(p, mui, include_withdrawn, guard)
-                    .map(|v| (p, v))
-            })
-    }
-
-    pub(crate) fn match_prefix(
-        &'a self,
-        search_pfx: PrefixId<AF>,
-        options: &MatchOptions,
-        guard: &'a Guard,
-    ) -> QueryResult<M> {
-        trace!("match_prefix rib {:?} {:?}", search_pfx, options);
-        let res = self.in_memory_tree.match_prefix(search_pfx, options);
-
-        trace!("res {:?}", res);
-        let mut res = QueryResult::from(res);
-
-        if let Some(Some(m)) = res.prefix.map(|p| {
-            self.get_value(
-                p.into(),
-                options.mui,
-                options.include_withdrawn,
-                guard,
-            )
-            .and_then(|v| if v.is_empty() { None } else { Some(v) })
-        }) {
-            res.prefix_meta = m;
-        } else {
-            res.prefix = None;
-            res.match_type = MatchType::EmptyMatch;
-        }
-
-        if options.include_more_specifics {
-            res.more_specifics = res.more_specifics.map(|p| {
-                p.iter()
-                    .filter_map(|mut r| {
-                        if let Some(m) = self.get_value(
-                            r.prefix.into(),
-                            options.mui,
-                            options.include_withdrawn,
-                            guard,
-                        ) {
-                            r.meta = m;
-                            Some(r)
-                        } else {
-                            None
-                        }
-                    })
-                    .collect()
-            });
-        }
-        if options.include_less_specifics {
-            res.less_specifics = res.less_specifics.map(|p| {
-                p.iter()
-                    .filter_map(|mut r| {
-                        if let Some(m) = self.get_value(
-                            r.prefix.into(),
-                            options.mui,
-                            options.include_withdrawn,
-                            guard,
-                        ) {
-                            r.meta = m;
-                            Some(r)
-                        } else {
-                            None
-                        }
-                    })
-                    .collect()
-            });
-        }
-
-        res
-    }
-
-    pub(crate) fn best_path(
-        &'a self,
-        search_pfx: PrefixId<AF>,
-        guard: &Guard,
-    ) -> Option<Result<PublicRecord<M>, PrefixStoreError>> {
-        self.prefix_cht
-            .non_recursive_retrieve_prefix(search_pfx)
-            .0
-            .map(|p_rec| {
-                p_rec.get_path_selections(guard).best().map_or_else(
-                    || Err(PrefixStoreError::BestPathNotFound),
-                    |mui| {
-                        p_rec
-                            .record_map
-                            .get_record_for_mui(mui, false)
-                            .ok_or(PrefixStoreError::StoreNotReadyError)
-                    },
-                )
-            })
-    }
-
-    pub(crate) fn calculate_and_store_best_and_backup_path(
-        &self,
-        search_pfx: PrefixId<AF>,
-        tbi: &<M as Meta>::TBI,
-        guard: &Guard,
-    ) -> Result<(Option<u32>, Option<u32>), PrefixStoreError> {
-        self.prefix_cht
-            .non_recursive_retrieve_prefix(search_pfx)
-            .0
-            .map_or(Err(PrefixStoreError::StoreNotReadyError), |p_rec| {
-                p_rec.calculate_and_store_best_backup(tbi, guard)
-            })
-    }
-
-    pub(crate) fn is_ps_outdated(
-        &self,
-        search_pfx: PrefixId<AF>,
-        guard: &Guard,
-    ) -> Result<bool, PrefixStoreError> {
-        self.prefix_cht
-            .non_recursive_retrieve_prefix(search_pfx)
-            .0
-            .map_or(Err(PrefixStoreError::StoreNotReadyError), |p| {
-                Ok(p.is_ps_outdated(guard))
-            })
-    }
-}
-
-#[derive(Debug)]
-pub(crate) struct TreeQueryResult<AF: AddressFamily> {
-    pub match_type: MatchType,
-    pub prefix: Option<PrefixId<AF>>,
-    pub less_specifics: Option<Vec<PrefixId<AF>>>,
-    pub more_specifics: Option<Vec<PrefixId<AF>>>,
-}
-
-impl<AF: AddressFamily, M: Meta> From<TreeQueryResult<AF>>
-    for QueryResult<M>
-{
-    fn from(value: TreeQueryResult<AF>) -> Self {
-        Self {
-            match_type: value.match_type,
-            prefix: value.prefix.map(|p| p.into()),
-            prefix_meta: vec![],
-            less_specifics: value
-                .less_specifics
-                .map(|ls| ls.into_iter().map(|p| (p, vec![])).collect()),
-            more_specifics: value
-                .more_specifics
-                .map(|ms| ms.into_iter().map(|p| (p, vec![])).collect()),
-        }
-    }
-}
-
-impl<AF: AddressFamily, M: Meta> From<TreeQueryResult<AF>>
-    for FamilyQueryResult<AF, M>
-{
-    fn from(value: TreeQueryResult<AF>) -> Self {
-        Self {
-            match_type: value.match_type,
-            prefix: value.prefix,
-            prefix_meta: vec![],
-            less_specifics: None,
-            more_specifics: None,
-        }
-    }
-}
-
-pub(crate) type FamilyRecord<AF, M> =
-    Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>;
-
-pub(crate) struct FamilyQueryResult<AF: AddressFamily, M: Meta> {
-    pub match_type: MatchType,
-    pub prefix: Option<PrefixId<AF>>,
-    pub prefix_meta: Vec<PublicRecord<M>>,
-    pub less_specifics: Option<FamilyRecord<AF, M>>,
-    pub more_specifics: Option<FamilyRecord<AF, M>>,
-}
-
-impl<AF: AddressFamily, M: Meta> From<FamilyQueryResult<AF, M>>
-    for QueryResult<M>
-{
-    fn from(value: FamilyQueryResult<AF, M>) -> Self {
-        QueryResult {
-            match_type: value.match_type,
-            prefix: value.prefix.map(|p| p.into()),
-            prefix_meta: value.prefix_meta,
-            less_specifics: value
-                .less_specifics
-                .map(|ls| ls.into_iter().collect()),
-            more_specifics: value
-                .more_specifics
-                .map(|ms| ms.into_iter().collect()),
-        }
-    }
-}
diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index 5a18a530..be629ca6 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -9,18 +9,14 @@ use crossbeam_epoch::{self as epoch};
 use epoch::{Guard, Owned};
 use zerocopy::TryFromBytes;
 
-use crate::in_memory::tree::TreeBitMap;
+use crate::in_memory::tree_bitmap::TreeBitMap;
 use crate::persist::lsm_tree::LongKey;
 use crate::prefix_cht::cht::PrefixCht;
-// use crate::prelude::multi::RouteStatus;
 use crate::stats::CreatedNodes;
 use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
 use crate::types::{PrefixId, RouteStatus};
 use crate::{types::errors::PrefixStoreError, types::PublicRecord};
 
-// Make sure to also import the other methods for the Rib, so the proc macro
-// create_store can use them.
-pub use crate::in_memory::iterators;
 pub use crate::rib::starcast_af_query;
 
 use crate::{IPv4, IPv6, Meta};
diff --git a/src/tests.rs b/src/tests.rs
index 646e2206..af0bc5f2 100644
--- a/src/tests.rs
+++ b/src/tests.rs
@@ -5,8 +5,8 @@ use std::error::Error;
 
 #[test]
 fn test_af_1() -> Result<(), Box<dyn Error>> {
-    use crate::local_array::bit_span::BitSpan;
-    use crate::prelude::multi::StrideNodeId;
+    use crate::in_memory::tree_bitmap_node::StrideNodeId;
+    use crate::types::BitSpan;
     use crate::AddressFamily;
     use crate::IPv4;
 
@@ -34,9 +34,8 @@ fn test_af_1() -> Result<(), Box<dyn Error>> {
 
 #[test]
 fn test_af_2() -> Result<(), Box<dyn Error>> {
-    use crate::local_array::bit_span::BitSpan;
-    use crate::prelude::multi::StrideNodeId;
     use crate::IPv4;
+    use crate::{in_memory::tree_bitmap_node::StrideNodeId, types::BitSpan};
 
     let bit_addr: IPv4 = 0b1111_1111_1111_1111_1111_1111_1111_1111.into();
     let nu_prefix = StrideNodeId::dangerously_new_with_id_as_is(bit_addr, 8);
diff --git a/tests/best-path.rs b/tests/best-path.rs
index ef28ab34..35daa0cf 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -1,14 +1,14 @@
 use inetnum::addr::Prefix;
 use inetnum::asn::Asn;
 use log::trace;
-use rotonda_store::prelude::multi::PrefixStoreError;
-use rotonda_store::prelude::multi::Record;
-use rotonda_store::prelude::multi::RouteStatus;
-use rotonda_store::rib::MemoryOnlyConfig;
+use rotonda_store::errors::PrefixStoreError;
 use rotonda_store::IncludeHistory;
 use rotonda_store::MatchOptions;
+use rotonda_store::MemoryOnlyConfig;
 use rotonda_store::Meta;
-use rotonda_store::MultiThreadedStore;
+use rotonda_store::Record;
+use rotonda_store::RouteStatus;
+use rotonda_store::StarCastRib;
 use routecore::bgp::aspath::HopPath;
 use routecore::bgp::path_attributes::BgpIdentifier;
 use routecore::bgp::path_attributes::PaMap;
@@ -75,7 +75,7 @@ fn test_best_path_1(// tree_bitmap: MultiThreadedStore<Ipv4Route>,
     crate::common::init();
 
     let tree_bitmap =
-        std::sync::Arc::new(std::sync::Arc::new(MultiThreadedStore::<
+        std::sync::Arc::new(std::sync::Arc::new(StarCastRib::<
             Ipv4Route,
             MemoryOnlyConfig,
         >::try_default()?));
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index 911a85ce..ef040ea1 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -2,11 +2,9 @@ use std::{str::FromStr, sync::atomic::Ordering};
 
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
-    meta_examples::NoMeta,
-    prelude::multi::RouteStatus,
-    rib::{Config, MemoryOnlyConfig},
-    test_types::BeBytesAsn,
-    IncludeHistory, MatchOptions, MultiThreadedStore, PublicRecord as Record,
+    meta_examples::NoMeta, rib::starcast_af::Config, test_types::BeBytesAsn,
+    IncludeHistory, MatchOptions, MemoryOnlyConfig, Record, RouteStatus,
+    StarCastRib,
 };
 
 mod common {
@@ -27,7 +25,7 @@ rotonda_store::all_strategies![
 ];
 
 fn test_concurrent_updates_1<C: Config + Sync + Send + 'static>(
-    tree_bitmap: MultiThreadedStore<BeBytesAsn, C>,
+    tree_bitmap: StarCastRib<BeBytesAsn, C>,
 ) -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
@@ -417,7 +415,7 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
 
     let cur_ltime = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
 
-    let tree_bitmap = std::sync::Arc::new(MultiThreadedStore::<
+    let tree_bitmap = std::sync::Arc::new(StarCastRib::<
         BeBytesAsn,
         MemoryOnlyConfig,
     >::try_default()?);
@@ -734,7 +732,7 @@ fn more_specifics_short_lengths() -> Result<(), Box<dyn std::error::Error>> {
     crate::common::init();
 
     println!("PersistOnly strategy starting...");
-    let tree_bitmap = std::sync::Arc::new(MultiThreadedStore::<
+    let tree_bitmap = std::sync::Arc::new(StarCastRib::<
         NoMeta,
         MemoryOnlyConfig,
     >::try_default()?);
diff --git a/tests/full-table.rs b/tests/full-table.rs
index 2a3b8950..7f376dc5 100644
--- a/tests/full-table.rs
+++ b/tests/full-table.rs
@@ -3,7 +3,11 @@
 mod tests {
     use inetnum::addr::Prefix;
     use inetnum::asn::Asn;
-    use rotonda_store::{prelude::multi::*, prelude::*};
+    use rotonda_store::rib::starcast_af::Config;
+    use rotonda_store::{
+        epoch, IncludeHistory, MatchOptions, MatchType, Meta, PrefixRecord,
+        Record, RouteStatus, StarCastRib,
+    };
 
     use std::error::Error;
     use std::fs::File;
@@ -60,7 +64,7 @@ mod tests {
 
     // #[test]
     fn test_full_table_from_csv<C: Config>(
-        tree_bitmap: MultiThreadedStore<AsnList, C>,
+        tree_bitmap: StarCastRib<AsnList, C>,
     ) -> Result<(), Box<dyn Error>> {
         // These constants are all contingent on the exact csv file,
         // being loaded!
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index a3efc793..90d6fce5 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -3,12 +3,11 @@
 use std::error::Error;
 
 use inetnum::addr::Prefix;
-
 use rotonda_store::{
-    meta_examples::PrefixAs, prelude::multi::RouteStatus, rib::Config,
-    IncludeHistory, MatchOptions, MatchType, MultiThreadedStore,
-    PublicRecord as Record,
+    meta_examples::PrefixAs, rib::starcast_af::Config, IncludeHistory,
+    MatchOptions, MatchType, Record, RouteStatus, StarCastRib,
 };
+
 mod common {
     use std::io::Write;
 
@@ -28,7 +27,7 @@ rotonda_store::all_strategies![
 
 // #[test]
 fn test_more_specifics_without_less_specifics<C: Config>(
-    tree_bitmap: MultiThreadedStore<PrefixAs, C>,
+    tree_bitmap: StarCastRib<PrefixAs, C>,
 ) -> Result<(), Box<dyn Error>> {
     crate::common::init();
 
@@ -130,7 +129,7 @@ rotonda_store::all_strategies![
 ];
 
 fn test_more_specifics_with_less_specifics<C: Config>(
-    tree_bitmap: MultiThreadedStore<PrefixAs, C>,
+    tree_bitmap: StarCastRib<PrefixAs, C>,
 ) -> Result<(), Box<dyn Error>> {
     crate::common::init();
 
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 7e8863df..9209bf0f 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -1,7 +1,9 @@
 // type Prefix4<'a> = Prefix<u32, PrefixAs>;
 use inetnum::addr::Prefix;
-use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::{prelude::multi::*, prelude::*};
+use rotonda_store::{
+    epoch, meta_examples::PrefixAs, rib::starcast_af::Config, IncludeHistory,
+    MatchOptions, MatchType, Record, RouteStatus, StarCastRib,
+};
 
 use std::error::Error;
 
@@ -24,7 +26,7 @@ rotonda_store::all_strategies![
 
 // #[test]
 fn test_more_specifics<C: Config>(
-    tree_bitmap: MultiThreadedStore<PrefixAs, C>,
+    tree_bitmap: StarCastRib<PrefixAs, C>,
 ) -> Result<(), Box<dyn Error>> {
     crate::common::init();
 
@@ -265,7 +267,7 @@ rotonda_store::all_strategies![
 ];
 
 fn test_brunos_more_specifics<C: Config>(
-    tree_bitmap: MultiThreadedStore<PrefixAs, C>,
+    tree_bitmap: StarCastRib<PrefixAs, C>,
 ) -> Result<(), Box<dyn Error>> {
     tree_bitmap.insert(
         &Prefix::new(std::net::Ipv4Addr::new(168, 181, 224, 0).into(), 22)
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 5b38256e..1920fb60 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -16,9 +16,11 @@ mod tests {
     use inetnum::addr::Prefix;
     use log::trace;
     use rotonda_store::{
+        epoch,
         meta_examples::{NoMeta, PrefixAs},
-        prelude::multi::*,
-        prelude::*,
+        rib::starcast_af::Config,
+        IncludeHistory, IntoIpAddr, MatchOptions, MatchType, Record,
+        RouteStatus, StarCastRib,
     };
 
     rotonda_store::all_strategies![
@@ -29,7 +31,7 @@ mod tests {
 
     // #[test]
     fn test_insert_extremes_ipv4<C: Config>(
-        trie: MultiThreadedStore<NoMeta, C>,
+        trie: StarCastRib<NoMeta, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         let min_pfx = Prefix::new_relaxed(
             std::net::Ipv4Addr::new(0, 0, 0, 0).into(),
@@ -107,7 +109,7 @@ mod tests {
 
     // #[test]
     fn test_tree_ipv4<C: Config>(
-        tree_bitmap: MultiThreadedStore<PrefixAs, C>,
+        tree_bitmap: StarCastRib<PrefixAs, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
@@ -424,7 +426,7 @@ mod tests {
 
     // #[test]
     fn test_ranges_ipv4<C: Config>(
-        _tree_bitmap: MultiThreadedStore<NoMeta, C>,
+        _tree_bitmap: StarCastRib<NoMeta, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         // for persist_strategy in [
         //     PersistStrategy::MemoryOnly,
@@ -433,7 +435,7 @@ mod tests {
         //     // PersistStrategy::PersistHistory,
 
         for i_net in 0..255 {
-            let tree_bitmap = MultiThreadedStore::<NoMeta, C>::try_default()?;
+            let tree_bitmap = StarCastRib::<NoMeta, C>::try_default()?;
 
             let pfx_vec: Vec<Prefix> = (1..32)
                 .collect::<Vec<u8>>()
@@ -497,7 +499,7 @@ mod tests {
 
     // #[test]
     fn test_multi_ranges_ipv4<C: Config>(
-        tree_bitmap: MultiThreadedStore<NoMeta, C>,
+        tree_bitmap: StarCastRib<NoMeta, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
diff --git a/tests/treebitmap_v6.rs b/tests/treebitmap_v6.rs
index 43dd9e9d..43839fe1 100644
--- a/tests/treebitmap_v6.rs
+++ b/tests/treebitmap_v6.rs
@@ -13,8 +13,12 @@ mod common {
 mod tests {
     use inetnum::addr::Prefix;
     use rotonda_store::{
-        meta_examples::NoMeta, meta_examples::PrefixAs, prelude::multi::*,
-        prelude::*,
+        epoch,
+        meta_examples::{NoMeta, PrefixAs},
+        rib::starcast_af::Config,
+        IncludeHistory, IntoIpAddr, MatchOptions, MatchType,
+        MemoryOnlyConfig, PersistOnlyConfig, Record, RouteStatus,
+        StarCastRib,
     };
 
     rotonda_store::all_strategies![
@@ -25,7 +29,7 @@ mod tests {
 
     // #[test]
     fn test_arbitrary_insert_ipv6<C: Config>(
-        trie: MultiThreadedStore<NoMeta, C>,
+        trie: StarCastRib<NoMeta, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
         // let trie = &mut MultiThreadedStore::<NoMeta>::try_default()?;
@@ -74,7 +78,7 @@ mod tests {
 
     // #[test]
     fn test_insert_extremes_ipv6<C: Config>(
-        trie: MultiThreadedStore<NoMeta, C>,
+        trie: StarCastRib<NoMeta, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
@@ -168,7 +172,7 @@ mod tests {
     // the end of a prefix-length array).
     // #[test]
     fn test_max_levels<C: Config>(
-        tree_bitmap: MultiThreadedStore<PrefixAs, C>,
+        tree_bitmap: StarCastRib<PrefixAs, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         crate::common::init();
 
@@ -351,7 +355,7 @@ mod tests {
 
     // #[test]
     fn test_tree_ipv6<C: Config>(
-        tree_bitmap: MultiThreadedStore<PrefixAs, C>,
+        tree_bitmap: StarCastRib<PrefixAs, C>,
     ) -> Result<(), Box<dyn std::error::Error>> {
         // let tree_bitmap = MultiThreadedStore::<PrefixAs>::try_default()?;
         let pfxs = vec![
@@ -670,20 +674,76 @@ mod tests {
         Ok(())
     }
 
-    // rotonda_store::all_strategies![
-    //     ranges_ipv4;
-    //     test_ranges_ipv4;
-    //     NoMeta
-    // ];
-
+    // This test cannot be run with the current test creation macro. The
+    // test recreates the store for each outer loop!
     #[test]
-    fn test_ranges_ipv4(// tree_bitmap: MultiThreadedStore<NoMeta>,
-    ) -> Result<(), Box<dyn std::error::Error>> {
+    fn test_ranges_ipv6_mo() -> Result<(), Box<dyn std::error::Error>> {
         for i_net in 0..255 {
             let tree_bitmap =
-                MultiThreadedStore::<NoMeta, MemoryOnlyConfig>::try_default(
+                StarCastRib::<NoMeta, MemoryOnlyConfig>::try_default()?;
+
+            let pfx_vec: Vec<Prefix> = (1..32)
+                .collect::<Vec<u8>>()
+                .into_iter()
+                .map(|i_len| {
+                    Prefix::new_relaxed(
+                        std::net::Ipv6Addr::new(i_net, 0, 0, 0, 0, 0, 0, 0)
+                            .into(),
+                        i_len,
+                    )
+                    .unwrap()
+                })
+                .collect();
+
+            let mut i_len_s = 0;
+            for pfx in pfx_vec {
+                i_len_s += 1;
+                tree_bitmap.insert(
+                    &pfx,
+                    Record::new(0, 0, RouteStatus::Active, NoMeta::Empty),
+                    None,
                 )?;
 
+                let res_pfx = Prefix::new_relaxed(
+                    std::net::Ipv6Addr::new(i_net, 0, 0, 0, 0, 0, 0, 0)
+                        .into(),
+                    i_len_s,
+                );
+
+                let guard = &epoch::pin();
+                for s_len in i_len_s..32 {
+                    let pfx = Prefix::new_relaxed(
+                        std::net::Ipv6Addr::new(i_net, 0, 0, 0, 0, 0, 0, 0)
+                            .into(),
+                        s_len,
+                    )?;
+                    let res = tree_bitmap.match_prefix(
+                        &pfx,
+                        &MatchOptions {
+                            match_type: MatchType::LongestMatch,
+                            include_withdrawn: false,
+                            include_less_specifics: false,
+                            include_more_specifics: false,
+                            mui: None,
+                            include_history: IncludeHistory::None,
+                        },
+                        guard,
+                    );
+                    println!("{:?}", pfx);
+
+                    assert_eq!(res.prefix.unwrap(), res_pfx?);
+                }
+            }
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_ranges_ipv6_po() -> Result<(), Box<dyn std::error::Error>> {
+        for i_net in 0..255 {
+            let tree_bitmap =
+                StarCastRib::<NoMeta, PersistOnlyConfig>::try_default()?;
+
             let pfx_vec: Vec<Prefix> = (1..32)
                 .collect::<Vec<u8>>()
                 .into_iter()

From 3aecbb95df62ffbf9dc40d92259c08f18d338c64 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 5 Mar 2025 15:39:17 +0100
Subject: [PATCH 110/147] even more reshuffling

---
 src/bin/load_mrt.rs                           |   6 +-
 src/cht/mod.rs                                |   1 -
 src/in_memory/mod.rs                          |   9 --
 src/lib.rs                                    |  33 +++---
 src/{persist/lsm_tree.rs => lsm_tree/mod.rs}  |  50 ++++----
 src/persist/mod.rs                            |   1 -
 src/prefix_cht/cht.rs                         |  11 +-
 src/prefix_cht/iterators.rs                   |   9 +-
 src/rib/mod.rs                                |  13 +-
 src/rib/starcast.rs                           |   6 +-
 src/rib/starcast_af.rs                        |  51 ++++----
 src/rib/starcast_af_query.rs                  |  16 +--
 .../atomic_bitmap.rs                          |   2 +-
 .../tree_bitmap.rs => tree_bitmap/mod.rs}     |  20 +++-
 src/{in_memory => tree_bitmap}/node_cht.rs    |   2 +-
 .../tree_bitmap_iterators.rs                  |   4 +-
 .../tree_bitmap_node.rs                       |   4 +-
 .../tree_bitmap_query.rs                      |   2 +-
 src/types/match_options.rs                    |   4 +-
 src/{ => types}/meta_examples.rs              |   0
 src/types/mod.rs                              |  16 ++-
 src/types/prefix_id.rs                        |  59 +--------
 src/types/prefix_record.rs                    | 112 +++++++++---------
 src/types/route_status.rs                     |  56 +++++++++
 src/types/stats.rs                            |   3 +-
 src/{ => types}/tests.rs                      |   4 +-
 tests/concurrency.rs                          |   5 +-
 tests/full-table.rs                           |   2 +-
 tests/more-more-specifics.rs                  |   4 +-
 tests/more-specifics.rs                       |   4 +-
 tests/treebitmap.rs                           |   3 +-
 tests/treebitmap_v6.rs                        |   3 +-
 32 files changed, 246 insertions(+), 269 deletions(-)
 delete mode 100644 src/in_memory/mod.rs
 rename src/{persist/lsm_tree.rs => lsm_tree/mod.rs} (97%)
 delete mode 100644 src/persist/mod.rs
 rename src/{in_memory => tree_bitmap}/atomic_bitmap.rs (98%)
 rename src/{in_memory/tree_bitmap.rs => tree_bitmap/mod.rs} (99%)
 rename src/{in_memory => tree_bitmap}/node_cht.rs (98%)
 rename src/{in_memory => tree_bitmap}/tree_bitmap_iterators.rs (99%)
 rename src/{in_memory => tree_bitmap}/tree_bitmap_node.rs (99%)
 rename src/{in_memory => tree_bitmap}/tree_bitmap_query.rs (99%)
 rename src/{ => types}/meta_examples.rs (100%)
 create mode 100644 src/types/route_status.rs
 rename src/{ => types}/tests.rs (92%)

diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index 1d6aa538..cb2d83b5 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -1,12 +1,12 @@
-use rotonda_store::rib::starcast_af::Config;
-use rotonda_store::rib::starcast_af::PersistStrategy;
-use rotonda_store::rib::starcast_af::UpsertReport;
+use rotonda_store::Config;
 use rotonda_store::MemoryOnlyConfig;
 use rotonda_store::PersistHistoryConfig;
 use rotonda_store::PersistOnlyConfig;
+use rotonda_store::PersistStrategy;
 use rotonda_store::Record;
 use rotonda_store::RouteStatus;
 use rotonda_store::StarCastRib;
+use rotonda_store::UpsertReport;
 use rotonda_store::WriteAheadConfig;
 use routecore::bgp::aspath::HopPath;
 use routecore::bgp::message::update_builder::StandardCommunitiesList;
diff --git a/src/cht/mod.rs b/src/cht/mod.rs
index 4c1acf1e..e24eeecf 100644
--- a/src/cht/mod.rs
+++ b/src/cht/mod.rs
@@ -1,6 +1,5 @@
 mod oncebox;
 
-pub(crate) use oncebox::OnceBox;
 pub(crate) use oncebox::OnceBoxSlice;
 
 use crate::rib::STRIDE_SIZE;
diff --git a/src/in_memory/mod.rs b/src/in_memory/mod.rs
deleted file mode 100644
index 4a031dd6..00000000
--- a/src/in_memory/mod.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-pub(crate) mod atomic_bitmap;
-pub(crate) mod node_cht;
-
-pub(crate) mod tree_bitmap;
-pub(crate) mod tree_bitmap_iterators;
-pub(crate) mod tree_bitmap_node;
-mod tree_bitmap_query;
-
-pub(crate) use tree_bitmap::TreeBitMap;
diff --git a/src/lib.rs b/src/lib.rs
index 1c74eb79..e3e1218f 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -10,34 +10,31 @@
 //! Part of the Rotonda modular BGP engine.
 
 //! Read more about the data-structure in this [blog post](https://blog.nlnetlabs.nl/donkeys-mules-horses/).
+mod cht;
+mod lsm_tree;
+mod prefix_cht;
+mod rib;
+mod tree_bitmap;
+mod types;
 
 #[macro_use]
 mod macros;
 
-pub(crate) mod types;
-
-// Public Interfaces
-
-/// Some simple metadata implementations
-pub mod meta_examples;
-
-pub(crate) mod cht;
-pub(crate) mod in_memory;
-pub(crate) mod persist;
-pub(crate) mod prefix_cht;
-mod tests;
+pub(crate) use lsm_tree::LsmTree;
+pub(crate) use tree_bitmap::TreeBitMap;
 
 // re-exports
 pub use crossbeam_epoch::{self as epoch, Guard};
 pub use inetnum::addr;
 
-pub mod rib;
+// Public Interfaces on the root of the crate
 
 pub use rib::starcast::StarCastRib;
 pub use rib::starcast_af::{
-    MemoryOnlyConfig, PersistHistoryConfig, PersistOnlyConfig,
-    WriteAheadConfig,
+    Config, MemoryOnlyConfig, PersistHistoryConfig, PersistOnlyConfig,
+    PersistStrategy, UpsertReport, WriteAheadConfig,
 };
+pub use types::af::AddressFamily;
 pub use types::af::IPv4;
 pub use types::af::IPv6;
 pub use types::af::IntoIpAddr;
@@ -45,10 +42,10 @@ pub use types::errors;
 pub use types::match_options::{
     IncludeHistory, MatchOptions, MatchType, QueryResult,
 };
-pub use types::prefix_id::RouteStatus;
+pub use types::meta_examples;
+pub use types::prefix_record::Meta;
 pub use types::prefix_record::PublicPrefixRecord as PrefixRecord;
 pub use types::prefix_record::PublicRecord as Record;
+pub use types::route_status::RouteStatus;
 pub use types::stats;
 pub use types::test_types;
-pub use types::AddressFamily;
-pub use types::Meta;
diff --git a/src/persist/lsm_tree.rs b/src/lsm_tree/mod.rs
similarity index 97%
rename from src/persist/lsm_tree.rs
rename to src/lsm_tree/mod.rs
index 8bdfa4d4..d2301623 100644
--- a/src/persist/lsm_tree.rs
+++ b/src/lsm_tree/mod.rs
@@ -13,36 +13,26 @@ use zerocopy::{
 };
 
 use crate::rib::Counters;
-use crate::types::prefix_record::PublicRecord;
 use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
-use crate::types::{AddressFamily, Meta};
+use crate::types::AddressFamily;
+use crate::types::PublicRecord;
 use crate::types::{PrefixId, RouteStatus};
-
-type ZeroCopyError<'a, T> = zerocopy::ConvertError<
-    zerocopy::AlignmentError<&'a [u8], T>,
-    zerocopy::SizeError<&'a [u8], T>,
-    zerocopy::ValidityError<&'a [u8], T>,
->;
-type ZeroCopyMutError<'a, T> = zerocopy::ConvertError<
-    zerocopy::AlignmentError<&'a mut [u8], T>,
-    zerocopy::SizeError<&'a mut [u8], T>,
-    zerocopy::ValidityError<&'a mut [u8], T>,
->;
+use crate::Meta;
 
 pub(crate) trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
     TryFromBytes + KnownLayout + IntoBytes + Unaligned + Immutable
 {
-    fn mut_from_bytes(
-        bytes: &mut [u8],
-    ) -> std::result::Result<&mut Self, ZeroCopyMutError<'_, Self>> {
-        Self::try_mut_from_bytes(bytes.as_mut_bytes())
-    }
+    // fn mut_from_bytes(
+    //     bytes: &mut [u8],
+    // ) -> std::result::Result<&mut Self, ZeroCopyMutError<'_, Self>> {
+    //     Self::try_mut_from_bytes(bytes.as_mut_bytes())
+    // }
 
-    fn from_bytes(
-        bytes: &[u8],
-    ) -> std::result::Result<&Self, ZeroCopyError<'_, Self>> {
-        Self::try_ref_from_bytes(bytes.as_bytes())
-    }
+    // fn from_bytes(
+    //     bytes: &[u8],
+    // ) -> std::result::Result<&Self, ZeroCopyError<'_, Self>> {
+    //     Self::try_ref_from_bytes(bytes.as_bytes())
+    // }
 
     fn header(bytes: &[u8]) -> &LongKey<AF> {
         LongKey::try_ref_from_bytes(bytes.as_bytes()).unwrap()
@@ -54,7 +44,7 @@ pub(crate) trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
         LongKey::try_mut_from_bytes(bytes.as_mut_bytes()).unwrap()
     }
 
-    fn short_key(bytes: &[u8]) -> &ShortKey<AF> {
+    fn _short_key(bytes: &[u8]) -> &ShortKey<AF> {
         trace!("short key from bytes {:?}", bytes);
         let s_b = &bytes[..(AF::BITS as usize / 8) + 6];
         trace!("short key {:?}", s_b);
@@ -141,7 +131,7 @@ impl<AF: AddressFamily> From<(PrefixId<AF>, u32, u64, RouteStatus)>
     }
 }
 
-pub struct PersistTree<
+pub struct LsmTree<
     AF: AddressFamily,
     K: KeySize<AF, KEY_SIZE>,
     // The size in bytes of the prefix in the persisted storage (disk), this
@@ -163,10 +153,10 @@ impl<
         K: KeySize<AF, KEY_SIZE>,
         // const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
-    > PersistTree<AF, K, KEY_SIZE>
+    > LsmTree<AF, K, KEY_SIZE>
 {
-    pub fn new(persist_path: &Path) -> PersistTree<AF, K, KEY_SIZE> {
-        PersistTree::<AF, K, KEY_SIZE> {
+    pub fn new(persist_path: &Path) -> LsmTree<AF, K, KEY_SIZE> {
+        LsmTree::<AF, K, KEY_SIZE> {
             tree: lsm_tree::Config::new(persist_path).open().unwrap(),
             counters: Counters::default(),
             _af: PhantomData,
@@ -181,7 +171,7 @@ impl<
     pub fn _remove(&self, key: &[u8]) {
         self.tree.remove_weak(key, 0);
         // the first byte of the prefix holds the length of the prefix.
-        self.counters.dec_prefixes_count(key[0]);
+        self.counters._dec_prefixes_count(key[0]);
     }
 
     pub fn get_records_for_prefix(
@@ -879,7 +869,7 @@ impl<
         K: KeySize<AF, KEY_SIZE>,
         // const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
-    > std::fmt::Debug for PersistTree<AF, K, KEY_SIZE>
+    > std::fmt::Debug for LsmTree<AF, K, KEY_SIZE>
 {
     fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         todo!()
diff --git a/src/persist/mod.rs b/src/persist/mod.rs
deleted file mode 100644
index 2b0475be..00000000
--- a/src/persist/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub(crate) mod lsm_tree;
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index d17e1a6c..ee88da3f 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -14,7 +14,8 @@ use crate::{
     cht::{bits_for_len, Cht, OnceBoxSlice, Value},
     rib::starcast_af::UpsertReport,
     types::{
-        errors::PrefixStoreError, AddressFamily, PrefixId, PublicRecord,
+        errors::PrefixStoreError, prefix_record::PublicRecord, AddressFamily,
+        PrefixId,
     },
     Meta,
 };
@@ -49,7 +50,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         }
     }
 
-    pub fn len(&self) -> usize {
+    pub fn _len(&self) -> usize {
         let c_map = Arc::clone(&self.0);
         let record_map = c_map.lock().unwrap();
         record_map.len()
@@ -181,7 +182,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
             .collect::<Vec<_>>()
     }
 
-    pub fn as_records(&self) -> Vec<PublicRecord<M>> {
+    pub fn _as_records(&self) -> Vec<PublicRecord<M>> {
         let c_map = Arc::clone(&self.0);
         let record_map = c_map.lock().unwrap();
         record_map
@@ -342,7 +343,7 @@ impl PathSelections {
         self.path_selection_muis.0
     }
 
-    pub fn backup(&self) -> Option<u32> {
+    pub fn _backup(&self) -> Option<u32> {
         self.path_selection_muis.1
     }
 }
@@ -365,7 +366,7 @@ pub struct StoredPrefix<AF: AddressFamily, M: Meta> {
     pub next_bucket: PrefixSet<AF, M>,
 }
 
-impl<AF: AddressFamily, M: crate::types::Meta> StoredPrefix<AF, M> {
+impl<AF: AddressFamily, M: Meta> StoredPrefix<AF, M> {
     pub(crate) fn new(pfx_id: PrefixId<AF>, level: u8) -> Self {
         // start calculation size of next set, it's dependent on the level
         // we're in.
diff --git a/src/prefix_cht/iterators.rs b/src/prefix_cht/iterators.rs
index 0026d5b0..4298c24c 100644
--- a/src/prefix_cht/iterators.rs
+++ b/src/prefix_cht/iterators.rs
@@ -2,14 +2,9 @@ use log::trace;
 use roaring::RoaringBitmap;
 
 use crate::{
-    in_memory::{
-        tree_bitmap_node::{
-            NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter,
-        },
-        TreeBitMap,
-    },
-    // prelude::multi::PrefixId,
+    tree_bitmap::{NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter},
     types::{AddressFamily, BitSpan, PrefixId},
+    TreeBitMap,
 };
 
 pub(crate) struct _MoreSpecificPrefixIter<
diff --git a/src/rib/mod.rs b/src/rib/mod.rs
index e982b458..d6a1db80 100644
--- a/src/rib/mod.rs
+++ b/src/rib/mod.rs
@@ -1,8 +1,7 @@
-pub mod starcast;
-pub mod starcast_af;
-pub mod starcast_af_query;
+pub(crate) mod starcast;
+pub(crate) mod starcast_af;
+pub(crate) mod starcast_af_query;
 
-pub use starcast::StarCastRib;
-pub use starcast::BIT_SPAN_SIZE;
-pub use starcast::STRIDE_SIZE;
-pub use starcast_af::Counters;
+pub(crate) use starcast::BIT_SPAN_SIZE;
+pub(crate) use starcast::STRIDE_SIZE;
+pub(crate) use starcast_af::Counters;
diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index b1dd2b1f..21ab4c4d 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -1,7 +1,5 @@
 use crossbeam_epoch::Guard;
 use inetnum::addr::Prefix;
-// use crate::prelude::multi::*;
-// use crate::prelude::*;
 use rand::prelude::*;
 
 use crate::{
@@ -683,9 +681,9 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     #[cfg(feature = "cli")]
     pub fn print_funky_stats(&self) {
         println!("\nStats for IPv4 multi-threaded store\n");
-        println!("{}", self.v4.in_memory_tree);
+        println!("{}", self.v4.tree_bitmap);
         println!("Stats for IPv6 multi-threaded store\n");
-        println!("{}", self.v6.in_memory_tree);
+        println!("{}", self.v6.tree_bitmap);
     }
 
     // The Store statistics.
diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index be629ca6..f494b127 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -1,4 +1,3 @@
-use super::super::persist::lsm_tree::PersistTree;
 use std::path::Path;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
@@ -9,15 +8,15 @@ use crossbeam_epoch::{self as epoch};
 use epoch::{Guard, Owned};
 use zerocopy::TryFromBytes;
 
-use crate::in_memory::tree_bitmap::TreeBitMap;
-use crate::persist::lsm_tree::LongKey;
 use crate::prefix_cht::cht::PrefixCht;
 use crate::stats::CreatedNodes;
 use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
 use crate::types::{PrefixId, RouteStatus};
-use crate::{types::errors::PrefixStoreError, types::PublicRecord};
-
-pub use crate::rib::starcast_af_query;
+use crate::TreeBitMap;
+use crate::{lsm_tree::LongKey, LsmTree};
+use crate::{
+    types::errors::PrefixStoreError, types::prefix_record::PublicRecord,
+};
 
 use crate::{IPv4, IPv6, Meta};
 
@@ -192,7 +191,7 @@ impl Counters {
         self.prefixes[len as usize].fetch_add(1, Ordering::Relaxed);
     }
 
-    pub fn dec_prefixes_count(&self, len: u8) {
+    pub fn _dec_prefixes_count(&self, len: u8) {
         self.prefixes[len as usize].fetch_sub(1, Ordering::Relaxed);
     }
 
@@ -326,9 +325,9 @@ pub(crate) struct StarCastAfRib<
     const KEY_SIZE: usize,
 > {
     pub config: C,
-    pub(crate) in_memory_tree: TreeBitMap<AF, N_ROOT_SIZE>,
+    pub(crate) tree_bitmap: TreeBitMap<AF, N_ROOT_SIZE>,
     pub(crate) prefix_cht: PrefixCht<AF, M, P_ROOT_SIZE>,
-    pub(crate) persist_tree: Option<PersistTree<AF, LongKey<AF>, KEY_SIZE>>,
+    pub(crate) persist_tree: Option<LsmTree<AF, LongKey<AF>, KEY_SIZE>>,
     pub counters: Counters,
 }
 
@@ -360,13 +359,13 @@ impl<
             _ => {
                 let persist_path = &config.persist_path().unwrap();
                 let pp_ref = &Path::new(persist_path);
-                Some(PersistTree::new(pp_ref))
+                Some(LsmTree::new(pp_ref))
             }
         };
 
         let store = StarCastAfRib {
             config,
-            in_memory_tree: TreeBitMap::<AF, N_ROOT_SIZE>::new()?,
+            tree_bitmap: TreeBitMap::<AF, N_ROOT_SIZE>::new()?,
             persist_tree,
             counters: Counters::default(),
             prefix_cht: PrefixCht::<AF, M, P_ROOT_SIZE>::init(),
@@ -383,7 +382,7 @@ impl<
     ) -> Result<UpsertReport, PrefixStoreError> {
         trace!("try insertingf {:?}", prefix);
         let guard = &epoch::pin();
-        self.in_memory_tree
+        self.tree_bitmap
             .set_prefix_exists(prefix, record.multi_uniq_id)
             .and_then(|(retry_count, exists)| {
                 trace!("exists, upsert it");
@@ -453,7 +452,7 @@ impl<
             PersistStrategy::PersistOnly => {
                 if let Some(persist_tree) = &self.persist_tree {
                     let (retry_count, exists) = self
-                        .in_memory_tree
+                        .tree_bitmap
                         .set_prefix_exists(prefix, record.multi_uniq_id)?;
                     persist_tree.persist_record_w_short_key(prefix, &record);
                     Ok(UpsertReport {
@@ -471,14 +470,14 @@ impl<
 
     pub fn contains(&self, prefix: PrefixId<AF>, mui: Option<u32>) -> bool {
         if let Some(mui) = mui {
-            self.in_memory_tree.prefix_exists_for_mui(prefix, mui)
+            self.tree_bitmap.prefix_exists_for_mui(prefix, mui)
         } else {
-            self.in_memory_tree.prefix_exists(prefix)
+            self.tree_bitmap.prefix_exists(prefix)
         }
     }
 
     pub fn get_nodes_count(&self) -> usize {
-        self.in_memory_tree.get_nodes_count()
+        self.tree_bitmap.get_nodes_count()
     }
 
     // Change the status of the record for the specified (prefix, mui)
@@ -621,7 +620,7 @@ impl<
         guard: &Guard,
     ) -> Result<(), PrefixStoreError> {
         let current = self
-            .in_memory_tree
+            .tree_bitmap
             .withdrawn_muis_bmin
             .load(Ordering::Acquire, guard);
 
@@ -629,7 +628,7 @@ impl<
         new.insert(mui);
 
         loop {
-            match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
+            match self.tree_bitmap.withdrawn_muis_bmin.compare_exchange(
                 current,
                 Owned::new(new),
                 Ordering::AcqRel,
@@ -653,7 +652,7 @@ impl<
         guard: &Guard,
     ) -> Result<(), PrefixStoreError> {
         let current = self
-            .in_memory_tree
+            .tree_bitmap
             .withdrawn_muis_bmin
             .load(Ordering::Acquire, guard);
 
@@ -661,7 +660,7 @@ impl<
         new.remove(mui);
 
         loop {
-            match self.in_memory_tree.withdrawn_muis_bmin.compare_exchange(
+            match self.tree_bitmap.withdrawn_muis_bmin.compare_exchange(
                 current,
                 Owned::new(new),
                 Ordering::AcqRel,
@@ -682,7 +681,7 @@ impl<
     // functions.
     pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
         unsafe {
-            self.in_memory_tree
+            self.tree_bitmap
                 .withdrawn_muis_bmin
                 .load(Ordering::Acquire, guard)
                 .as_ref()
@@ -696,7 +695,7 @@ impl<
     // functions.
     pub(crate) fn is_mui_active(&self, mui: u32, guard: &Guard) -> bool {
         !unsafe {
-            self.in_memory_tree
+            self.tree_bitmap
                 .withdrawn_muis_bmin
                 .load(Ordering::Acquire, guard)
                 .as_ref()
@@ -707,7 +706,7 @@ impl<
 
     pub(crate) fn get_prefixes_count(&self) -> UpsertCounters {
         UpsertCounters {
-            in_memory_count: self.in_memory_tree.get_prefixes_count(),
+            in_memory_count: self.tree_bitmap.get_prefixes_count(),
             persisted_count: self
                 .persist_tree
                 .as_ref()
@@ -718,9 +717,7 @@ impl<
 
     pub fn get_prefixes_count_for_len(&self, len: u8) -> UpsertCounters {
         UpsertCounters {
-            in_memory_count: self
-                .in_memory_tree
-                .get_prefixes_count_for_len(len),
+            in_memory_count: self.tree_bitmap.get_prefixes_count_for_len(len),
             persisted_count: self
                 .persist_tree
                 .as_ref()
@@ -733,7 +730,7 @@ impl<
         &'a self,
         guard: &'a Guard,
     ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
-        self.in_memory_tree.prefixes_iter().map(|p| {
+        self.tree_bitmap.prefixes_iter().map(|p| {
             (
                 p,
                 self.get_value(p.into(), None, true, guard)
diff --git a/src/rib/starcast_af_query.rs b/src/rib/starcast_af_query.rs
index a2ca5926..b1a8d896 100644
--- a/src/rib/starcast_af_query.rs
+++ b/src/rib/starcast_af_query.rs
@@ -5,8 +5,8 @@ use zerocopy::TryFromBytes;
 
 use crate::rib::starcast_af::{Config, PersistStrategy, StarCastAfRib};
 use crate::types::prefix_record::ZeroCopyRecord;
-use crate::types::AddressFamily;
 use crate::types::PublicRecord;
+use crate::AddressFamily;
 use inetnum::addr::Prefix;
 
 use crate::{Meta, QueryResult};
@@ -43,7 +43,7 @@ impl<
                         prefix_id,
                         mui,
                         include_withdrawn,
-                        self.in_memory_tree.withdrawn_muis_bmin(guard),
+                        self.tree_bitmap.withdrawn_muis_bmin(guard),
                     )
                     .map(|v| {
                         v.iter()
@@ -69,7 +69,7 @@ impl<
                 prefix_id,
                 mui,
                 include_withdrawn,
-                self.in_memory_tree.withdrawn_muis_bmin(guard),
+                self.tree_bitmap.withdrawn_muis_bmin(guard),
             ),
         }
     }
@@ -87,7 +87,7 @@ impl<
             None
         };
         let more_specifics = self
-            .in_memory_tree
+            .tree_bitmap
             .more_specific_prefix_iter_from(prefix_id)
             .map(|p| {
                 self.get_value(prefix_id, mui, include_withdrawn, guard)
@@ -123,7 +123,7 @@ impl<
         };
 
         let less_specifics = self
-            .in_memory_tree
+            .tree_bitmap
             .less_specific_prefix_iter(prefix_id)
             .map(|p| {
                 self.get_value(prefix_id, mui, include_withdrawn, guard)
@@ -158,7 +158,7 @@ impl<
             None
         } else {
             Some(
-                self.in_memory_tree
+                self.tree_bitmap
                     .more_specific_prefix_iter_from(prefix_id)
                     .filter_map(move |p| {
                         self.get_value(p, mui, include_withdrawn, guard)
@@ -199,7 +199,7 @@ impl<
         include_withdrawn: bool,
         guard: &'a Guard,
     ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-        self.in_memory_tree
+        self.tree_bitmap
             .less_specific_prefix_iter(prefix_id)
             .filter_map(move |p| {
                 self.get_value(p, mui, include_withdrawn, guard)
@@ -214,7 +214,7 @@ impl<
         guard: &'a Guard,
     ) -> QueryResult<M> {
         trace!("match_prefix rib {:?} {:?}", search_pfx, options);
-        let res = self.in_memory_tree.match_prefix(search_pfx, options);
+        let res = self.tree_bitmap.match_prefix(search_pfx, options);
 
         trace!("res {:?}", res);
         let mut res = QueryResult::from(res);
diff --git a/src/in_memory/atomic_bitmap.rs b/src/tree_bitmap/atomic_bitmap.rs
similarity index 98%
rename from src/in_memory/atomic_bitmap.rs
rename to src/tree_bitmap/atomic_bitmap.rs
index 1514f253..27071ca3 100644
--- a/src/in_memory/atomic_bitmap.rs
+++ b/src/tree_bitmap/atomic_bitmap.rs
@@ -2,7 +2,7 @@ use parking_lot_core::SpinWait;
 use std::fmt::{Binary, Debug};
 use std::sync::atomic::{fence, AtomicU16, AtomicU32, Ordering};
 
-use crate::types::bit_span::BitSpan;
+use crate::types::BitSpan;
 
 use super::tree_bitmap_node;
 
diff --git a/src/in_memory/tree_bitmap.rs b/src/tree_bitmap/mod.rs
similarity index 99%
rename from src/in_memory/tree_bitmap.rs
rename to src/tree_bitmap/mod.rs
index 0475dc37..e19df85a 100644
--- a/src/in_memory/tree_bitmap.rs
+++ b/src/tree_bitmap/mod.rs
@@ -1,3 +1,15 @@
+mod atomic_bitmap;
+mod node_cht;
+
+mod tree_bitmap_iterators;
+mod tree_bitmap_node;
+mod tree_bitmap_query;
+
+pub(crate) use tree_bitmap_node::{
+    NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter, StrideNodeId,
+    TreeBitMapNode,
+};
+
 // ----------- THE STORE ----------------------------------------------------
 //
 // The CustomAllocStore provides in-memory storage for the BitTreeMapNodes
@@ -184,25 +196,23 @@
 // produce a fix for it).
 
 use crate::cht::{bits_for_len, Cht};
-use crate::in_memory::node_cht::{NodeCht, NodeSet, StoredNode};
 use crate::rib::STRIDE_SIZE;
 use crate::types::{BitSpan, PrefixId};
 use crossbeam_epoch::{Atomic, Guard};
 use log::{debug, error, log_enabled, trace};
+use node_cht::{NodeCht, NodeSet, StoredNode};
 use roaring::RoaringBitmap;
+use tree_bitmap_node::NewNodeOrIndex;
 
 use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering};
 use std::{fmt::Debug, marker::PhantomData};
 
-use super::atomic_bitmap::{AtomicBitmap, AtomicPfxBitArr, AtomicPtrBitArr};
 use crate::rib::Counters;
 use crate::types::AddressFamily;
+use atomic_bitmap::{AtomicBitmap, AtomicPfxBitArr, AtomicPtrBitArr};
 
-use crate::in_memory::tree_bitmap_node::{NewNodeOrIndex, StrideNodeId};
 use crate::types::errors::PrefixStoreError;
 
-use super::tree_bitmap_node::TreeBitMapNode;
-
 #[cfg(feature = "cli")]
 use ansi_term::Colour;
 
diff --git a/src/in_memory/node_cht.rs b/src/tree_bitmap/node_cht.rs
similarity index 98%
rename from src/in_memory/node_cht.rs
rename to src/tree_bitmap/node_cht.rs
index edc01be9..d0942a51 100644
--- a/src/in_memory/node_cht.rs
+++ b/src/tree_bitmap/node_cht.rs
@@ -27,7 +27,7 @@ where
 }
 
 #[derive(Debug)]
-pub struct NodeSet<AF: AddressFamily>(
+pub(crate) struct NodeSet<AF: AddressFamily>(
     OnceBoxSlice<StoredNode<AF>>,
     // A Bitmap index that keeps track of the `multi_uniq_id`s (mui) that are
     // present in value collections in the meta-data tree in the child nodes
diff --git a/src/in_memory/tree_bitmap_iterators.rs b/src/tree_bitmap/tree_bitmap_iterators.rs
similarity index 99%
rename from src/in_memory/tree_bitmap_iterators.rs
rename to src/tree_bitmap/tree_bitmap_iterators.rs
index d10f6c30..7c204539 100644
--- a/src/in_memory/tree_bitmap_iterators.rs
+++ b/src/tree_bitmap/tree_bitmap_iterators.rs
@@ -19,9 +19,9 @@
 // contention, since every lookup has to go through the levels near the root
 // in the TreeBitMap.
 
-use crate::in_memory::TreeBitMap;
+use crate::TreeBitMap;
 use crate::{
-    in_memory::tree_bitmap_node::{
+    tree_bitmap::tree_bitmap_node::{
         NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter,
     },
     types::{AddressFamily, BitSpan, PrefixId},
diff --git a/src/in_memory/tree_bitmap_node.rs b/src/tree_bitmap/tree_bitmap_node.rs
similarity index 99%
rename from src/in_memory/tree_bitmap_node.rs
rename to src/tree_bitmap/tree_bitmap_node.rs
index 983d7f6b..50dee3d0 100644
--- a/src/in_memory/tree_bitmap_node.rs
+++ b/src/tree_bitmap/tree_bitmap_node.rs
@@ -4,7 +4,7 @@ use std::{fmt::Debug, marker::PhantomData};
 use log::{log_enabled, trace};
 use parking_lot_core::SpinWait;
 
-use crate::in_memory::atomic_bitmap::{
+use crate::tree_bitmap::atomic_bitmap::{
     AtomicBitmap, AtomicPfxBitArr, AtomicPtrBitArr, CasResult,
 };
 use crate::types::BitSpan;
@@ -28,7 +28,7 @@ use crate::types::PrefixId;
 // in a treebitmap node is enabled by the storage backend for the
 // multi-threaded store, since holds its entries keyed on the [node|prefix]
 // id. (in contrast with arrays or `vec`s, that have
-pub struct TreeBitMapNode<AF>
+pub(crate) struct TreeBitMapNode<AF>
 where
     Self: Sized,
     AF: AddressFamily,
diff --git a/src/in_memory/tree_bitmap_query.rs b/src/tree_bitmap/tree_bitmap_query.rs
similarity index 99%
rename from src/in_memory/tree_bitmap_query.rs
rename to src/tree_bitmap/tree_bitmap_query.rs
index a3240102..c78ae45d 100644
--- a/src/in_memory/tree_bitmap_query.rs
+++ b/src/tree_bitmap/tree_bitmap_query.rs
@@ -3,8 +3,8 @@ use crate::types::AddressFamily;
 use crate::rib::starcast_af_query::TreeQueryResult;
 use crate::{MatchOptions, MatchType};
 
-use super::TreeBitMap;
 use crate::types::PrefixId;
+use crate::TreeBitMap;
 
 impl<AF, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE>
 where
diff --git a/src/types/match_options.rs b/src/types/match_options.rs
index fa768e09..1ce4f60d 100644
--- a/src/types/match_options.rs
+++ b/src/types/match_options.rs
@@ -1,7 +1,7 @@
+use crate::types::{prefix_record::RecordSet, PublicRecord};
+use crate::Meta;
 use std::fmt;
 
-use crate::types::{prefix_record::RecordSet, Meta, PublicRecord};
-
 use inetnum::addr::Prefix;
 
 //------------ MatchOptions / MatchType -------------------------------------
diff --git a/src/meta_examples.rs b/src/types/meta_examples.rs
similarity index 100%
rename from src/meta_examples.rs
rename to src/types/meta_examples.rs
diff --git a/src/types/mod.rs b/src/types/mod.rs
index ebd7193b..3ce30fca 100644
--- a/src/types/mod.rs
+++ b/src/types/mod.rs
@@ -1,15 +1,19 @@
+mod bit_span;
+mod prefix_id;
+mod tests;
+
 pub(crate) mod af;
-pub(crate) mod bit_span;
 pub(crate) mod match_options;
-pub(crate) mod prefix_id;
 pub(crate) mod prefix_record;
+pub(crate) mod route_status;
 
-pub use af::AddressFamily;
+pub(crate) use af::AddressFamily;
 pub(crate) use bit_span::BitSpan;
-pub(crate) use prefix_id::{PrefixId, RouteStatus};
+pub(crate) use prefix_id::PrefixId;
+pub(crate) use prefix_record::PublicRecord;
+pub(crate) use route_status::RouteStatus;
 
-pub use prefix_record::Meta;
 pub mod errors;
+pub mod meta_examples;
 pub mod stats;
 pub mod test_types;
-pub(crate) use prefix_record::PublicRecord;
diff --git a/src/types/prefix_id.rs b/src/types/prefix_id.rs
index ce3e2e4b..a8f463a9 100644
--- a/src/types/prefix_id.rs
+++ b/src/types/prefix_id.rs
@@ -1,62 +1,7 @@
-use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes, Unaligned};
+use zerocopy::TryFromBytes;
 
 use crate::AddressFamily;
 
-use super::errors::PrefixStoreError;
-
-#[derive(
-    Clone,
-    Copy,
-    Debug,
-    Hash,
-    PartialEq,
-    Eq,
-    TryFromBytes,
-    KnownLayout,
-    Immutable,
-    Unaligned,
-    IntoBytes,
-)]
-#[repr(u8)]
-pub enum RouteStatus {
-    Active = 1,
-    InActive = 2,
-    Withdrawn = 3,
-}
-
-impl std::fmt::Display for RouteStatus {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            RouteStatus::Active => write!(f, "active"),
-            RouteStatus::InActive => write!(f, "inactive"),
-            RouteStatus::Withdrawn => write!(f, "withdrawn"),
-        }
-    }
-}
-
-impl From<RouteStatus> for u8 {
-    fn from(value: RouteStatus) -> Self {
-        match value {
-            RouteStatus::Active => 1,
-            RouteStatus::InActive => 2,
-            RouteStatus::Withdrawn => 3,
-        }
-    }
-}
-
-impl TryFrom<u8> for RouteStatus {
-    type Error = PrefixStoreError;
-
-    fn try_from(value: u8) -> Result<Self, Self::Error> {
-        match value {
-            1 => Ok(RouteStatus::Active),
-            2 => Ok(RouteStatus::InActive),
-            3 => Ok(RouteStatus::Withdrawn),
-            _ => Err(PrefixStoreError::StoreNotReadyError),
-        }
-    }
-}
-
 //------------ PrefixId ------------------------------------------------------
 #[derive(
     Hash,
@@ -122,7 +67,7 @@ impl<AF: AddressFamily> PrefixId<AF> {
     // The lsm tree, used for persistence, stores the prefix in the key with
     // len first, so that key range lookups can be made for more-specifics in
     // each prefix length.
-    pub fn to_len_first_bytes<const PREFIX_SIZE: usize>(
+    pub fn len_first_bytes<const PREFIX_SIZE: usize>(
         &self,
     ) -> [u8; PREFIX_SIZE] {
         let bytes = &mut [0_u8; PREFIX_SIZE];
diff --git a/src/types/prefix_record.rs b/src/types/prefix_record.rs
index 28789051..9d86b2f8 100644
--- a/src/types/prefix_record.rs
+++ b/src/types/prefix_record.rs
@@ -373,69 +373,69 @@ pub struct RecordSingleSet<M: Meta> {
     pub v6: Vec<PublicPrefixSingleRecord<M>>,
 }
 
-impl<M: Meta> RecordSingleSet<M> {
-    pub fn new() -> Self {
-        Self {
-            v4: Default::default(),
-            v6: Default::default(),
-        }
-    }
-
-    pub fn push(&mut self, prefix: Prefix, meta: M) {
-        match prefix.addr() {
-            std::net::IpAddr::V4(_) => &mut self.v4,
-            std::net::IpAddr::V6(_) => &mut self.v6,
-        }
-        .push(PublicPrefixSingleRecord::new(prefix, meta));
-    }
+// impl<M: Meta> RecordSingleSet<M> {
+//     pub fn new() -> Self {
+//         Self {
+//             v4: Default::default(),
+//             v6: Default::default(),
+//         }
+//     }
 
-    pub fn is_empty(&self) -> bool {
-        self.v4.is_empty() && self.v6.is_empty()
-    }
+//     pub fn push(&mut self, prefix: Prefix, meta: M) {
+//         match prefix.addr() {
+//             std::net::IpAddr::V4(_) => &mut self.v4,
+//             std::net::IpAddr::V6(_) => &mut self.v6,
+//         }
+//         .push(PublicPrefixSingleRecord::new(prefix, meta));
+//     }
 
-    pub fn iter(&self) -> RecordSetSingleIter<M> {
-        RecordSetSingleIter {
-            v4: if self.v4.is_empty() {
-                None
-            } else {
-                Some(self.v4.iter())
-            },
-            v6: self.v6.iter(),
-        }
-    }
+//     pub fn is_empty(&self) -> bool {
+//         self.v4.is_empty() && self.v6.is_empty()
+//     }
 
-    #[must_use]
-    pub fn reverse(mut self) -> RecordSingleSet<M> {
-        self.v4.reverse();
-        self.v6.reverse();
-        self
-    }
+//     pub fn iter(&self) -> RecordSetSingleIter<M> {
+//         RecordSetSingleIter {
+//             v4: if self.v4.is_empty() {
+//                 None
+//             } else {
+//                 Some(self.v4.iter())
+//             },
+//             v6: self.v6.iter(),
+//         }
+//     }
 
-    pub fn len(&self) -> usize {
-        self.v4.len() + self.v6.len()
-    }
-}
+//     #[must_use]
+//     pub fn reverse(mut self) -> RecordSingleSet<M> {
+//         self.v4.reverse();
+//         self.v6.reverse();
+//         self
+//     }
 
-impl<M: Meta> Default for RecordSingleSet<M> {
-    fn default() -> Self {
-        Self::new()
-    }
-}
+//     pub fn len(&self) -> usize {
+//         self.v4.len() + self.v6.len()
+//     }
+// }
 
-impl<M: Meta> fmt::Display for RecordSingleSet<M> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let arr_str_v4 =
-            self.v4.iter().fold("".to_string(), |pfx_arr, pfx| {
-                format!("{} {}", pfx_arr, *pfx)
-            });
-        let arr_str_v6 =
-            self.v6.iter().fold("".to_string(), |pfx_arr, pfx| {
-                format!("{} {}", pfx_arr, *pfx)
-            });
+// impl<M: Meta> Default for RecordSingleSet<M> {
+//     fn default() -> Self {
+//         Self::new()
+//     }
+// }
 
-        write!(f, "V4: [{}], V6: [{}]", arr_str_v4, arr_str_v6)
-    }
-}
+// impl<M: Meta> fmt::Display for RecordSingleSet<M> {
+//     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+//         let arr_str_v4 =
+//             self.v4.iter().fold("".to_string(), |pfx_arr, pfx| {
+//                 format!("{} {}", pfx_arr, *pfx)
+//             });
+//         let arr_str_v6 =
+//             self.v6.iter().fold("".to_string(), |pfx_arr, pfx| {
+//                 format!("{} {}", pfx_arr, *pfx)
+//             });
+
+//         write!(f, "V4: [{}], V6: [{}]", arr_str_v4, arr_str_v6)
+//     }
+// }
 
 impl<M: Meta>
     From<(
diff --git a/src/types/route_status.rs b/src/types/route_status.rs
new file mode 100644
index 00000000..c8453ee9
--- /dev/null
+++ b/src/types/route_status.rs
@@ -0,0 +1,56 @@
+use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes, Unaligned};
+
+use super::errors::PrefixStoreError;
+
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    Hash,
+    PartialEq,
+    Eq,
+    TryFromBytes,
+    KnownLayout,
+    Immutable,
+    Unaligned,
+    IntoBytes,
+)]
+#[repr(u8)]
+pub enum RouteStatus {
+    Active = 1,
+    InActive = 2,
+    Withdrawn = 3,
+}
+
+impl std::fmt::Display for RouteStatus {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RouteStatus::Active => write!(f, "active"),
+            RouteStatus::InActive => write!(f, "inactive"),
+            RouteStatus::Withdrawn => write!(f, "withdrawn"),
+        }
+    }
+}
+
+impl From<RouteStatus> for u8 {
+    fn from(value: RouteStatus) -> Self {
+        match value {
+            RouteStatus::Active => 1,
+            RouteStatus::InActive => 2,
+            RouteStatus::Withdrawn => 3,
+        }
+    }
+}
+
+impl TryFrom<u8> for RouteStatus {
+    type Error = PrefixStoreError;
+
+    fn try_from(value: u8) -> Result<Self, Self::Error> {
+        match value {
+            1 => Ok(RouteStatus::Active),
+            2 => Ok(RouteStatus::InActive),
+            3 => Ok(RouteStatus::Withdrawn),
+            _ => Err(PrefixStoreError::StoreNotReadyError),
+        }
+    }
+}
diff --git a/src/types/stats.rs b/src/types/stats.rs
index 97b370c0..1eaa4fec 100644
--- a/src/types/stats.rs
+++ b/src/types/stats.rs
@@ -7,8 +7,7 @@ use std::{
 };
 
 use crate::{
-    in_memory::tree_bitmap_node::TreeBitMapNode, rib::STRIDE_SIZE,
-    types::AddressFamily,
+    rib::STRIDE_SIZE, tree_bitmap::TreeBitMapNode, types::AddressFamily,
 };
 
 pub struct StrideStats<AF: AddressFamily> {
diff --git a/src/tests.rs b/src/types/tests.rs
similarity index 92%
rename from src/tests.rs
rename to src/types/tests.rs
index af0bc5f2..a070b4f7 100644
--- a/src/tests.rs
+++ b/src/types/tests.rs
@@ -5,7 +5,7 @@ use std::error::Error;
 
 #[test]
 fn test_af_1() -> Result<(), Box<dyn Error>> {
-    use crate::in_memory::tree_bitmap_node::StrideNodeId;
+    use crate::tree_bitmap::StrideNodeId;
     use crate::types::BitSpan;
     use crate::AddressFamily;
     use crate::IPv4;
@@ -35,7 +35,7 @@ fn test_af_1() -> Result<(), Box<dyn Error>> {
 #[test]
 fn test_af_2() -> Result<(), Box<dyn Error>> {
     use crate::IPv4;
-    use crate::{in_memory::tree_bitmap_node::StrideNodeId, types::BitSpan};
+    use crate::{tree_bitmap::StrideNodeId, types::BitSpan};
 
     let bit_addr: IPv4 = 0b1111_1111_1111_1111_1111_1111_1111_1111.into();
     let nu_prefix = StrideNodeId::dangerously_new_with_id_as_is(bit_addr, 8);
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index ef040ea1..c95ed148 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -2,9 +2,8 @@ use std::{str::FromStr, sync::atomic::Ordering};
 
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
-    meta_examples::NoMeta, rib::starcast_af::Config, test_types::BeBytesAsn,
-    IncludeHistory, MatchOptions, MemoryOnlyConfig, Record, RouteStatus,
-    StarCastRib,
+    meta_examples::NoMeta, test_types::BeBytesAsn, Config, IncludeHistory,
+    MatchOptions, MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
 };
 
 mod common {
diff --git a/tests/full-table.rs b/tests/full-table.rs
index 7f376dc5..87d230f4 100644
--- a/tests/full-table.rs
+++ b/tests/full-table.rs
@@ -3,7 +3,7 @@
 mod tests {
     use inetnum::addr::Prefix;
     use inetnum::asn::Asn;
-    use rotonda_store::rib::starcast_af::Config;
+    use rotonda_store::Config;
     use rotonda_store::{
         epoch, IncludeHistory, MatchOptions, MatchType, Meta, PrefixRecord,
         Record, RouteStatus, StarCastRib,
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index 90d6fce5..57e10054 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -4,8 +4,8 @@ use std::error::Error;
 
 use inetnum::addr::Prefix;
 use rotonda_store::{
-    meta_examples::PrefixAs, rib::starcast_af::Config, IncludeHistory,
-    MatchOptions, MatchType, Record, RouteStatus, StarCastRib,
+    meta_examples::PrefixAs, Config, IncludeHistory, MatchOptions, MatchType,
+    Record, RouteStatus, StarCastRib,
 };
 
 mod common {
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 9209bf0f..12a53099 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -1,8 +1,8 @@
 // type Prefix4<'a> = Prefix<u32, PrefixAs>;
 use inetnum::addr::Prefix;
 use rotonda_store::{
-    epoch, meta_examples::PrefixAs, rib::starcast_af::Config, IncludeHistory,
-    MatchOptions, MatchType, Record, RouteStatus, StarCastRib,
+    epoch, meta_examples::PrefixAs, Config, IncludeHistory, MatchOptions,
+    MatchType, Record, RouteStatus, StarCastRib,
 };
 
 use std::error::Error;
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 1920fb60..4d8f0933 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -18,8 +18,7 @@ mod tests {
     use rotonda_store::{
         epoch,
         meta_examples::{NoMeta, PrefixAs},
-        rib::starcast_af::Config,
-        IncludeHistory, IntoIpAddr, MatchOptions, MatchType, Record,
+        Config, IncludeHistory, IntoIpAddr, MatchOptions, MatchType, Record,
         RouteStatus, StarCastRib,
     };
 
diff --git a/tests/treebitmap_v6.rs b/tests/treebitmap_v6.rs
index 43839fe1..78f10b7d 100644
--- a/tests/treebitmap_v6.rs
+++ b/tests/treebitmap_v6.rs
@@ -15,8 +15,7 @@ mod tests {
     use rotonda_store::{
         epoch,
         meta_examples::{NoMeta, PrefixAs},
-        rib::starcast_af::Config,
-        IncludeHistory, IntoIpAddr, MatchOptions, MatchType,
+        Config, IncludeHistory, IntoIpAddr, MatchOptions, MatchType,
         MemoryOnlyConfig, PersistOnlyConfig, Record, RouteStatus,
         StarCastRib,
     };

From ff5d69b6093aa2fb8194ea23f287f1fb905dabb0 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 5 Mar 2025 16:51:36 +0100
Subject: [PATCH 111/147] writeahead -> persist historical

---
 Cargo.toml                 |  4 ++--
 src/rib/starcast_af.rs     | 14 ++++++++------
 src/types/match_options.rs |  7 +++++++
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index ad584beb..7afb0a0b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,7 +13,7 @@ rust-version.workspace = true
 license.workspace = true
 
 [workspace.package]
-version = "0.4.1"
+version = "0.5.0"
 edition = "2021"
 authors = ["NLnet Labs <routing-team@nlnetlabs.nl>"]
 license = "BSD-3-Clause"
@@ -35,7 +35,7 @@ clap        = { version = "4.4", optional = true, features = ["derive"] }
 rayon = { version = "1.10", optional = true }
 memmap2 = { version = "0.9", optional = true }
 rand = { version = "0.9" }
-lsm-tree = { version = "2.6.3" }
+lsm-tree = { version = "2.6.6" }
 serde = "1.0.216"
 serde_derive = "1.0.216"
 serde_json = "1.0.133"
diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index f494b127..68957359 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -29,15 +29,17 @@ use crate::AddressFamily;
 /// or persisted.
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum PersistStrategy {
-    /// Current records are stored both in-memory and persisted. Additionally
-    /// historical records are persisted.
+    /// Current records are stored both in-memory and persisted. Historical
+    /// records are persisted.
     WriteAhead,
-    /// Current records are stored in-memory, historical records are pesisted.
+    /// Current records are stored in-memory, historical records are
+    /// persisted.
     PersistHistory,
     /// Current records are stored in-memory, historical records are discarded
-    /// when nwer records appear.
+    /// when newer records appear.
     MemoryOnly,
-    /// Both current and historical records are persisted.
+    /// Current records are persisted immediately. No records are stored in
+    /// memory. Historical records are discarded when newer records appear.
     PersistOnly,
 }
 
@@ -417,7 +419,7 @@ impl<
         match self.config.persist_strategy() {
             PersistStrategy::WriteAhead => {
                 if let Some(persist_tree) = &self.persist_tree {
-                    persist_tree.persist_record_w_short_key(prefix, &record);
+                    persist_tree.persist_record_w_long_key(prefix, &record);
 
                     self.prefix_cht
                         .upsert_prefix(
diff --git a/src/types/match_options.rs b/src/types/match_options.rs
index 1ce4f60d..b10f860a 100644
--- a/src/types/match_options.rs
+++ b/src/types/match_options.rs
@@ -60,10 +60,17 @@ impl std::fmt::Display for MatchType {
     }
 }
 
+/// Match option to indicate that the result should return historical records,
+/// for the requested prefixes and the more- and less-specific prefixes. This
+/// option is ignored if the persist strategy config option is anythin other
+/// than `PersistHistory` or WriteAhead`.
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum IncludeHistory {
+    /// Do not return any historical records.
     None,
+    /// Return historical records for the requested prefix only.
     SearchPrefix,
+    /// Return historical records for all prefixes in the result.
     All,
 }
 

From c9bbaf44a49cdb28610d39de879f6edc4c1efdb6 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 6 Mar 2025 10:48:52 +0100
Subject: [PATCH 112/147] make all fields of TreeBitMap private

---
 src/rib/starcast_af.rs | 83 ++++++++++--------------------------------
 src/tree_bitmap/mod.rs | 64 +++++++++++++++++++++++++++-----
 2 files changed, 74 insertions(+), 73 deletions(-)

diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index 68957359..0a9e0bae 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -5,7 +5,7 @@ use inetnum::addr::Prefix;
 use log::{info, trace};
 
 use crossbeam_epoch::{self as epoch};
-use epoch::{Guard, Owned};
+use epoch::Guard;
 use zerocopy::TryFromBytes;
 
 use crate::prefix_cht::cht::PrefixCht;
@@ -621,29 +621,7 @@ impl<
         mui: u32,
         guard: &Guard,
     ) -> Result<(), PrefixStoreError> {
-        let current = self
-            .tree_bitmap
-            .withdrawn_muis_bmin
-            .load(Ordering::Acquire, guard);
-
-        let mut new = unsafe { current.as_ref() }.unwrap().clone();
-        new.insert(mui);
-
-        loop {
-            match self.tree_bitmap.withdrawn_muis_bmin.compare_exchange(
-                current,
-                Owned::new(new),
-                Ordering::AcqRel,
-                Ordering::Acquire,
-                guard,
-            ) {
-                Ok(_) => return Ok(()),
-                Err(updated) => {
-                    new =
-                        unsafe { updated.current.as_ref() }.unwrap().clone();
-                }
-            }
-        }
+        self.tree_bitmap.mark_mui_as_withdrawn(mui, guard)
     }
 
     // Change the status of the mui globally to Active. Iterators and match
@@ -653,57 +631,36 @@ impl<
         mui: u32,
         guard: &Guard,
     ) -> Result<(), PrefixStoreError> {
-        let current = self
-            .tree_bitmap
-            .withdrawn_muis_bmin
-            .load(Ordering::Acquire, guard);
-
-        let mut new = unsafe { current.as_ref() }.unwrap().clone();
-        new.remove(mui);
-
-        loop {
-            match self.tree_bitmap.withdrawn_muis_bmin.compare_exchange(
-                current,
-                Owned::new(new),
-                Ordering::AcqRel,
-                Ordering::Acquire,
-                guard,
-            ) {
-                Ok(_) => return Ok(()),
-                Err(updated) => {
-                    new =
-                        unsafe { updated.current.as_ref() }.unwrap().clone();
-                }
-            }
-        }
+        self.tree_bitmap.mark_mui_as_active(mui, guard)
     }
 
     // Whether this mui is globally withdrawn. Note that this overrules
     // (by default) any (prefix, mui) combination in iterators and match
     // functions.
     pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
-        unsafe {
-            self.tree_bitmap
-                .withdrawn_muis_bmin
-                .load(Ordering::Acquire, guard)
-                .as_ref()
-        }
-        .unwrap()
-        .contains(mui)
+        // unsafe {
+        self.tree_bitmap
+            .withdrawn_muis_bmin(guard)
+            // .load(Ordering::Acquire, guard)
+            // .as_ref()
+            // }
+            // .unwrap()
+            .contains(mui)
     }
 
     // Whether this mui is globally active. Note that the local statuses of
     // records (prefix, mui) may be set to withdrawn in iterators and match
     // functions.
     pub(crate) fn is_mui_active(&self, mui: u32, guard: &Guard) -> bool {
-        !unsafe {
-            self.tree_bitmap
-                .withdrawn_muis_bmin
-                .load(Ordering::Acquire, guard)
-                .as_ref()
-        }
-        .unwrap()
-        .contains(mui)
+        // !unsafe {
+        !self
+            .tree_bitmap
+            .withdrawn_muis_bmin(guard)
+            // .load(Ordering::Acquire, guard)
+            // .as_ref()
+            // }
+            // .unwrap()
+            .contains(mui)
     }
 
     pub(crate) fn get_prefixes_count(&self) -> UpsertCounters {
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index e19df85a..01bfdbe6 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -198,7 +198,7 @@ pub(crate) use tree_bitmap_node::{
 use crate::cht::{bits_for_len, Cht};
 use crate::rib::STRIDE_SIZE;
 use crate::types::{BitSpan, PrefixId};
-use crossbeam_epoch::{Atomic, Guard};
+use crossbeam_epoch::{Atomic, Guard, Owned, Shared};
 use log::{debug, error, log_enabled, trace};
 use node_cht::{NodeCht, NodeSet, StoredNode};
 use roaring::RoaringBitmap;
@@ -220,8 +220,8 @@ use ansi_term::Colour;
 
 #[derive(Debug)]
 pub struct TreeBitMap<AF: AddressFamily, const ROOT_SIZE: usize> {
-    pub(crate) node_cht: NodeCht<AF, ROOT_SIZE>,
-    pub(crate) withdrawn_muis_bmin: Atomic<RoaringBitmap>,
+    node_cht: NodeCht<AF, ROOT_SIZE>,
+    withdrawn_muis_bmin: Atomic<RoaringBitmap>,
     counters: Counters,
     default_route_exists: AtomicBool,
 }
@@ -491,35 +491,79 @@ Giving up this node. This shouldn't happen!",
         }
     }
 
+    pub fn mark_mui_as_active(
+        &self,
+        mui: u32,
+        guard: &Guard,
+    ) -> Result<(), PrefixStoreError> {
+        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
+
+        let mut new = unsafe { current.as_ref() }.unwrap().clone();
+        new.remove(mui);
+
+        self.update_withdrawn_muis_bmin(current, new, guard)
+    }
+
+    pub fn mark_mui_as_withdrawn(
+        &self,
+        mui: u32,
+        guard: &Guard,
+    ) -> Result<(), PrefixStoreError> {
+        let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
+
+        let mut new = unsafe { current.as_ref() }.unwrap().clone();
+        new.insert(mui);
+
+        self.update_withdrawn_muis_bmin(current, new, guard)
+    }
+
+    pub(crate) fn update_withdrawn_muis_bmin<'a>(
+        &self,
+        current: Shared<'a, RoaringBitmap>,
+        mut new: RoaringBitmap,
+        guard: &'a Guard,
+    ) -> Result<(), PrefixStoreError> {
+        loop {
+            match self.withdrawn_muis_bmin.compare_exchange(
+                current,
+                Owned::new(new),
+                Ordering::AcqRel,
+                Ordering::Acquire,
+                guard,
+            ) {
+                Ok(_) => return Ok(()),
+                Err(updated) => {
+                    new =
+                        unsafe { updated.current.as_ref() }.unwrap().clone();
+                }
+            }
+        }
+    }
+
     fn store_node(
         &self,
         id: StrideNodeId<AF>,
         multi_uniq_id: u32,
-        next_node: TreeBitMapNode<AF>,
+        new_node: TreeBitMapNode<AF>,
     ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
         if log_enabled!(log::Level::Trace) {
             debug!(
                 "{} store: Store node {}: {:?} mui {}",
                 std::thread::current().name().unwrap_or("unnamed-thread"),
                 id,
-                next_node,
+                new_node,
                 multi_uniq_id
             );
         }
         self.counters.inc_nodes_count();
 
         let mut nodes = self.node_cht.root_for_len(id.len());
-        let new_node = next_node;
         let mut level = 0;
         let mut retry_count = 0;
 
         loop {
             let this_level = bits_for_len(id.len(), level);
 
-            // if this_level > (id.len() / 4) {
-            //     return Err(PrefixStoreError::StoreNotReadyError);
-            // }
-
             trace!("{:032b}", id.len());
             trace!("id {:?}", id);
             trace!("multi_uniq_id {}", multi_uniq_id);

From c8453c1491273e35deed23a7edf2327538471615 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 6 Mar 2025 18:36:47 +0100
Subject: [PATCH 113/147] sort confusing init methods

---
 src/cht/mod.rs              |  5 +++--
 src/prefix_cht/cht.rs       | 19 ++++++++++---------
 src/tree_bitmap/mod.rs      | 22 +++++++++++++---------
 src/tree_bitmap/node_cht.rs | 18 +++++-------------
 4 files changed, 31 insertions(+), 33 deletions(-)

diff --git a/src/cht/mod.rs b/src/cht/mod.rs
index e24eeecf..4b1b864b 100644
--- a/src/cht/mod.rs
+++ b/src/cht/mod.rs
@@ -5,7 +5,8 @@ pub(crate) use oncebox::OnceBoxSlice;
 use crate::rib::STRIDE_SIZE;
 
 pub(crate) trait Value {
-    fn init(size: usize) -> Self;
+    fn init_with_p2_children(size: usize) -> Self;
+    fn init_leaf() -> Self;
 }
 
 #[derive(Debug)]
@@ -20,7 +21,7 @@ impl<V: Value, const ROOT_SIZE: usize, const STRIDES_PER_BUCKET: usize>
 {
     pub(crate) fn init() -> Self {
         Self(std::array::from_fn::<_, ROOT_SIZE, _>(|_| {
-            V::init(STRIDE_SIZE as usize)
+            V::init_with_p2_children(STRIDE_SIZE as usize)
         }))
     }
 
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index ee88da3f..364cf1b6 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -382,14 +382,18 @@ impl<AF: AddressFamily, M: Meta> StoredPrefix<AF, M> {
                 1 << (next_level - this_level),
                 pfx_id.get_len()
             );
-            PrefixSet::init(next_level.saturating_sub(this_level))
+            PrefixSet::init_with_p2_children(
+                next_level.saturating_sub(this_level) as usize,
+            )
         } else {
             debug!(
                 "{} store: INSERT at LAST LEVEL with empty bucket at prefix len {}",
                 std::thread::current().name().unwrap_or("unnamed-thread"),
                 pfx_id.get_len()
             );
-            PrefixSet::init(next_level.saturating_sub(this_level))
+            PrefixSet::init_with_p2_children(
+                next_level.saturating_sub(this_level) as usize,
+            )
         };
         // End of calculation
 
@@ -496,16 +500,13 @@ pub struct PrefixSet<AF: AddressFamily, M: Meta>(
     pub OnceBoxSlice<StoredPrefix<AF, M>>,
 );
 
-impl<AF: AddressFamily, M: Meta> PrefixSet<AF, M> {
-    pub fn init(p2_size: u8) -> Self {
-        PrefixSet(OnceBoxSlice::new(p2_size))
-    }
-}
-
 impl<AF: AddressFamily, M: Meta> Value for PrefixSet<AF, M> {
-    fn init(p2_size: usize) -> Self {
+    fn init_with_p2_children(p2_size: usize) -> Self {
         PrefixSet(OnceBoxSlice::new(p2_size as u8))
     }
+    fn init_leaf() -> Self {
+        PrefixSet(OnceBoxSlice::new(0))
+    }
 }
 
 #[derive(Debug)]
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 01bfdbe6..ffd9eaaf 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -195,7 +195,7 @@ pub(crate) use tree_bitmap_node::{
 // an actual memory leak in the mt-prefix-store (and even more if they can
 // produce a fix for it).
 
-use crate::cht::{bits_for_len, Cht};
+use crate::cht::{bits_for_len, Cht, Value};
 use crate::rib::STRIDE_SIZE;
 use crate::types::{BitSpan, PrefixId};
 use crossbeam_epoch::{Atomic, Guard, Owned, Shared};
@@ -601,8 +601,9 @@ lvl{}",
 
                     // A weird trick to create either a NodeSet with 16 nodes,
                     // or one without any (for the last stride)
-                    let node_set =
-                        NodeSet::init(next_level.saturating_sub(this_level));
+                    let node_set = NodeSet::init_with_p2_children(
+                        next_level.saturating_sub(this_level) as usize,
+                    );
 
                     let ptrbitarr = new_node.ptrbitarr.load();
                     let pfxbitarr = new_node.pfxbitarr.load();
@@ -730,8 +731,9 @@ lvl{}",
                 None => {
                     let this_level = bits_for_len(id.len(), level);
                     let next_level = bits_for_len(id.len(), level + 1);
-                    let node_set =
-                        NodeSet::init(next_level.saturating_sub(this_level));
+                    let node_set = NodeSet::init_with_p2_children(
+                        next_level.saturating_sub(this_level) as usize,
+                    );
 
                     // See if we can create the node
                     (node, _) =
@@ -810,8 +812,9 @@ lvl{}",
                 None => {
                     let this_level = bits_for_len(id.len(), level);
                     let next_level = bits_for_len(id.len(), level + 1);
-                    let node_set =
-                        NodeSet::init(next_level.saturating_sub(this_level));
+                    let node_set = NodeSet::init_with_p2_children(
+                        next_level.saturating_sub(this_level) as usize,
+                    );
 
                     // See if we can create the node
                     (node, _) =
@@ -888,8 +891,9 @@ lvl{}",
                 None => {
                     let this_level = bits_for_len(id.len(), level);
                     let next_level = bits_for_len(id.len(), level + 1);
-                    let node_set =
-                        NodeSet::init(next_level.saturating_sub(this_level));
+                    let node_set = NodeSet::init_with_p2_children(
+                        next_level.saturating_sub(this_level) as usize,
+                    );
 
                     // See if we can create the node
                     (node, _) =
diff --git a/src/tree_bitmap/node_cht.rs b/src/tree_bitmap/node_cht.rs
index d0942a51..9f5b18cb 100644
--- a/src/tree_bitmap/node_cht.rs
+++ b/src/tree_bitmap/node_cht.rs
@@ -35,18 +35,6 @@ pub(crate) struct NodeSet<AF: AddressFamily>(
 );
 
 impl<AF: AddressFamily> NodeSet<AF> {
-    pub(crate) fn init(p2_size: u8) -> Self {
-        if log_enabled!(log::Level::Debug) {
-            debug!(
-                "{} store: creating space for {} nodes",
-                std::thread::current().name().unwrap_or("unnamed-thread"),
-                1 << p2_size
-            );
-        }
-
-        NodeSet(OnceBoxSlice::new(p2_size), RoaringBitmap::new().into())
-    }
-
     pub(crate) fn rbm(&self) -> &RwLock<RoaringBitmap> {
         &self.1
     }
@@ -87,7 +75,7 @@ impl<AF: AddressFamily> NodeSet<AF> {
 }
 
 impl<AF: AddressFamily> Value for NodeSet<AF> {
-    fn init(p2_size: usize) -> Self {
+    fn init_with_p2_children(p2_size: usize) -> Self {
         if log_enabled!(log::Level::Debug) {
             debug!(
                 "{} store: creating space for {} nodes",
@@ -101,4 +89,8 @@ impl<AF: AddressFamily> Value for NodeSet<AF> {
             RoaringBitmap::new().into(),
         )
     }
+
+    fn init_leaf() -> Self {
+        NodeSet(OnceBoxSlice::new(0), RoaringBitmap::new().into())
+    }
 }

From 78bf6f12426ab6a411a45671c5137360c99f8ecf Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 6 Mar 2025 20:55:12 +0100
Subject: [PATCH 114/147] allow zero-children nodes

---
 src/cht/oncebox.rs          | 29 +++++++++++------------------
 src/prefix_cht/cht.rs       |  3 ++-
 src/tree_bitmap/node_cht.rs |  9 ++++-----
 3 files changed, 17 insertions(+), 24 deletions(-)

diff --git a/src/cht/oncebox.rs b/src/cht/oncebox.rs
index 891378dc..76c4d063 100644
--- a/src/cht/oncebox.rs
+++ b/src/cht/oncebox.rs
@@ -64,14 +64,14 @@ impl<T> Drop for OnceBox<T> {
 #[derive(Debug, Default)]
 pub(crate) struct OnceBoxSlice<T> {
     ptr: AtomicPtr<OnceBox<T>>,
-    p2_size: u8,
+    size: usize,
 }
 
 impl<T> OnceBoxSlice<T> {
-    pub fn new(p2_size: u8) -> Self {
+    pub fn new(size: usize) -> Self {
         Self {
             ptr: AtomicPtr::new(null_mut()),
-            p2_size,
+            size,
         }
     }
 
@@ -84,8 +84,7 @@ impl<T> OnceBoxSlice<T> {
         if ptr.is_null() {
             None
         } else {
-            let slice =
-                unsafe { slice::from_raw_parts(ptr, 1 << self.p2_size) };
+            let slice = unsafe { slice::from_raw_parts(ptr, self.size) };
             slice.get(idx).and_then(|inner| inner.get())
         }
     }
@@ -95,7 +94,7 @@ impl<T> OnceBoxSlice<T> {
         idx: usize,
         create: impl FnOnce() -> T,
     ) -> (&T, bool) {
-        // assert!(idx < (1 << self.p2_size));
+        // assert!(idx < self.p2_size);
         let slice = self.get_or_make_slice();
         slice[idx].get_or_init(create)
     }
@@ -103,12 +102,12 @@ impl<T> OnceBoxSlice<T> {
     fn get_or_make_slice(&self) -> &[OnceBox<T>] {
         let ptr = self.ptr.load(Ordering::Relaxed);
         if !ptr.is_null() {
-            return unsafe { slice::from_raw_parts(ptr, 1 << self.p2_size) };
+            return unsafe { slice::from_raw_parts(ptr, self.size) };
         }
 
         // Create a slice, set it, get again.
-        let mut vec = Vec::with_capacity(1 << self.p2_size);
-        for _ in 0..(1 << self.p2_size) {
+        let mut vec = Vec::with_capacity(self.size);
+        for _ in 0..(self.size) {
             vec.push(OnceBox::new())
         }
         // Convert Vec<[OnceBox<T>] -> Box<[OnceBox<T>] -> &mut [OnceBox<T>]
@@ -130,16 +129,13 @@ impl<T> OnceBoxSlice<T> {
                 // return current.
                 assert!(!current.is_null());
                 let _ = unsafe {
-                    Box::from_raw(slice::from_raw_parts_mut(
-                        ptr,
-                        1 << self.p2_size,
-                    ))
+                    Box::from_raw(slice::from_raw_parts_mut(ptr, self.size))
                 };
                 current
             }
         };
 
-        unsafe { slice::from_raw_parts(res, 1 << self.p2_size) }
+        unsafe { slice::from_raw_parts(res, self.size) }
     }
 }
 
@@ -148,10 +144,7 @@ impl<T> Drop for OnceBoxSlice<T> {
         let ptr = self.ptr.swap(null_mut(), Ordering::Relaxed);
         if !ptr.is_null() {
             let _ = unsafe {
-                Box::from_raw(slice::from_raw_parts_mut(
-                    ptr,
-                    1 << self.p2_size,
-                ))
+                Box::from_raw(slice::from_raw_parts_mut(ptr, self.size))
             };
         }
     }
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index 364cf1b6..ef8a5310 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -502,7 +502,8 @@ pub struct PrefixSet<AF: AddressFamily, M: Meta>(
 
 impl<AF: AddressFamily, M: Meta> Value for PrefixSet<AF, M> {
     fn init_with_p2_children(p2_size: usize) -> Self {
-        PrefixSet(OnceBoxSlice::new(p2_size as u8))
+        let size = if p2_size == 0 { 0 } else { 1 << p2_size };
+        PrefixSet(OnceBoxSlice::new(size))
     }
     fn init_leaf() -> Self {
         PrefixSet(OnceBoxSlice::new(0))
diff --git a/src/tree_bitmap/node_cht.rs b/src/tree_bitmap/node_cht.rs
index 9f5b18cb..98757e5e 100644
--- a/src/tree_bitmap/node_cht.rs
+++ b/src/tree_bitmap/node_cht.rs
@@ -80,14 +80,13 @@ impl<AF: AddressFamily> Value for NodeSet<AF> {
             debug!(
                 "{} store: creating space for {} nodes",
                 std::thread::current().name().unwrap_or("unnamed-thread"),
-                1 << p2_size
+                2_usize.pow(p2_size as u32)
             );
         }
 
-        NodeSet(
-            OnceBoxSlice::new(p2_size as u8),
-            RoaringBitmap::new().into(),
-        )
+        let size = if p2_size == 0 { 0 } else { 1 << p2_size };
+
+        NodeSet(OnceBoxSlice::new(size), RoaringBitmap::new().into())
     }
 
     fn init_leaf() -> Self {

From 4e70d594b2d97a599d2fc016e358ad16a7a409ef Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 6 Mar 2025 21:04:31 +0100
Subject: [PATCH 115/147] no node creation on retrieve_node{_for_mui}

---
 src/tree_bitmap/mod.rs | 78 ++----------------------------------------
 1 file changed, 2 insertions(+), 76 deletions(-)

diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index ffd9eaaf..52c0959b 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -810,49 +810,12 @@ lvl{}",
                 // thread running this method, so our thread enounters an
                 // empty node in the store.
                 None => {
-                    let this_level = bits_for_len(id.len(), level);
-                    let next_level = bits_for_len(id.len(), level + 1);
-                    let node_set = NodeSet::init_with_p2_children(
-                        next_level.saturating_sub(this_level) as usize,
-                    );
-
-                    // See if we can create the node
-                    (node, _) =
-                        nodes.read().get_or_init(index, || StoredNode {
-                            node_id: id,
-                            node: TreeBitMapNode::new(),
-                            node_set,
-                        });
-
-                    // We may have lost, and a different node than we
-                    // intended could live here, if so go a level deeper
-                    if id == node.node_id {
-                        // Nope, its ours or at least the node we need.
-
-                        return Some(&node.node);
-                    };
+                    return None;
                 }
                 Some(this_node) => {
                     node = this_node;
                     if id == this_node.node_id {
                         // YES, It's the one we're looking for!
-
-                        // Update the rbm_index in this node with the
-                        // multi_uniq_id that the caller specified. This
-                        // is the only atomic operation we need to do
-                        // here. The NodeSet that the index is attached
-                        // to, does not need to be written to, it's part
-                        // of a trie, so it just needs to "exist" (and it
-                        // already does).
-                        // let retry_count =
-                        //     this_node.node_set.update_rbm_index(mui).ok();
-
-                        //     trace!("Retry_count rbm index {:?}", retry_count);
-                        //     trace!(
-                        //     "add multi uniq id to bitmap index {} for node {}",
-                        //     mui,
-                        //     this_node.node
-                        // );
                         return Some(&this_node.node);
                     };
                 }
@@ -889,27 +852,7 @@ lvl{}",
                 // thread running this method, so our thread enounters an
                 // empty node in the store.
                 None => {
-                    let this_level = bits_for_len(id.len(), level);
-                    let next_level = bits_for_len(id.len(), level + 1);
-                    let node_set = NodeSet::init_with_p2_children(
-                        next_level.saturating_sub(this_level) as usize,
-                    );
-
-                    // See if we can create the node
-                    (node, _) =
-                        nodes.read().get_or_init(index, || StoredNode {
-                            node_id: id,
-                            node: TreeBitMapNode::new(),
-                            node_set,
-                        });
-
-                    // We may have lost, and a different node than we
-                    // intended could live here, if so go a level deeper
-                    if id == node.node_id {
-                        // Nope, its ours or at least the node we need.
-
-                        return Some(&node.node);
-                    };
+                    return None;
                 }
                 Some(this_node) => {
                     // node = this_node;
@@ -925,23 +868,6 @@ lvl{}",
 
                     if id == this_node.node_id {
                         // YES, It's the one we're looking for!
-
-                        // Update the rbm_index in this node with the
-                        // multi_uniq_id that the caller specified. This
-                        // is the only atomic operation we need to do
-                        // here. The NodeSet that the index is attached
-                        // to, does not need to be written to, it's part
-                        // of a trie, so it just needs to "exist" (and it
-                        // already does).
-                        // let retry_count =
-                        //     this_node.node_set.update_rbm_index(mui).ok();
-
-                        //     trace!("Retry_count rbm index {:?}", retry_count);
-                        //     trace!(
-                        //     "add multi uniq id to bitmap index {} for node {}",
-                        //     mui,
-                        //     this_node.node
-                        // );
                         return Some(&this_node.node);
                     };
                 }

From f29817236cb25fc0b6fe99d26912636dab33e23e Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 7 Mar 2025 09:52:59 +0100
Subject: [PATCH 116/147] nodeset_size fn instead of bits_for_len for lookups

---
 src/cht/mod.rs         |  5 +++++
 src/prefix_cht/cht.rs  | 17 +++++++---------
 src/tree_bitmap/mod.rs | 46 ++++++++++++++++--------------------------
 3 files changed, 29 insertions(+), 39 deletions(-)

diff --git a/src/cht/mod.rs b/src/cht/mod.rs
index 4b1b864b..30c81bc7 100644
--- a/src/cht/mod.rs
+++ b/src/cht/mod.rs
@@ -1,5 +1,6 @@
 mod oncebox;
 
+use num_traits::Saturating;
 pub(crate) use oncebox::OnceBoxSlice;
 
 use crate::rib::STRIDE_SIZE;
@@ -40,3 +41,7 @@ pub(crate) fn bits_for_len(len: u8, lvl: u8) -> u8 {
         len
     }
 }
+
+pub fn nodeset_size(len: u8, lvl: u8) -> u8 {
+    4_u8.saturating_sub((4 * (lvl + 1)).saturating_sub(len))
+}
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index ef8a5310..c0d82d95 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -9,6 +9,7 @@ use inetnum::addr::Prefix;
 use log::{debug, log_enabled, trace};
 use roaring::RoaringBitmap;
 
+use crate::cht::nodeset_size;
 use crate::RouteStatus;
 use crate::{
     cht::{bits_for_len, Cht, OnceBoxSlice, Value},
@@ -371,29 +372,25 @@ impl<AF: AddressFamily, M: Meta> StoredPrefix<AF, M> {
         // start calculation size of next set, it's dependent on the level
         // we're in.
         // let pfx_id = PrefixId::new(record.net, record.len);
-        let this_level = bits_for_len(pfx_id.get_len(), level);
-        let next_level = bits_for_len(pfx_id.get_len(), level + 1);
+        // let this_level = bits_for_len(pfx_id.get_len(), level);
+        let next_level = nodeset_size(pfx_id.get_len(), level + 1);
 
-        trace!("this level {} next level {}", this_level, next_level);
+        trace!("next level {}", next_level);
         let next_bucket: PrefixSet<AF, M> = if next_level > 0 {
             debug!(
                 "{} store: INSERT with new bucket of size {} at prefix len {}",
                 std::thread::current().name().unwrap_or("unnamed-thread"),
-                1 << (next_level - this_level),
+                1 << next_level,
                 pfx_id.get_len()
             );
-            PrefixSet::init_with_p2_children(
-                next_level.saturating_sub(this_level) as usize,
-            )
+            PrefixSet::init_with_p2_children(next_level as usize)
         } else {
             debug!(
                 "{} store: INSERT at LAST LEVEL with empty bucket at prefix len {}",
                 std::thread::current().name().unwrap_or("unnamed-thread"),
                 pfx_id.get_len()
             );
-            PrefixSet::init_with_p2_children(
-                next_level.saturating_sub(this_level) as usize,
-            )
+            PrefixSet::init_with_p2_children(next_level as usize)
         };
         // End of calculation
 
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 52c0959b..06db196a 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -195,7 +195,7 @@ pub(crate) use tree_bitmap_node::{
 // an actual memory leak in the mt-prefix-store (and even more if they can
 // produce a fix for it).
 
-use crate::cht::{bits_for_len, Cht, Value};
+use crate::cht::{bits_for_len, nodeset_size, Cht, Value};
 use crate::rib::STRIDE_SIZE;
 use crate::types::{BitSpan, PrefixId};
 use crossbeam_epoch::{Atomic, Guard, Owned, Shared};
@@ -562,7 +562,7 @@ Giving up this node. This shouldn't happen!",
         let mut retry_count = 0;
 
         loop {
-            let this_level = bits_for_len(id.len(), level);
+            // let this_level = bits_for_len(id.len(), level);
 
             trace!("{:032b}", id.len());
             trace!("id {:?}", id);
@@ -574,35 +574,27 @@ Giving up this node. This shouldn't happen!",
             match nodes.read().get(index) {
                 None => {
                     // No node exists, so we create one here.
-                    let next_level = bits_for_len(id.len(), level + 1);
+                    let next_level = nodeset_size(id.len(), level + 1);
 
                     if log_enabled!(log::Level::Trace) {
                         trace!(
-                            "Empty node found,creating new node {} len{}
-lvl{}",
+                "Empty node found,creating new node {} len{} vl{}",
                             id,
                             id.len(),
                             level + 1
                         );
                         trace!("Next level {}", next_level);
-                        trace!(
-                            "Creating space for {} nodes",
-                            if next_level >= this_level {
-                                1 << (next_level - this_level)
-                            } else {
-                                1
-                            }
-                        );
+                        trace!("Creating space for {} nodes", next_level);
                     }
 
                     trace!("multi uniq id {}", multi_uniq_id);
-                    trace!("this level {}", this_level);
                     trace!("next level {}", next_level);
 
                     // A weird trick to create either a NodeSet with 16 nodes,
                     // or one without any (for the last stride)
                     let node_set = NodeSet::init_with_p2_children(
-                        next_level.saturating_sub(this_level) as usize,
+                        // next_level.saturating_sub(this_level) as usize,
+                        next_level as usize,
                     );
 
                     let ptrbitarr = new_node.ptrbitarr.load();
@@ -639,8 +631,7 @@ lvl{}",
 
                     if log_enabled!(log::Level::Trace) {
                         trace!(
-                            "
-                                {} store: Node here exists {:?}",
+                            "{} store: Node here exists {:?}",
                             std::thread::current()
                                 .name()
                                 .unwrap_or("unnamed-thread"),
@@ -681,9 +672,7 @@ lvl{}",
                         // call to create it.
                         level += 1;
                         trace!(
-                            "Collision with node_id {},
-                                 move to next level: {} len{} next_lvl{}
-                                 index {}",
+"Collision with node_id {}, move to next level: {} len{} next_lvl{} index {}",
                             stored_node.node_id,
                             id,
                             id.len(),
@@ -691,15 +680,14 @@ lvl{}",
                             index
                         );
 
-                        match bits_for_len(id.len(), level) {
+                        match nodeset_size(id.len(), level) {
                             // on to the next level!
                             next_bit_shift if next_bit_shift > 0 => {
                                 nodes = &stored_node.node_set;
                             }
                             // There's no next level!
                             _ => panic!(
-                                "out of storage levels,
-                                    current level is {}",
+                                "out of storage levels, current level is {}",
                                 level
                             ),
                         }
@@ -729,10 +717,10 @@ lvl{}",
                 // thread running this method, so our thread enounters an
                 // empty node in the store.
                 None => {
-                    let this_level = bits_for_len(id.len(), level);
-                    let next_level = bits_for_len(id.len(), level + 1);
+                    // let this_level = bits_for_len(id.len(), level);
+                    let next_level = nodeset_size(id.len(), level + 1);
                     let node_set = NodeSet::init_with_p2_children(
-                        next_level.saturating_sub(this_level) as usize,
+                        next_level as usize, // next_level.saturating_sub(this_level) as usize,
                     );
 
                     // See if we can create the node
@@ -780,7 +768,7 @@ lvl{}",
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match bits_for_len(id.len(), level) {
+            match nodeset_size(id.len(), level) {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -822,7 +810,7 @@ lvl{}",
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match bits_for_len(id.len(), level) {
+            match nodeset_size(id.len(), level) {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;
@@ -874,7 +862,7 @@ lvl{}",
             }
             // It isn't ours. Move one level deeper.
             level += 1;
-            match bits_for_len(id.len(), level) {
+            match nodeset_size(id.len(), level) {
                 // on to the next level!
                 next_bit_shift if next_bit_shift > 0 => {
                     nodes = &node.node_set;

From cf27364098db0d68aa11eaf3b4b47f92ec362bc3 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Fri, 7 Mar 2025 14:29:49 +0100
Subject: [PATCH 117/147] documentation for nodeset_size

---
 src/cht/mod.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/src/cht/mod.rs b/src/cht/mod.rs
index 30c81bc7..b8b550b6 100644
--- a/src/cht/mod.rs
+++ b/src/cht/mod.rs
@@ -42,6 +42,65 @@ pub(crate) fn bits_for_len(len: u8, lvl: u8) -> u8 {
     }
 }
 
+// This output of this function is exactly same as this (for all values of len
+// and lvl we care for at least):
+//
+// let res = 4 * (lvl + 1);
+// if res < len {
+//     4
+// } else if res >= len + 4 {
+//     0
+// } else if len % 4 == 0 {
+//     4
+// } else {
+//     len % 4
+// }
+//
+// The gist of this function is that, we want exactly the numnber of slots in
+// our NodeSet that we can fill. This means:
+// - for any len smaller than STRIDE_SIZE (4), we only have one level, and
+//   that level takes `len` slots. There are no next levels in in these lens.
+// - len of STRIDE_SIZE and bigger have as many levels as can fit full
+//   STRIDE_SIZES, so with len 4, that is still one level (lvl = 0), for
+//   len 5 that is two levels, one of size 4 (lvl = 0), and one of size 1
+//   (lvl = 1). From len 9 there's three levels and so on.
+// - The first len, level combination beyond the max. size of lvl should
+//   return a 0, so the looper knows that it has to go to the next len.
+//
+// This is the output of the first values of len, lvl
+//
+// len, lvl : input parameters
+// ts       : total size of id
+// ns       : number of child slots for the NodeSet for this len, lvl
+//
+// len  lvl  ts ns
+// 00   00   00 0
+// 01   00   01 1
+// 01   01   00 0
+// 02   00   02 2
+// 02   01   00 0
+// 03   00   03 3
+// 03   01   00 0
+// 04   00   04 4
+// 04   01   00 0
+// 05   00   04 4
+// 05   01   05 1
+// 05   02   00 0
+// 06   00   04 4
+// 06   01   06 2
+// 06   02   00 0
+// 07   00   04 4
+// 07   01   07 3
+// 07   02   00 0
+// 08   00   04 4
+// 08   01   08 4
+// 08   02   00 0
+// 09   00   04 4
+// 09   01   08 4
+// 09   02   09 1
+// 09   03   00 0
+// 10   00   04 4
+// ...
 pub fn nodeset_size(len: u8, lvl: u8) -> u8 {
     4_u8.saturating_sub((4 * (lvl + 1)).saturating_sub(len))
 }

From dcc55e89b92b2f3e6393494ab56d9fe4447ae505 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Mon, 10 Mar 2025 19:42:03 +0100
Subject: [PATCH 118/147] last part for bits_for_len replacement

---
 src/cht/mod.rs              | 47 +++++++++++++++++++++++++++----------
 src/prefix_cht/cht.rs       | 24 ++++++++-----------
 src/tree_bitmap/mod.rs      | 19 +++++++--------
 src/tree_bitmap/node_cht.rs |  4 ----
 4 files changed, 53 insertions(+), 41 deletions(-)

diff --git a/src/cht/mod.rs b/src/cht/mod.rs
index b8b550b6..930075c7 100644
--- a/src/cht/mod.rs
+++ b/src/cht/mod.rs
@@ -1,13 +1,11 @@
 mod oncebox;
 
-use num_traits::Saturating;
 pub(crate) use oncebox::OnceBoxSlice;
 
 use crate::rib::STRIDE_SIZE;
 
 pub(crate) trait Value {
     fn init_with_p2_children(size: usize) -> Self;
-    fn init_leaf() -> Self;
 }
 
 #[derive(Debug)]
@@ -31,17 +29,6 @@ impl<V: Value, const ROOT_SIZE: usize, const STRIDES_PER_BUCKET: usize>
     }
 }
 
-pub(crate) fn bits_for_len(len: u8, lvl: u8) -> u8 {
-    let res = STRIDE_SIZE * (lvl + 1);
-    if res < len {
-        res
-    } else if res >= len + STRIDE_SIZE {
-        0
-    } else {
-        len
-    }
-}
-
 // This output of this function is exactly same as this (for all values of len
 // and lvl we care for at least):
 //
@@ -104,3 +91,37 @@ pub(crate) fn bits_for_len(len: u8, lvl: u8) -> u8 {
 pub fn nodeset_size(len: u8, lvl: u8) -> u8 {
     4_u8.saturating_sub((4 * (lvl + 1)).saturating_sub(len))
 }
+
+// The value of the set of the parent of this one. used to calculate the shift
+// offset in the hash for the CHT, so this is basically the `nodeset_size`
+// shifted on (len, lvl) combination downwards.
+//
+// len lvl prev
+// 00  00  00
+// 01  00  00
+// 01  01  01
+// 02  00  00
+// 02  01  02
+// 03  00  00
+// 03  01  03
+// 04  00  00
+// 04  01  04
+// 05  00  00
+// 05  01  04
+// 05  02  05
+// 06  00  00
+// 06  01  04
+// 06  02  06
+// 07  00  00
+// 07  01  04
+// 07  02  07
+// 08  00  00
+// 08  01  04
+// 08  02  08
+// 09  00  00
+// 09  01  04
+// 09  02  08
+// 09  03  09
+pub fn prev_node_size(len: u8, lvl: u8) -> u8 {
+    (lvl * 4) - lvl.saturating_sub(len >> 2) * ((lvl * 4) - len)
+}
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index c0d82d95..c28d84c4 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -9,10 +9,10 @@ use inetnum::addr::Prefix;
 use log::{debug, log_enabled, trace};
 use roaring::RoaringBitmap;
 
-use crate::cht::nodeset_size;
+use crate::cht::{nodeset_size, prev_node_size};
 use crate::RouteStatus;
 use crate::{
-    cht::{bits_for_len, Cht, OnceBoxSlice, Value},
+    cht::{Cht, OnceBoxSlice, Value},
     rib::starcast_af::UpsertReport,
     types::{
         errors::PrefixStoreError, prefix_record::PublicRecord, AddressFamily,
@@ -502,9 +502,6 @@ impl<AF: AddressFamily, M: Meta> Value for PrefixSet<AF, M> {
         let size = if p2_size == 0 { 0 } else { 1 << p2_size };
         PrefixSet(OnceBoxSlice::new(size))
     }
-    fn init_leaf() -> Self {
-        PrefixSet(OnceBoxSlice::new(0))
-    }
 }
 
 #[derive(Debug)]
@@ -788,12 +785,12 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
 
     pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
         // And, this is all of our hashing function.
-        let last_level = if level > 0 {
-            bits_for_len(id.get_len(), level - 1)
-        } else {
-            0
-        };
-        let this_level = bits_for_len(id.get_len(), level);
+        // let last_level = if level > 0 {
+        //     bits_for_len(id.get_len(), level.saturating_sub(1))
+        // } else {
+        //     0
+        // };
+        let last_level = prev_node_size(id.get_len(), level);
         // trace!(
         //     "bits division {}; no of bits {}",
         //     this_level,
@@ -806,10 +803,9 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
         //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
         // );
         // HASHING FUNCTION
+        let size = nodeset_size(id.get_len(), level);
         ((id.get_net() << AF::from_u32(last_level as u32))
-            >> AF::from_u8(
-                (<AF>::BITS - (this_level - last_level)) % <AF>::BITS,
-            ))
+            >> AF::from_u8((<AF>::BITS - size) % <AF>::BITS))
         .dangerously_truncate_to_u32() as usize
     }
 }
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 06db196a..6a14fb37 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -195,7 +195,7 @@ pub(crate) use tree_bitmap_node::{
 // an actual memory leak in the mt-prefix-store (and even more if they can
 // produce a fix for it).
 
-use crate::cht::{bits_for_len, nodeset_size, Cht, Value};
+use crate::cht::{bits_for_len, nodeset_size, prev_node_size, Cht, Value};
 use crate::rib::STRIDE_SIZE;
 use crate::types::{BitSpan, PrefixId};
 use crossbeam_epoch::{Atomic, Guard, Owned, Shared};
@@ -988,12 +988,12 @@ Giving up this node. This shouldn't happen!",
 
     pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
         // And, this is all of our hashing function.
-        let last_level = if level > 0 {
-            bits_for_len(id.len(), level - 1)
-        } else {
-            0
-        };
-        let this_level = bits_for_len(id.len(), level);
+        // let last_level = if level > 0 {
+        //     bits_for_len(id.len(), level - 1)
+        // } else {
+        //     0
+        // };
+        let last_level = prev_node_size(id.len(), level);
         // trace!("bits division {}", this_level);
         // trace!(
         //     "calculated index ({} << {}) >> {}",
@@ -1002,10 +1002,9 @@ Giving up this node. This shouldn't happen!",
         //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
         // );
         // HASHING FUNCTION
+        let size = nodeset_size(id.len(), level);
         ((id.bits() << AF::from_u8(last_level))
-            >> AF::from_u8(
-                (<AF>::BITS - (this_level - last_level)) % <AF>::BITS,
-            ))
+            >> AF::from_u8((<AF>::BITS - size) % <AF>::BITS))
         .dangerously_truncate_to_u32() as usize
     }
 }
diff --git a/src/tree_bitmap/node_cht.rs b/src/tree_bitmap/node_cht.rs
index 98757e5e..7c64de74 100644
--- a/src/tree_bitmap/node_cht.rs
+++ b/src/tree_bitmap/node_cht.rs
@@ -88,8 +88,4 @@ impl<AF: AddressFamily> Value for NodeSet<AF> {
 
         NodeSet(OnceBoxSlice::new(size), RoaringBitmap::new().into())
     }
-
-    fn init_leaf() -> Self {
-        NodeSet(OnceBoxSlice::new(0), RoaringBitmap::new().into())
-    }
 }

From 0110ad335ebe583fa7d85204b206f139929a473a Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 12 Mar 2025 12:23:26 +0100
Subject: [PATCH 119/147] remove non-existing import

---
 src/lib.rs             | 19 +++++++++----------
 src/tree_bitmap/mod.rs |  2 +-
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index e3e1218f..6a3cf622 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,15 +1,10 @@
-//! A treebitmap based IP Prefix Store
-
-//! IP prefixes storage and retrieval data structures for IPv4 and IPv6 prefixes.
-//! This crate contains structures for both single and multi-threaded contexts, as
-//! well as async contexts.
+//! A library that provides abstractions for a BGP Routing Information Base (RIB) for different AFI/SAFI types, as a database.
 //!
-//! The underlying tree structure is based on the tree bitmap as outlined in
-//! [this paper](https://www.cs.cornell.edu/courses/cs419/2005sp/tree-bitmap.pdf).
+//! The data structures provides by this crate can be used to store and query routes (or other metadata keyed on IP prefixes, or a comparable bitarray) in memory and on-disk, for both current and historical data.
 //!
-//! Part of the Rotonda modular BGP engine.
+//! [^1]:[Paper](https://www.cs.cornell.edu/courses/cs419/2005sp/tree-bitmap.pdf).
+//! [^2]: Read more about the data-structure in this [blogpost](https://blog.nlnetlabs.nl/donkeys-mules-horses/).
 
-//! Read more about the data-structure in this [blog post](https://blog.nlnetlabs.nl/donkeys-mules-horses/).
 mod cht;
 mod lsm_tree;
 mod prefix_cht;
@@ -42,10 +37,14 @@ pub use types::errors;
 pub use types::match_options::{
     IncludeHistory, MatchOptions, MatchType, QueryResult,
 };
+
+#[doc(hidden)]
 pub use types::meta_examples;
+#[doc(hidden)]
+pub use types::test_types;
+
 pub use types::prefix_record::Meta;
 pub use types::prefix_record::PublicPrefixRecord as PrefixRecord;
 pub use types::prefix_record::PublicRecord as Record;
 pub use types::route_status::RouteStatus;
 pub use types::stats;
-pub use types::test_types;
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 6a14fb37..6ce05c0e 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -195,7 +195,7 @@ pub(crate) use tree_bitmap_node::{
 // an actual memory leak in the mt-prefix-store (and even more if they can
 // produce a fix for it).
 
-use crate::cht::{bits_for_len, nodeset_size, prev_node_size, Cht, Value};
+use crate::cht::{nodeset_size, prev_node_size, Cht, Value};
 use crate::rib::STRIDE_SIZE;
 use crate::types::{BitSpan, PrefixId};
 use crossbeam_epoch::{Atomic, Guard, Owned, Shared};

From 11ce0ed3f601154089dcfb1761265f8aa74b865a Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 12 Mar 2025 15:58:32 +0100
Subject: [PATCH 120/147] saner pub modules and top level items

---
 src/types/af.rs            |   4 +-
 src/types/match_options.rs |   7 +-
 src/types/meta_examples.rs | 142 ----------
 src/types/mod.rs           |   7 +-
 src/types/prefix_record.rs | 566 ++++---------------------------------
 src/types/stats.rs         | 166 ++++++++++-
 src/types/test_types.rs    |  84 +++++-
 7 files changed, 304 insertions(+), 672 deletions(-)
 delete mode 100644 src/types/meta_examples.rs

diff --git a/src/types/af.rs b/src/types/af.rs
index f3875472..ac43108d 100644
--- a/src/types/af.rs
+++ b/src/types/af.rs
@@ -6,8 +6,8 @@ use crate::types::BitSpan;
 //------------ AddressFamily (trait) ----------------------------------------
 /// The address family of an IP address as a Trait.
 ///
-/// The idea of this trait is that each family will have a separate type to
-/// be able to only take the amount of memory needs. Useful when building
+/// The idea of this trait is that each family will have a separate type to be
+/// able to only take the exact amount of memory needed. Useful when building
 /// trees with large amounts of addresses/prefixes. Used by rotonda-store for
 /// this purpose.
 pub trait AddressFamily:
diff --git a/src/types/match_options.rs b/src/types/match_options.rs
index b10f860a..f9a52ce9 100644
--- a/src/types/match_options.rs
+++ b/src/types/match_options.rs
@@ -1,9 +1,10 @@
-use crate::types::{prefix_record::RecordSet, PublicRecord};
-use crate::Meta;
+use crate::types::{prefix_record::RecordSet, Record};
 use std::fmt;
 
 use inetnum::addr::Prefix;
 
+use super::prefix_record::Meta;
+
 //------------ MatchOptions / MatchType -------------------------------------
 
 /// Options for the `match_prefix` method
@@ -90,7 +91,7 @@ pub struct QueryResult<M: Meta> {
     /// The resulting prefix record
     pub prefix: Option<Prefix>,
     /// The meta data associated with the resulting prefix record
-    pub prefix_meta: Vec<PublicRecord<M>>,
+    pub prefix_meta: Vec<Record<M>>,
     /// The less-specifics of the resulting prefix together with their meta
     /// data
     pub less_specifics: Option<RecordSet<M>>,
diff --git a/src/types/meta_examples.rs b/src/types/meta_examples.rs
deleted file mode 100644
index 4cc482ba..00000000
--- a/src/types/meta_examples.rs
+++ /dev/null
@@ -1,142 +0,0 @@
-//------------ PrefixAs Metadata impl ---------------------------------------
-
-use inetnum::asn::Asn;
-
-use crate::Meta;
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
-pub struct PrefixAs([u8; 4]);
-
-impl PrefixAs {
-    pub fn new(asn: Asn) -> Self {
-        PrefixAs(u32::from_be_bytes(asn.to_raw()).to_le_bytes())
-    }
-
-    pub fn new_from_u32(value: u32) -> Self {
-        PrefixAs(value.to_le_bytes())
-    }
-
-    pub fn asn(&self) -> Asn {
-        Asn::from_u32(u32::from_le_bytes(self.0))
-    }
-}
-
-// impl MergeUpdate for PrefixAs {
-//     type UserDataIn = ();
-//     type UserDataOut = ();
-
-//     fn merge_update(
-//         &mut self,
-//         update_record: PrefixAs,
-//         _: Option<&Self::UserDataIn>,
-//     ) -> Result<(), Box<dyn std::error::Error>> {
-//         self.0 = update_record.0;
-//         Ok(())
-//     }
-
-//     fn clone_merge_update(
-//         &self,
-//         update_meta: &Self,
-//         _: Option<&Self::UserDataIn>,
-//     ) -> Result<(Self, Self::UserDataOut), Box<dyn std::error::Error>>
-//     where
-//         Self: std::marker::Sized,
-//     {
-//         Ok((PrefixAs(update_meta.0), ()))
-//     }
-// }
-
-impl Meta for PrefixAs {
-    type Orderable<'a> = Asn;
-    type TBI = ();
-    fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
-        u32::from_le_bytes(self.0).into()
-    }
-}
-
-impl AsRef<[u8]> for PrefixAs {
-    fn as_ref(&self) -> &[u8] {
-        self.0.as_ref()
-    }
-}
-
-impl From<Vec<u8>> for PrefixAs {
-    fn from(value: Vec<u8>) -> Self {
-        Self(*value.first_chunk::<4>().unwrap())
-    }
-}
-
-impl std::fmt::Display for PrefixAs {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "AS{}", u32::from_le_bytes(self.0))
-    }
-}
-
-// Hash implementation that always returns the same hash, so that all
-// records get thrown on one big heap.
-// impl std::hash::Hash for PrefixAs {
-//     fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-//         0.hash(state);
-//     }
-// }
-
-/// Tree-wide empty meta-data type
-///
-/// A special type that indicates that there's no metadata in the tree
-/// storing the prefixes. Note that this is different from a tree with
-/// optional meta-data.
-#[derive(Clone, Copy, Hash)]
-pub enum NoMeta {
-    Empty,
-}
-
-impl std::fmt::Debug for NoMeta {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str("")
-    }
-}
-
-impl std::fmt::Display for NoMeta {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str("NoMeta")
-    }
-}
-
-impl Meta for NoMeta {
-    type Orderable<'a> = ();
-    type TBI = ();
-    fn as_orderable(&self, _tbi: Self::TBI) {}
-}
-
-impl AsRef<[u8]> for NoMeta {
-    fn as_ref(&self) -> &[u8] {
-        &[]
-    }
-}
-
-impl From<Vec<u8>> for NoMeta {
-    fn from(_value: Vec<u8>) -> Self {
-        Self::Empty
-    }
-}
-
-// impl MergeUpdate for NoMeta {
-//     type UserDataIn = ();
-//     type UserDataOut = ();
-
-//     fn merge_update(
-//         &mut self,
-//         _: NoMeta,
-//         _: Option<&Self::UserDataIn>,
-//     ) -> Result<(), Box<dyn std::error::Error>> {
-//         Ok(())
-//     }
-
-//     fn clone_merge_update(
-//         &self,
-//         _: &NoMeta,
-//         _: Option<&Self::UserDataIn>,
-//     ) -> Result<(Self, Self::UserDataOut), Box<dyn std::error::Error>> {
-//         Ok((NoMeta::Empty, ()))
-//     }
-// }
diff --git a/src/types/mod.rs b/src/types/mod.rs
index 3ce30fca..6b1f3560 100644
--- a/src/types/mod.rs
+++ b/src/types/mod.rs
@@ -3,17 +3,16 @@ mod prefix_id;
 mod tests;
 
 pub(crate) mod af;
-pub(crate) mod match_options;
-pub(crate) mod prefix_record;
+pub mod match_options;
+pub mod prefix_record;
 pub(crate) mod route_status;
 
 pub(crate) use af::AddressFamily;
 pub(crate) use bit_span::BitSpan;
 pub(crate) use prefix_id::PrefixId;
-pub(crate) use prefix_record::PublicRecord;
+pub(crate) use prefix_record::Record;
 pub(crate) use route_status::RouteStatus;
 
 pub mod errors;
-pub mod meta_examples;
 pub mod stats;
 pub mod test_types;
diff --git a/src/types/prefix_record.rs b/src/types/prefix_record.rs
index 9d86b2f8..109421ad 100644
--- a/src/types/prefix_record.rs
+++ b/src/types/prefix_record.rs
@@ -1,257 +1,47 @@
 use std::fmt;
 use std::fmt::Debug;
-use std::{cmp::Ordering, sync::Arc};
 
-// use crate::prelude::multi::PrefixId;
 use crate::types::AddressFamily;
-use crate::types::RouteStatus;
 use inetnum::addr::Prefix;
-use zerocopy::{
-    Immutable, IntoBytes, KnownLayout, NetworkEndian, TryFromBytes,
-    Unaligned, U128, U32,
-};
+use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes, Unaligned};
 
 use super::PrefixId;
 
-//------------ InternalPrefixRecord -----------------------------------------
+pub use super::route_status::RouteStatus;
 
-// This struct is used for the SingleThreadedStore only.
-#[derive(Clone, Copy)]
-pub struct InternalPrefixRecord<AF, M>
-where
-    M: Meta,
-    AF: AddressFamily,
-{
-    pub net: AF,
-    pub len: u8,
-    pub meta: M,
-}
-
-impl<M, AF> InternalPrefixRecord<AF, M>
-where
-    M: Meta,
-    AF: AddressFamily,
-{
-    pub fn new_with_meta(
-        net: AF,
-        len: u8,
-        meta: M,
-    ) -> InternalPrefixRecord<AF, M> {
-        Self { net, len, meta }
-    }
-
-    // This should never fail, since there shouldn't be a invalid prefix in
-    // this record in the first place.
-    pub fn prefix_into_pub(&self) -> Prefix {
-        Prefix::new(self.net.into_ipaddr(), self.len)
-            .unwrap_or_else(|p| panic!("can't convert {:?} into prefix.", p))
-    }
-
-    pub fn get_prefix_id(&self) -> PrefixId<AF> {
-        PrefixId::new(self.net, self.len)
-    }
-
-    pub fn get_meta(&self) -> &M {
-        &self.meta
-    }
-}
-
-impl<M, AF> std::fmt::Display for InternalPrefixRecord<AF, M>
-where
-    M: Meta,
-    AF: AddressFamily,
-{
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{}/{} {}",
-            AddressFamily::fmt_net(self.net),
-            self.len,
-            self.meta
-        )
-    }
-}
-
-impl<AF, M> Ord for InternalPrefixRecord<AF, M>
-where
-    M: Meta,
-    AF: AddressFamily,
-{
-    fn cmp(&self, other: &Self) -> Ordering {
-        (self.net >> AF::from_u8(AF::BITS - self.len))
-            .cmp(&(other.net >> AF::from_u8((AF::BITS - other.len) % 32)))
-    }
-}
-
-impl<AF, M> PartialEq for InternalPrefixRecord<AF, M>
-where
-    M: Meta,
-    AF: AddressFamily,
-{
-    fn eq(&self, other: &Self) -> bool {
-        self.net >> AF::from_u8(AF::BITS - self.len)
-            == other.net >> AF::from_u8((AF::BITS - other.len) % 32)
-    }
-}
-
-impl<AF, M> PartialOrd for InternalPrefixRecord<AF, M>
-where
-    M: Meta,
-    AF: AddressFamily,
-{
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(
-            (self.net >> AF::from_u8(AF::BITS - self.len)).cmp(
-                &(other.net >> AF::from_u8((AF::BITS - other.len) % 32)),
-            ),
-        )
-    }
-}
-
-impl<AF, M> Eq for InternalPrefixRecord<AF, M>
-where
-    M: Meta,
-    AF: AddressFamily,
-{
-}
-
-impl<T, AF> Debug for InternalPrefixRecord<AF, T>
-where
-    AF: AddressFamily,
-    T: Meta,
-{
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_fmt(format_args!(
-            "{}/{} with {:?}",
-            AddressFamily::fmt_net(self.net),
-            self.len,
-            self.meta
-        ))
-    }
-}
+//------------ Meta ----------------------------------------------------------
 
-// impl<AF, T> std::hash::Hash for InternalPrefixRecord<AF, T>
-// where
-//     AF: AddressFamily + PrimInt + Debug,
-//     T: Meta,
-// {
-//     fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-//         self.net.hash(state);
-//         self.len.hash(state);
-//     }
-// }
-
-impl<AF, M> From<InternalPrefixRecord<AF, M>> for PrefixId<AF>
-where
-    AF: AddressFamily,
-    M: Meta,
-{
-    fn from(record: InternalPrefixRecord<AF, M>) -> Self {
-        Self::new(record.net, record.len)
-    }
-}
-
-impl<AF, T> From<&InternalPrefixRecord<AF, T>> for PrefixId<AF>
-where
-    AF: AddressFamily,
-    T: Meta,
-{
-    fn from(record: &InternalPrefixRecord<AF, T>) -> Self {
-        Self::new(record.net, record.len)
-    }
-}
-
-impl<M: Meta> From<PublicPrefixSingleRecord<M>>
-    for InternalPrefixRecord<crate::IPv4, M>
-{
-    fn from(record: PublicPrefixSingleRecord<M>) -> Self {
-        Self {
-            net: if let std::net::IpAddr::V4(ip) = record.prefix.addr() {
-                U32::<NetworkEndian>::from(ip.octets())
-            } else {
-                0.into()
-            },
-            len: record.prefix.len(),
-            meta: record.meta,
-        }
-    }
-}
-
-impl<M: Meta> From<PublicPrefixSingleRecord<M>>
-    for InternalPrefixRecord<crate::IPv6, M>
-{
-    fn from(record: PublicPrefixSingleRecord<M>) -> Self {
-        Self {
-            net: if let std::net::IpAddr::V6(ip) = record.prefix.addr() {
-                U128::<NetworkEndian>::from(ip.octets())
-            } else {
-                0.into()
-            },
-            len: record.prefix.len(),
-            meta: record.meta,
-        }
-    }
-}
-
-//------------ PublicPrefixSingleRecord --------------------------------------
-
-#[derive(Clone, Debug)]
-pub struct PublicPrefixSingleRecord<M: Meta> {
-    pub prefix: Prefix,
-    pub meta: M,
-}
-
-impl<M: Meta> PublicPrefixSingleRecord<M> {
-    pub fn new(prefix: Prefix, meta: M) -> Self {
-        Self { prefix, meta }
-    }
-
-    pub fn new_from_record<AF: AddressFamily>(
-        record: InternalPrefixRecord<AF, M>,
-    ) -> Self {
-        Self {
-            prefix: record.prefix_into_pub(),
-            meta: record.meta,
-        }
-    }
-}
-
-impl<AF, M> From<(PrefixId<AF>, Arc<M>)> for PublicPrefixSingleRecord<M>
+/// Trait for types that can be used as metadata of a record
+pub trait Meta
 where
-    AF: AddressFamily,
-    M: Meta,
+    Self: fmt::Debug
+        + fmt::Display
+        + Clone
+        + Sized
+        + Send
+        + Sync
+        + AsRef<[u8]>
+        + From<Vec<u8>>,
 {
-    fn from(record: (PrefixId<AF>, Arc<M>)) -> Self {
-        Self {
-            prefix: record.0.into(),
-            meta: (*record.1).clone(),
-        }
-    }
-}
-
-impl<M: Meta> std::fmt::Display for PublicPrefixSingleRecord<M> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{} :{:?}", self.prefix, self.meta)
-    }
-}
+    type Orderable<'a>: Ord
+    where
+        Self: 'a;
+    type TBI: Copy;
 
-impl<M: Meta> From<(Prefix, M)> for PublicPrefixSingleRecord<M> {
-    fn from((prefix, meta): (Prefix, M)) -> Self {
-        Self { prefix, meta }
-    }
+    fn as_orderable(&self, tbi: Self::TBI) -> Self::Orderable<'_>;
 }
 
 //------------ PublicRecord --------------------------------------------------
 
 #[derive(Clone, Debug)]
-pub struct PublicRecord<M> {
+pub struct Record<M> {
     pub multi_uniq_id: u32,
     pub ltime: u64,
     pub status: RouteStatus,
     pub meta: M,
 }
 
-impl<M> PublicRecord<M> {
+impl<M> Record<M> {
     pub fn new(
         multi_uniq_id: u32,
         ltime: u64,
@@ -267,7 +57,7 @@ impl<M> PublicRecord<M> {
     }
 }
 
-impl<M: std::fmt::Display> std::fmt::Display for PublicRecord<M> {
+impl<M: std::fmt::Display> std::fmt::Display for Record<M> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(
             f,
@@ -279,7 +69,7 @@ impl<M: std::fmt::Display> std::fmt::Display for PublicRecord<M> {
 
 #[derive(KnownLayout, Immutable, Unaligned, IntoBytes, TryFromBytes)]
 #[repr(C, packed)]
-pub struct ZeroCopyRecord<AF: AddressFamily> {
+pub(crate) struct ZeroCopyRecord<AF: AddressFamily> {
     pub prefix: PrefixId<AF>,
     pub multi_uniq_id: u32,
     pub ltime: u64,
@@ -303,7 +93,7 @@ impl<AF: AddressFamily + std::fmt::Display> std::fmt::Display
 
 #[derive(KnownLayout, Immutable, Unaligned, IntoBytes, TryFromBytes)]
 #[repr(C, packed)]
-pub struct ValueHeader {
+pub(crate) struct ValueHeader {
     pub ltime: u64,
     pub status: RouteStatus,
 }
@@ -318,28 +108,27 @@ impl std::fmt::Display for ValueHeader {
 //------------ PublicPrefixRecord --------------------------------------------
 
 #[derive(Clone, Debug)]
-pub struct PublicPrefixRecord<M: Meta> {
+pub struct PrefixRecord<M: Meta> {
     pub prefix: Prefix,
-    pub meta: Vec<PublicRecord<M>>,
+    pub meta: Vec<Record<M>>,
 }
 
-impl<M: Meta> PublicPrefixRecord<M> {
-    pub fn new(prefix: Prefix, meta: Vec<PublicRecord<M>>) -> Self {
+impl<M: Meta> PrefixRecord<M> {
+    pub fn new(prefix: Prefix, meta: Vec<Record<M>>) -> Self {
         Self { prefix, meta }
     }
 
-    pub fn get_record_for_mui(&self, mui: u32) -> Option<&PublicRecord<M>> {
+    pub fn get_record_for_mui(&self, mui: u32) -> Option<&Record<M>> {
         self.meta.iter().find(|r| r.multi_uniq_id == mui)
     }
 }
 
-impl<AF, M> From<(PrefixId<AF>, Vec<PublicRecord<M>>)>
-    for PublicPrefixRecord<M>
+impl<AF, M> From<(PrefixId<AF>, Vec<Record<M>>)> for PrefixRecord<M>
 where
     AF: AddressFamily,
     M: Meta,
 {
-    fn from(record: (PrefixId<AF>, Vec<PublicRecord<M>>)) -> Self {
+    fn from(record: (PrefixId<AF>, Vec<Record<M>>)) -> Self {
         Self {
             prefix: record.0.into(),
             meta: record.1,
@@ -347,9 +136,7 @@ where
     }
 }
 
-impl<M: Meta + std::fmt::Display> std::fmt::Display
-    for PublicPrefixRecord<M>
-{
+impl<M: Meta + std::fmt::Display> std::fmt::Display for PrefixRecord<M> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{}: [", self.prefix)?;
         for rec in &self.meta {
@@ -359,206 +146,18 @@ impl<M: Meta + std::fmt::Display> std::fmt::Display
     }
 }
 
-impl<M: Meta> From<(Prefix, Vec<PublicRecord<M>>)> for PublicPrefixRecord<M> {
-    fn from((prefix, meta): (Prefix, Vec<PublicRecord<M>>)) -> Self {
+impl<M: Meta> From<(Prefix, Vec<Record<M>>)> for PrefixRecord<M> {
+    fn from((prefix, meta): (Prefix, Vec<Record<M>>)) -> Self {
         Self { prefix, meta }
     }
 }
 
-//------------ RecordSingleSet -----------------------------------------------
-
-#[derive(Clone, Debug)]
-pub struct RecordSingleSet<M: Meta> {
-    pub v4: Vec<PublicPrefixSingleRecord<M>>,
-    pub v6: Vec<PublicPrefixSingleRecord<M>>,
-}
-
-// impl<M: Meta> RecordSingleSet<M> {
-//     pub fn new() -> Self {
-//         Self {
-//             v4: Default::default(),
-//             v6: Default::default(),
-//         }
-//     }
-
-//     pub fn push(&mut self, prefix: Prefix, meta: M) {
-//         match prefix.addr() {
-//             std::net::IpAddr::V4(_) => &mut self.v4,
-//             std::net::IpAddr::V6(_) => &mut self.v6,
-//         }
-//         .push(PublicPrefixSingleRecord::new(prefix, meta));
-//     }
-
-//     pub fn is_empty(&self) -> bool {
-//         self.v4.is_empty() && self.v6.is_empty()
-//     }
-
-//     pub fn iter(&self) -> RecordSetSingleIter<M> {
-//         RecordSetSingleIter {
-//             v4: if self.v4.is_empty() {
-//                 None
-//             } else {
-//                 Some(self.v4.iter())
-//             },
-//             v6: self.v6.iter(),
-//         }
-//     }
-
-//     #[must_use]
-//     pub fn reverse(mut self) -> RecordSingleSet<M> {
-//         self.v4.reverse();
-//         self.v6.reverse();
-//         self
-//     }
-
-//     pub fn len(&self) -> usize {
-//         self.v4.len() + self.v6.len()
-//     }
-// }
-
-// impl<M: Meta> Default for RecordSingleSet<M> {
-//     fn default() -> Self {
-//         Self::new()
-//     }
-// }
-
-// impl<M: Meta> fmt::Display for RecordSingleSet<M> {
-//     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-//         let arr_str_v4 =
-//             self.v4.iter().fold("".to_string(), |pfx_arr, pfx| {
-//                 format!("{} {}", pfx_arr, *pfx)
-//             });
-//         let arr_str_v6 =
-//             self.v6.iter().fold("".to_string(), |pfx_arr, pfx| {
-//                 format!("{} {}", pfx_arr, *pfx)
-//             });
-
-//         write!(f, "V4: [{}], V6: [{}]", arr_str_v4, arr_str_v6)
-//     }
-// }
-
-impl<M: Meta>
-    From<(
-        Vec<PublicPrefixSingleRecord<M>>,
-        Vec<PublicPrefixSingleRecord<M>>,
-    )> for RecordSingleSet<M>
-{
-    fn from(
-        (v4, v6): (
-            Vec<PublicPrefixSingleRecord<M>>,
-            Vec<PublicPrefixSingleRecord<M>>,
-        ),
-    ) -> Self {
-        Self { v4, v6 }
-    }
-}
-
-impl<M: Meta> std::iter::FromIterator<Arc<PublicPrefixSingleRecord<M>>>
-    for RecordSingleSet<M>
-{
-    fn from_iter<I: IntoIterator<Item = Arc<PublicPrefixSingleRecord<M>>>>(
-        iter: I,
-    ) -> Self {
-        let mut v4 = vec![];
-        let mut v6 = vec![];
-        for pfx in iter {
-            let u_pfx = pfx.prefix;
-            match u_pfx.addr() {
-                std::net::IpAddr::V4(_) => {
-                    v4.push(PublicPrefixSingleRecord::new(
-                        u_pfx,
-                        pfx.meta.clone(),
-                    ));
-                }
-                std::net::IpAddr::V6(_) => {
-                    v6.push(PublicPrefixSingleRecord::new(
-                        u_pfx,
-                        pfx.meta.clone(),
-                    ));
-                }
-            }
-        }
-        Self { v4, v6 }
-    }
-}
-
-impl<AF: AddressFamily, M: Meta>
-    std::iter::FromIterator<(PrefixId<AF>, Arc<M>)> for RecordSingleSet<M>
-{
-    fn from_iter<I: IntoIterator<Item = (PrefixId<AF>, Arc<M>)>>(
-        iter: I,
-    ) -> Self {
-        let mut v4 = vec![];
-        let mut v6 = vec![];
-        for pfx in iter {
-            let u_pfx = Prefix::from(pfx.0);
-            match u_pfx.addr() {
-                std::net::IpAddr::V4(_) => {
-                    v4.push(PublicPrefixSingleRecord::new(
-                        u_pfx,
-                        (*pfx.1).clone(),
-                    ));
-                }
-                std::net::IpAddr::V6(_) => {
-                    v6.push(PublicPrefixSingleRecord::new(
-                        u_pfx,
-                        (*pfx.1).clone(),
-                    ));
-                }
-            }
-        }
-        Self { v4, v6 }
-    }
-}
-
-impl<'a, AF: AddressFamily, M: Meta + 'a>
-    std::iter::FromIterator<&'a InternalPrefixRecord<AF, M>>
-    for RecordSingleSet<M>
-{
-    fn from_iter<I: IntoIterator<Item = &'a InternalPrefixRecord<AF, M>>>(
-        iter: I,
-    ) -> Self {
-        let mut v4 = vec![];
-        let mut v6 = vec![];
-        for pfx in iter {
-            let u_pfx = (*pfx).prefix_into_pub();
-            match u_pfx.addr() {
-                std::net::IpAddr::V4(_) => {
-                    v4.push(PublicPrefixSingleRecord::new(
-                        u_pfx,
-                        pfx.meta.clone(),
-                    ));
-                }
-                std::net::IpAddr::V6(_) => {
-                    v6.push(PublicPrefixSingleRecord::new(
-                        u_pfx,
-                        pfx.meta.clone(),
-                    ));
-                }
-            }
-        }
-        Self { v4, v6 }
-    }
-}
-
-impl<M: Meta> std::ops::Index<usize> for RecordSingleSet<M> {
-    type Output = PublicPrefixSingleRecord<M>;
-
-    fn index(&self, index: usize) -> &Self::Output {
-        if index < self.v4.len() {
-            &self.v4[index]
-        } else {
-            &self.v6[index - self.v4.len()]
-        }
-    }
-}
-
 //------------ RecordSet -----------------------------------------------------
 
 #[derive(Clone, Debug)]
 pub struct RecordSet<M: Meta> {
-    pub v4: Vec<PublicPrefixRecord<M>>,
-    pub v6: Vec<PublicPrefixRecord<M>>,
+    pub v4: Vec<PrefixRecord<M>>,
+    pub v6: Vec<PrefixRecord<M>>,
 }
 
 impl<M: Meta> RecordSet<M> {
@@ -569,12 +168,12 @@ impl<M: Meta> RecordSet<M> {
         }
     }
 
-    pub fn push(&mut self, prefix: Prefix, meta: Vec<PublicRecord<M>>) {
+    pub fn push(&mut self, prefix: Prefix, meta: Vec<Record<M>>) {
         match prefix.addr() {
             std::net::IpAddr::V4(_) => &mut self.v4,
             std::net::IpAddr::V6(_) => &mut self.v6,
         }
-        .push(PublicPrefixRecord::new(prefix, meta));
+        .push(PrefixRecord::new(prefix, meta));
     }
 
     pub fn is_empty(&self) -> bool {
@@ -625,32 +224,26 @@ impl<M: Meta> fmt::Display for RecordSet<M> {
     }
 }
 
-impl<M: Meta> From<(Vec<PublicPrefixRecord<M>>, Vec<PublicPrefixRecord<M>>)>
+impl<M: Meta> From<(Vec<PrefixRecord<M>>, Vec<PrefixRecord<M>>)>
     for RecordSet<M>
 {
-    fn from(
-        (v4, v6): (Vec<PublicPrefixRecord<M>>, Vec<PublicPrefixRecord<M>>),
-    ) -> Self {
+    fn from((v4, v6): (Vec<PrefixRecord<M>>, Vec<PrefixRecord<M>>)) -> Self {
         Self { v4, v6 }
     }
 }
 
-impl<M: Meta> std::iter::FromIterator<PublicPrefixRecord<M>>
-    for RecordSet<M>
-{
-    fn from_iter<I: IntoIterator<Item = PublicPrefixRecord<M>>>(
-        iter: I,
-    ) -> Self {
+impl<M: Meta> std::iter::FromIterator<PrefixRecord<M>> for RecordSet<M> {
+    fn from_iter<I: IntoIterator<Item = PrefixRecord<M>>>(iter: I) -> Self {
         let mut v4 = vec![];
         let mut v6 = vec![];
         for pfx in iter {
             let u_pfx = pfx.prefix;
             match u_pfx.addr() {
                 std::net::IpAddr::V4(_) => {
-                    v4.push(PublicPrefixRecord::new(u_pfx, pfx.meta));
+                    v4.push(PrefixRecord::new(u_pfx, pfx.meta));
                 }
                 std::net::IpAddr::V6(_) => {
-                    v6.push(PublicPrefixRecord::new(u_pfx, pfx.meta));
+                    v6.push(PrefixRecord::new(u_pfx, pfx.meta));
                 }
             }
         }
@@ -659,12 +252,9 @@ impl<M: Meta> std::iter::FromIterator<PublicPrefixRecord<M>>
 }
 
 impl<AF: AddressFamily, M: Meta>
-    std::iter::FromIterator<(PrefixId<AF>, Vec<PublicRecord<M>>)>
-    for RecordSet<M>
+    std::iter::FromIterator<(PrefixId<AF>, Vec<Record<M>>)> for RecordSet<M>
 {
-    fn from_iter<
-        I: IntoIterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)>,
-    >(
+    fn from_iter<I: IntoIterator<Item = (PrefixId<AF>, Vec<Record<M>>)>>(
         iter: I,
     ) -> Self {
         let mut v4 = vec![];
@@ -673,10 +263,10 @@ impl<AF: AddressFamily, M: Meta>
             let u_pfx = Prefix::from(pfx.0);
             match u_pfx.addr() {
                 std::net::IpAddr::V4(_) => {
-                    v4.push(PublicPrefixRecord::new(u_pfx, pfx.1));
+                    v4.push(PrefixRecord::new(u_pfx, pfx.1));
                 }
                 std::net::IpAddr::V6(_) => {
-                    v6.push(PublicPrefixRecord::new(u_pfx, pfx.1));
+                    v6.push(PrefixRecord::new(u_pfx, pfx.1));
                 }
             }
         }
@@ -684,10 +274,10 @@ impl<AF: AddressFamily, M: Meta>
     }
 }
 
-impl<'a, M: Meta + 'a> std::iter::FromIterator<&'a PublicPrefixRecord<M>>
+impl<'a, M: Meta + 'a> std::iter::FromIterator<&'a PrefixRecord<M>>
     for RecordSet<M>
 {
-    fn from_iter<I: IntoIterator<Item = &'a PublicPrefixRecord<M>>>(
+    fn from_iter<I: IntoIterator<Item = &'a PrefixRecord<M>>>(
         iter: I,
     ) -> Self {
         let mut v4 = vec![];
@@ -696,10 +286,10 @@ impl<'a, M: Meta + 'a> std::iter::FromIterator<&'a PublicPrefixRecord<M>>
             let u_pfx = pfx.prefix;
             match u_pfx.addr() {
                 std::net::IpAddr::V4(_) => {
-                    v4.push(PublicPrefixRecord::new(u_pfx, pfx.meta.clone()));
+                    v4.push(PrefixRecord::new(u_pfx, pfx.meta.clone()));
                 }
                 std::net::IpAddr::V6(_) => {
-                    v6.push(PublicPrefixRecord::new(u_pfx, pfx.meta.clone()));
+                    v6.push(PrefixRecord::new(u_pfx, pfx.meta.clone()));
                 }
             }
         }
@@ -708,7 +298,7 @@ impl<'a, M: Meta + 'a> std::iter::FromIterator<&'a PublicPrefixRecord<M>>
 }
 
 impl<M: Meta> std::ops::Index<usize> for RecordSet<M> {
-    type Output = PublicPrefixRecord<M>;
+    type Output = PrefixRecord<M>;
 
     fn index(&self, index: usize) -> &Self::Output {
         if index < self.v4.len() {
@@ -719,40 +309,16 @@ impl<M: Meta> std::ops::Index<usize> for RecordSet<M> {
     }
 }
 
-//------------ RecordSetSingleIter -------------------------------------------
-
-#[derive(Clone, Debug)]
-pub struct RecordSetSingleIter<'a, M: Meta> {
-    v4: Option<std::slice::Iter<'a, PublicPrefixSingleRecord<M>>>,
-    v6: std::slice::Iter<'a, PublicPrefixSingleRecord<M>>,
-}
-
-impl<M: Meta> Iterator for RecordSetSingleIter<'_, M> {
-    type Item = PublicPrefixSingleRecord<M>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.v4.is_none() {
-            return self.v6.next().map(|res| res.to_owned());
-        }
-
-        if let Some(res) = self.v4.as_mut().and_then(|v4| v4.next()) {
-            return Some(res.to_owned());
-        }
-        self.v4 = None;
-        self.next()
-    }
-}
-
 //------------ RecordSetIter -------------------------------------------------
 
 #[derive(Clone, Debug)]
 pub struct RecordSetIter<'a, M: Meta> {
-    v4: Option<std::slice::Iter<'a, PublicPrefixRecord<M>>>,
-    v6: std::slice::Iter<'a, PublicPrefixRecord<M>>,
+    v4: Option<std::slice::Iter<'a, PrefixRecord<M>>>,
+    v6: std::slice::Iter<'a, PrefixRecord<M>>,
 }
 
 impl<M: Meta> Iterator for RecordSetIter<'_, M> {
-    type Item = PublicPrefixRecord<M>;
+    type Item = PrefixRecord<M>;
 
     fn next(&mut self) -> Option<Self::Item> {
         if self.v4.is_none() {
@@ -766,25 +332,3 @@ impl<M: Meta> Iterator for RecordSetIter<'_, M> {
         self.next()
     }
 }
-
-//----------------------- meta-data traits/types-----------------------------
-
-/// Trait for types that can be used as metadata of a record
-pub trait Meta
-where
-    Self: fmt::Debug
-        + fmt::Display
-        + Clone
-        + Sized
-        + Send
-        + Sync
-        + AsRef<[u8]>
-        + From<Vec<u8>>,
-{
-    type Orderable<'a>: Ord
-    where
-        Self: 'a;
-    type TBI: Copy;
-
-    fn as_orderable(&self, tbi: Self::TBI) -> Self::Orderable<'_>;
-}
diff --git a/src/types/stats.rs b/src/types/stats.rs
index 1eaa4fec..5ed8e4ca 100644
--- a/src/types/stats.rs
+++ b/src/types/stats.rs
@@ -1,26 +1,22 @@
 //------------ Types for Statistics -----------------------------------------
 
-// use crate::stride::{Stride3, Stride4, Stride5, Stride6, Stride7, Stride8};
 use std::{
     fmt::{Debug, Display},
     marker::PhantomData,
+    sync::atomic::{AtomicUsize, Ordering},
 };
 
-use crate::{
-    rib::STRIDE_SIZE, tree_bitmap::TreeBitMapNode, types::AddressFamily,
-};
+use crate::{rib::STRIDE_SIZE, types::AddressFamily};
 
-pub struct StrideStats<AF: AddressFamily> {
-    pub node_size: usize,
-    pub created_nodes: Vec<CreatedNodes>,
-    pub prefixes_num: Vec<CreatedNodes>,
+pub(crate) struct StrideStats<AF: AddressFamily> {
+    pub(crate) created_nodes: Vec<CreatedNodes>,
+    pub(crate) prefixes_num: Vec<CreatedNodes>,
     _af: PhantomData<AF>,
 }
 
 impl<AF: AddressFamily> StrideStats<AF> {
     pub fn new() -> Self {
         Self {
-            node_size: std::mem::size_of::<TreeBitMapNode<AF>>(),
             created_nodes: Self::nodes_vec(AF::BITS / STRIDE_SIZE),
             prefixes_num: Self::nodes_vec(AF::BITS / STRIDE_SIZE),
             _af: PhantomData,
@@ -108,3 +104,155 @@ impl Debug for CreatedNodes {
         f.write_fmt(format_args!("/{}: {}", &self.depth_level, &self.count))
     }
 }
+
+//------------ Counters -----------------------------------------------------
+
+#[derive(Debug)]
+pub(crate) struct Counters {
+    // number of created nodes in the in-mem tree
+    nodes: AtomicUsize,
+    // number of unique prefixes in the store
+    prefixes: [AtomicUsize; 129],
+    // number of unique (prefix, mui) values inserted in the in-mem tree
+    routes: AtomicUsize,
+}
+
+impl Counters {
+    pub fn get_nodes_count(&self) -> usize {
+        self.nodes.load(Ordering::Relaxed)
+    }
+
+    pub fn inc_nodes_count(&self) {
+        self.nodes.fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub fn get_prefixes_count(&self) -> Vec<usize> {
+        self.prefixes
+            .iter()
+            .map(|pc| pc.load(Ordering::Relaxed))
+            .collect::<Vec<_>>()
+    }
+
+    pub fn inc_prefixes_count(&self, len: u8) {
+        self.prefixes[len as usize].fetch_add(1, Ordering::Relaxed);
+    }
+
+    pub fn _dec_prefixes_count(&self, len: u8) {
+        self.prefixes[len as usize].fetch_sub(1, Ordering::Relaxed);
+    }
+
+    pub fn get_prefix_stats(&self) -> Vec<CreatedNodes> {
+        self.prefixes
+            .iter()
+            .enumerate()
+            .filter_map(|(len, count)| -> Option<CreatedNodes> {
+                let count = count.load(Ordering::Relaxed);
+                if count != 0 {
+                    Some(CreatedNodes {
+                        depth_level: len as u8,
+                        count,
+                    })
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+
+    pub fn inc_routes_count(&self) {
+        self.routes.fetch_add(1, Ordering::Relaxed);
+    }
+}
+
+impl Default for Counters {
+    fn default() -> Self {
+        let mut prefixes: Vec<AtomicUsize> = Vec::with_capacity(129);
+        for _ in 0..=128 {
+            prefixes.push(AtomicUsize::new(0));
+        }
+
+        Self {
+            nodes: AtomicUsize::new(0),
+            prefixes: prefixes.try_into().unwrap(),
+            routes: AtomicUsize::new(0),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct UpsertCounters {
+    // number of unique inserted prefixes|routes in the in-mem tree
+    pub(crate) in_memory_count: usize,
+    // number of unique persisted prefixes|routes
+    pub(crate) persisted_count: usize,
+    // total number of unique inserted prefixes|routes in the RIB
+    pub(crate) total_count: usize,
+}
+
+impl UpsertCounters {
+    pub fn in_memory(&self) -> usize {
+        self.in_memory_count
+    }
+
+    pub fn persisted(&self) -> usize {
+        self.persisted_count
+    }
+
+    pub fn total(&self) -> usize {
+        self.total_count
+    }
+}
+
+impl std::fmt::Display for UpsertCounters {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Unique Items in-memory:\t{}", self.in_memory_count)?;
+        write!(f, "Unique persisted Items:\t{}", self.persisted_count)?;
+        write!(f, "Total inserted Items:\t{}", self.total_count)
+    }
+}
+
+impl std::ops::AddAssign for UpsertCounters {
+    fn add_assign(&mut self, rhs: Self) {
+        self.in_memory_count += rhs.in_memory_count;
+        self.persisted_count += rhs.persisted_count;
+        self.total_count += rhs.total_count;
+    }
+}
+
+impl std::ops::Add for UpsertCounters {
+    type Output = UpsertCounters;
+
+    fn add(self, rhs: Self) -> Self::Output {
+        Self {
+            in_memory_count: self.in_memory_count + rhs.in_memory_count,
+            persisted_count: self.persisted_count + rhs.persisted_count,
+            total_count: self.total_count + rhs.total_count,
+        }
+    }
+}
+
+//------------ StoreStats ----------------------------------------------
+
+#[derive(Debug)]
+pub struct StoreStats {
+    pub v4: Vec<CreatedNodes>,
+    pub v6: Vec<CreatedNodes>,
+}
+
+//------------ UpsertReport --------------------------------------------------
+
+#[derive(Debug)]
+pub struct UpsertReport {
+    // Indicates the number of Atomic Compare-and-Swap operations were
+    // necessary to create/update the Record entry. High numbers indicate
+    // contention.
+    pub cas_count: usize,
+    // Indicates whether this was the first mui record for this prefix was
+    // created. So, the prefix did not exist before hand.
+    pub prefix_new: bool,
+    // Indicates whether this mui was new for this prefix. False means an old
+    // value was overwritten.
+    pub mui_new: bool,
+    // The number of mui records for this prefix after the upsert operation.
+    pub mui_count: usize,
+}
diff --git a/src/types/test_types.rs b/src/types/test_types.rs
index abfc252a..7c40df00 100644
--- a/src/types/test_types.rs
+++ b/src/types/test_types.rs
@@ -1,7 +1,44 @@
 use inetnum::asn::Asn;
 
-use crate::Meta;
+use super::prefix_record::Meta;
 
+#[derive(Clone, Copy, Hash)]
+pub enum NoMeta {
+    Empty,
+}
+
+impl std::fmt::Debug for NoMeta {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("")
+    }
+}
+
+impl std::fmt::Display for NoMeta {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("NoMeta")
+    }
+}
+
+impl Meta for NoMeta {
+    type Orderable<'a> = ();
+    type TBI = ();
+    fn as_orderable(&self, _tbi: Self::TBI) {}
+}
+
+impl AsRef<[u8]> for NoMeta {
+    fn as_ref(&self) -> &[u8] {
+        &[]
+    }
+}
+
+impl From<Vec<u8>> for NoMeta {
+    fn from(_value: Vec<u8>) -> Self {
+        Self::Empty
+    }
+}
+
+//------------ BeBytesAsn ----------------------------------------------------
+//
 #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
 pub struct BeBytesAsn(pub [u8; 4]);
 
@@ -47,3 +84,48 @@ impl From<u32> for BeBytesAsn {
         Self(value.to_le_bytes())
     }
 }
+
+//------------ PrefixAs ------------------------------------------------------
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct PrefixAs([u8; 4]);
+
+impl PrefixAs {
+    pub fn new(asn: Asn) -> Self {
+        PrefixAs(u32::from_be_bytes(asn.to_raw()).to_le_bytes())
+    }
+
+    pub fn new_from_u32(value: u32) -> Self {
+        PrefixAs(value.to_le_bytes())
+    }
+
+    pub fn asn(&self) -> Asn {
+        Asn::from_u32(u32::from_le_bytes(self.0))
+    }
+}
+
+impl Meta for PrefixAs {
+    type Orderable<'a> = Asn;
+    type TBI = ();
+    fn as_orderable(&self, _tbi: Self::TBI) -> Asn {
+        u32::from_le_bytes(self.0).into()
+    }
+}
+
+impl AsRef<[u8]> for PrefixAs {
+    fn as_ref(&self) -> &[u8] {
+        self.0.as_ref()
+    }
+}
+
+impl From<Vec<u8>> for PrefixAs {
+    fn from(value: Vec<u8>) -> Self {
+        Self(*value.first_chunk::<4>().unwrap())
+    }
+}
+
+impl std::fmt::Display for PrefixAs {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "AS{}", u32::from_le_bytes(self.0))
+    }
+}

From 035937cc403552812ea44b6808053d0033821cf4 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 12 Mar 2025 16:15:43 +0100
Subject: [PATCH 121/147] move some types from config to stats

---
 src/rib/config.rs | 141 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 141 insertions(+)
 create mode 100644 src/rib/config.rs

diff --git a/src/rib/config.rs b/src/rib/config.rs
new file mode 100644
index 00000000..c50d768c
--- /dev/null
+++ b/src/rib/config.rs
@@ -0,0 +1,141 @@
+//------------ Config --------------------------------------------------------
+
+//! Configuration options for a RIB for AFI/SAFIs [IPv4, IPv6] with [Unicast,
+//! Multicast]
+
+/// Defines where records are stored, in-memory and/or persisted (to disk),
+/// and, whether new records for a unique (prefix, mui) pair are overwritten
+/// or persisted.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum PersistStrategy {
+    /// Current records are stored both in-memory and persisted. Historical
+    /// records are persisted.
+    WriteAhead,
+    /// Current records are stored in-memory, historical records are
+    /// persisted.
+    PersistHistory,
+    /// Current records are stored in-memory, historical records are discarded
+    /// when newer records appear.
+    MemoryOnly,
+    /// Current records are persisted immediately. No records are stored in
+    /// memory. Historical records are discarded when newer records appear.
+    PersistOnly,
+}
+
+pub trait Config: Clone + Default + std::fmt::Debug {
+    fn persist_strategy(&self) -> PersistStrategy;
+    fn persist_path(&self) -> Option<String>;
+    fn set_persist_path(&mut self, path: String);
+}
+
+//------------ MemoryOnlyConfig ----------------------------------------------
+
+#[derive(Copy, Clone, Debug)]
+pub struct MemoryOnlyConfig;
+
+impl Config for MemoryOnlyConfig {
+    fn persist_strategy(&self) -> PersistStrategy {
+        PersistStrategy::MemoryOnly
+    }
+
+    fn persist_path(&self) -> Option<String> {
+        None
+    }
+
+    fn set_persist_path(&mut self, _: String) {
+        unimplemented!()
+    }
+}
+
+impl Default for MemoryOnlyConfig {
+    fn default() -> Self {
+        Self
+    }
+}
+
+//------------ PeristOnlyConfig ----------------------------------------------
+
+impl Default for PersistOnlyConfig {
+    fn default() -> Self {
+        Self {
+            persist_path: "/tmp/rotonda/".to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct PersistOnlyConfig {
+    persist_path: String,
+}
+
+impl Config for PersistOnlyConfig {
+    fn persist_strategy(&self) -> PersistStrategy {
+        PersistStrategy::PersistOnly
+    }
+
+    fn persist_path(&self) -> Option<String> {
+        Some(self.persist_path.clone())
+    }
+
+    fn set_persist_path(&mut self, path: String) {
+        self.persist_path = path;
+    }
+}
+
+//------------ WriteAheadConfig ----------------------------------------------
+
+impl Default for WriteAheadConfig {
+    fn default() -> Self {
+        Self {
+            persist_path: "/tmp/rotonda/".to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct WriteAheadConfig {
+    persist_path: String,
+}
+
+impl Config for WriteAheadConfig {
+    fn persist_strategy(&self) -> PersistStrategy {
+        PersistStrategy::WriteAhead
+    }
+
+    fn persist_path(&self) -> Option<String> {
+        Some(self.persist_path.clone())
+    }
+
+    fn set_persist_path(&mut self, path: String) {
+        self.persist_path = path;
+    }
+}
+
+//------------ PersistHistoryConfig ------------------------------------------
+
+#[derive(Clone, Debug)]
+pub struct PersistHistoryConfig {
+    persist_path: String,
+}
+
+impl Config for PersistHistoryConfig {
+    fn persist_strategy(&self) -> PersistStrategy {
+        PersistStrategy::PersistHistory
+    }
+
+    fn persist_path(&self) -> Option<String> {
+        Some(self.persist_path.clone())
+    }
+
+    fn set_persist_path(&mut self, path: String) {
+        self.persist_path = path;
+    }
+}
+
+impl Default for PersistHistoryConfig {
+    fn default() -> Self {
+        Self {
+            persist_path: "/tmp/rotonda/".to_string(),
+        }
+    }
+}

From 0c5d3a387bf3dbd85c4ff0d49565dd0bbc76b706 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 12 Mar 2025 17:34:16 +0100
Subject: [PATCH 122/147] and all the skipped files from the last 3 commits...

---
 examples/exact_matches.rs                  |  11 +-
 examples/exact_matches_single.rs           |  10 +-
 examples/full_table_multiple_trees_json.rs |  14 +-
 examples/more_specifics.rs                 |  42 +--
 examples/multi_no_thread.rs                |  14 +-
 examples/multi_single_thread.rs            |   9 +-
 examples/multi_thread_1.rs                 |  10 +-
 examples/multi_thread_2.rs                 |  14 +-
 examples/multi_thread_3.rs                 |  14 +-
 examples/multi_thread_4.rs                 |  13 +-
 examples/multi_thread_multi_prefix.rs      |  55 ++--
 examples/multi_thread_single_prefix.rs     |  22 +-
 examples/numbers_treebitmap.rs             |   8 +-
 examples/real_single_thread_24.rs          |  13 +-
 examples/single_thread_24.rs               |  10 +-
 examples/treebitmap.rs                     |  19 +-
 src/bin/cli.rs                             |  12 +-
 src/bin/load_mrt.rs                        |  47 +--
 src/lib.rs                                 |  60 ++--
 src/lsm_tree/mod.rs                        |  11 +-
 src/macros.rs                              |   5 +-
 src/prefix_cht/cht.rs                      |  60 ++--
 src/rib/config.rs                          |  19 +-
 src/rib/mod.rs                             |   4 +-
 src/rib/starcast.rs                        |  28 +-
 src/rib/starcast_af.rs                     | 319 +--------------------
 src/rib/starcast_af_query.rs               |  26 +-
 src/tree_bitmap/mod.rs                     |   2 +-
 src/tree_bitmap/tree_bitmap_query.rs       |   2 +-
 src/types/match_options.rs                 |   4 +-
 tests/best-path.rs                         |  17 +-
 tests/concurrency.rs                       |  17 +-
 tests/full-table.rs                        |   7 +-
 tests/more-more-specifics.rs               |   8 +-
 tests/more-specifics.rs                    |   7 +-
 tests/treebitmap.rs                        |   8 +-
 tests/treebitmap_v6.rs                     |  14 +-
 37 files changed, 381 insertions(+), 574 deletions(-)

diff --git a/examples/exact_matches.rs b/examples/exact_matches.rs
index abb60652..39ea717b 100644
--- a/examples/exact_matches.rs
+++ b/examples/exact_matches.rs
@@ -1,9 +1,14 @@
 use inetnum::addr::Prefix;
-use rotonda_store::meta_examples::NoMeta;
+use rotonda_store::match_options::IncludeHistory;
+use rotonda_store::prefix_record::{Record, RouteStatus};
 // use rotonda_store::prelude::multi::*;
 use rotonda_store::{
-    epoch, IncludeHistory, IntoIpAddr, MatchOptions, MatchType,
-    MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
+    epoch,
+    match_options::{MatchOptions, MatchType},
+    rib::config::MemoryOnlyConfig,
+    rib::StarCastRib,
+    test_types::NoMeta,
+    IntoIpAddr,
 };
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
diff --git a/examples/exact_matches_single.rs b/examples/exact_matches_single.rs
index 34df1b92..ed15c3a6 100644
--- a/examples/exact_matches_single.rs
+++ b/examples/exact_matches_single.rs
@@ -1,8 +1,12 @@
 use inetnum::addr::Prefix;
-use rotonda_store::meta_examples::NoMeta;
 use rotonda_store::{
-    IncludeHistory, IntoIpAddr, MatchOptions, MatchType, MemoryOnlyConfig,
-    Record, RouteStatus, StarCastRib,
+    match_options::{IncludeHistory, MatchOptions, MatchType},
+    prefix_record::{Record, RouteStatus},
+    IntoIpAddr,
+};
+use rotonda_store::{
+    rib::{config::MemoryOnlyConfig, StarCastRib},
+    test_types::NoMeta,
 };
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index 4577f542..9987180b 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -1,10 +1,10 @@
 use inetnum::addr::Prefix;
 use rotonda_store::epoch;
-use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::{
-    IncludeHistory, MatchOptions, MatchType, MemoryOnlyConfig, PrefixRecord,
-    Record, RouteStatus, StarCastRib,
-};
+use rotonda_store::match_options::{IncludeHistory, MatchOptions, MatchType};
+use rotonda_store::prefix_record::{PrefixRecord, Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::test_types::PrefixAs;
 
 use std::error::Error;
 use std::fs::File;
@@ -56,9 +56,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             let mut rec_vec: Vec<PrefixRecord<PrefixAs>> = vec![];
             let config = MemoryOnlyConfig;
             let tree_bitmap =
-                StarCastRib::<PrefixAs, MemoryOnlyConfig>::new_with_config(
-                    config,
-                )?;
+                StarCastRib::<PrefixAs, _>::new_with_config(config)?;
 
             if let Err(err) = load_prefixes(&mut rec_vec) {
                 println!("error running example: {}", err);
diff --git a/examples/more_specifics.rs b/examples/more_specifics.rs
index 2985f47c..9ef16f80 100644
--- a/examples/more_specifics.rs
+++ b/examples/more_specifics.rs
@@ -1,14 +1,14 @@
+use inetnum::addr::Prefix;
 use rotonda_store::{
-    epoch, meta_examples::PrefixAs, IncludeHistory, IntoIpAddr, MatchOptions,
-    MatchType, Record, RouteStatus, StarCastRib,
+    epoch,
+    match_options::{IncludeHistory, MatchOptions, MatchType},
+    prefix_record::{Record, RouteStatus},
+    rib::{config::MemoryOnlyConfig, StarCastRib},
+    test_types::PrefixAs,
+    IntoIpAddr,
 };
 
-use inetnum::addr::Prefix;
-use rotonda_store::MemoryOnlyConfig;
-
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let tree_bitmap =
-        StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new_relaxed(
             0b0000_0000_0000_0000_0000_0000_0000_0000_u32.into_ipaddr(),
@@ -215,7 +215,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     for pfx in pfxs.into_iter() {
         // println!("insert {:?}", pfx);
         let p: Prefix = pfx.unwrap();
-        tree_bitmap.insert(
+        StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?.insert(
             &p,
             Record::new(0, 0, RouteStatus::Active, PrefixAs::new(666.into())),
             None,
@@ -281,18 +281,20 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     ] {
         println!("search for: {:?}", spfx);
         let guard = &epoch::pin();
-        let s_spfx = tree_bitmap.match_prefix(
-            &spfx.unwrap(),
-            &MatchOptions {
-                match_type: MatchType::ExactMatch,
-                include_withdrawn: false,
-                include_less_specifics: true,
-                include_more_specifics: true,
-                mui: None,
-                include_history: IncludeHistory::None,
-            },
-            guard,
-        );
+        let s_spfx =
+            StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?
+                .match_prefix(
+                    &spfx.unwrap(),
+                    &MatchOptions {
+                        match_type: MatchType::ExactMatch,
+                        include_withdrawn: false,
+                        include_less_specifics: true,
+                        include_more_specifics: true,
+                        mui: None,
+                        include_history: IncludeHistory::None,
+                    },
+                    guard,
+                );
         println!("em/m-s: {:#?}", s_spfx);
         println!("-----------");
     }
diff --git a/examples/multi_no_thread.rs b/examples/multi_no_thread.rs
index 32a10caa..d490484c 100644
--- a/examples/multi_no_thread.rs
+++ b/examples/multi_no_thread.rs
@@ -2,14 +2,12 @@ use inetnum::addr::Prefix;
 use log::trace;
 
 use rotonda_store::epoch;
-use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::IncludeHistory;
+use rotonda_store::match_options::{IncludeHistory, MatchOptions, MatchType};
+use rotonda_store::prefix_record::{Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::test_types::PrefixAs;
 use rotonda_store::IntoIpAddr;
-use rotonda_store::MatchOptions;
-use rotonda_store::MemoryOnlyConfig;
-use rotonda_store::Record;
-use rotonda_store::RouteStatus;
-use rotonda_store::StarCastRib;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
@@ -61,7 +59,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let s_spfx = tree_bitmap.match_prefix(
         &pfx.unwrap(),
         &MatchOptions {
-            match_type: rotonda_store::MatchType::ExactMatch,
+            match_type: MatchType::ExactMatch,
             include_withdrawn: true,
             include_less_specifics: true,
             include_more_specifics: true,
diff --git a/examples/multi_single_thread.rs b/examples/multi_single_thread.rs
index 5187bfae..2d82cc1e 100644
--- a/examples/multi_single_thread.rs
+++ b/examples/multi_single_thread.rs
@@ -1,14 +1,15 @@
 use inetnum::addr::Prefix;
 use log::trace;
-use rotonda_store::{
-    IntoIpAddr, MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
-};
+use rotonda_store::prefix_record::{Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::IntoIpAddr;
 use std::thread;
 use std::time::Duration;
 
 use rand::Rng;
 
-use rotonda_store::meta_examples::PrefixAs;
+use rotonda_store::test_types::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
diff --git a/examples/multi_thread_1.rs b/examples/multi_thread_1.rs
index 5f869298..3460e0f5 100644
--- a/examples/multi_thread_1.rs
+++ b/examples/multi_thread_1.rs
@@ -2,8 +2,12 @@ use std::{sync::Arc, thread};
 
 use inetnum::addr::Prefix;
 use rotonda_store::{
-    epoch, meta_examples::NoMeta, IncludeHistory, IntoIpAddr, MatchOptions,
-    MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
+    epoch,
+    match_options::{IncludeHistory, MatchOptions, MatchType},
+    prefix_record::{Record, RouteStatus},
+    rib::{config::MemoryOnlyConfig, StarCastRib},
+    test_types::NoMeta,
+    IntoIpAddr,
 };
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -53,7 +57,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         let s_spfx = tree_bitmap.match_prefix(
             &spfx.unwrap(),
             &MatchOptions {
-                match_type: rotonda_store::MatchType::ExactMatch,
+                match_type: MatchType::ExactMatch,
                 include_withdrawn: false,
                 include_less_specifics: true,
                 include_more_specifics: true,
diff --git a/examples/multi_thread_2.rs b/examples/multi_thread_2.rs
index 5ef0c9cd..9ed7e0cb 100644
--- a/examples/multi_thread_2.rs
+++ b/examples/multi_thread_2.rs
@@ -1,11 +1,11 @@
 use inetnum::addr::Prefix;
 use log::trace;
-use rotonda_store::epoch;
-use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::{
-    IncludeHistory, IntoIpAddr, MatchOptions, MemoryOnlyConfig, Record,
-    RouteStatus, StarCastRib,
-};
+use rotonda_store::match_options::{IncludeHistory, MatchOptions, MatchType};
+use rotonda_store::prefix_record::{Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::test_types::PrefixAs;
+use rotonda_store::{epoch, IntoIpAddr};
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
@@ -68,7 +68,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let s_spfx = tree_bitmap.match_prefix(
         &pfx.unwrap(),
         &MatchOptions {
-            match_type: rotonda_store::MatchType::ExactMatch,
+            match_type: MatchType::ExactMatch,
             include_withdrawn: true,
             include_less_specifics: true,
             include_more_specifics: true,
diff --git a/examples/multi_thread_3.rs b/examples/multi_thread_3.rs
index f60763cd..eb0a3eb4 100644
--- a/examples/multi_thread_3.rs
+++ b/examples/multi_thread_3.rs
@@ -1,14 +1,14 @@
 use inetnum::addr::Prefix;
 use log::trace;
-use rotonda_store::{epoch, IncludeHistory};
-use rotonda_store::{
-    IntoIpAddr, MatchOptions, MemoryOnlyConfig, Record, RouteStatus,
-    StarCastRib,
-};
+use rotonda_store::match_options::{IncludeHistory, MatchOptions, MatchType};
+use rotonda_store::prefix_record::{Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::{epoch, IntoIpAddr};
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
-use rotonda_store::meta_examples::PrefixAs;
+use rotonda_store::test_types::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
@@ -95,7 +95,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let s_spfx = tree_bitmap.match_prefix(
         &pfx.unwrap(),
         &MatchOptions {
-            match_type: rotonda_store::MatchType::ExactMatch,
+            match_type: MatchType::ExactMatch,
             include_withdrawn: true,
             include_less_specifics: true,
             include_more_specifics: true,
diff --git a/examples/multi_thread_4.rs b/examples/multi_thread_4.rs
index 3643805c..4de0929c 100644
--- a/examples/multi_thread_4.rs
+++ b/examples/multi_thread_4.rs
@@ -1,10 +1,11 @@
 use inetnum::addr::Prefix;
 use inetnum::asn::Asn;
 use log::trace;
-use rotonda_store::{
-    epoch, IncludeHistory, IntoIpAddr, MatchOptions, MemoryOnlyConfig, Meta,
-    Record, RouteStatus, StarCastRib,
-};
+use rotonda_store::match_options::{IncludeHistory, MatchOptions, MatchType};
+use rotonda_store::prefix_record::{Meta, Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::IntoIpAddr;
 use std::time::Duration;
 use std::{sync::Arc, thread};
 
@@ -125,12 +126,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     println!("------ end of inserts\n");
 
-    let guard = &epoch::pin();
+    let guard = &rotonda_store::epoch::pin();
 
     let s_spfx = tree_bitmap.match_prefix(
         &pfx.unwrap(),
         &MatchOptions {
-            match_type: rotonda_store::MatchType::ExactMatch,
+            match_type: MatchType::ExactMatch,
             include_withdrawn: true,
             include_less_specifics: true,
             include_more_specifics: true,
diff --git a/examples/multi_thread_multi_prefix.rs b/examples/multi_thread_multi_prefix.rs
index 7015aeae..a132172d 100644
--- a/examples/multi_thread_multi_prefix.rs
+++ b/examples/multi_thread_multi_prefix.rs
@@ -1,5 +1,10 @@
 use inetnum::addr::Prefix;
 use log::trace;
+use rotonda_store::match_options::{IncludeHistory, MatchOptions, MatchType};
+use rotonda_store::prefix_record::{Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::IntoIpAddr;
 
 use std::sync::atomic::AtomicU32;
 use std::sync::Arc;
@@ -8,8 +13,7 @@ use std::time::Duration;
 
 use rand::Rng;
 
-use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::{IncludeHistory, IntoIpAddr, MatchOptions, MemoryOnlyConfig, StarCastRib, Record, RouteStatus};
+use rotonda_store::test_types::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
@@ -33,8 +37,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             .name(i.to_string())
             .spawn(
                 move || -> Result<(), Box<dyn std::error::Error + Send>> {
-                    let mut rng = rand::rng(); 
-                    
+                    let mut rng = rand::rng();
+
                     // println!("park thread {}", i);
                     thread::park();
 
@@ -42,14 +46,21 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
                     let mut x = 0;
                     loop {
-                        let pfx = Prefix::new_relaxed(pfx_int.clone().load(std::sync::atomic::Ordering::Relaxed).into_ipaddr(), 32).unwrap();
+                        let pfx = Prefix::new_relaxed(
+                            pfx_int
+                                .clone()
+                                .load(std::sync::atomic::Ordering::Relaxed)
+                                .into_ipaddr(),
+                            32,
+                        )
+                        .unwrap();
                         let guard = &crossbeam_epoch::pin();
                         while x < 100 {
                             let asn = PrefixAs::new_from_u32(rng.random());
                             match tree_bitmap.insert(
                                 &pfx,
                                 Record::new(0, 0, RouteStatus::Active, asn),
-                                None
+                                None,
                             ) {
                                 Ok(metrics) => {
                                     if metrics.prefix_new {
@@ -63,18 +74,20 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                     println!("{}", e);
                                 }
                             };
-                            let _s_spfx = tree_bitmap.match_prefix(
-                                &pfx,
-                                &MatchOptions {
-                                    match_type: rotonda_store::MatchType::ExactMatch,
-                                    include_withdrawn: true,
-                                    include_less_specifics: true,
-                                    include_more_specifics: true,
-                                    mui: None,
-                                    include_history: IncludeHistory::None,
-                                },
-                                guard,
-                            ).prefix_meta;
+                            let _s_spfx = tree_bitmap
+                                .match_prefix(
+                                    &pfx,
+                                    &MatchOptions {
+                                        match_type: MatchType::ExactMatch,
+                                        include_withdrawn: true,
+                                        include_less_specifics: true,
+                                        include_more_specifics: true,
+                                        mui: None,
+                                        include_history: IncludeHistory::None,
+                                    },
+                                    guard,
+                                )
+                                .prefix_meta;
                             x += 1;
                         }
 
@@ -84,9 +97,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                         thread::park();
                         // thread::sleep(Duration::from_secs(3));
                         println!("wake thread {}", i);
-                        println!("prefix count {:?}", tree_bitmap.prefixes_count());
+                        println!(
+                            "prefix count {:?}",
+                            tree_bitmap.prefixes_count()
+                        );
                         x = 0;
-
                     }
                 },
             )
diff --git a/examples/multi_thread_single_prefix.rs b/examples/multi_thread_single_prefix.rs
index 660a2173..f56eefd4 100644
--- a/examples/multi_thread_single_prefix.rs
+++ b/examples/multi_thread_single_prefix.rs
@@ -1,4 +1,9 @@
 use log::trace;
+use rotonda_store::match_options::{IncludeHistory, MatchOptions, MatchType};
+use rotonda_store::prefix_record::{Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::IntoIpAddr;
 
 use std::sync::Arc;
 use std::thread;
@@ -6,11 +11,7 @@ use std::time::Duration;
 
 use rand::Rng;
 
-use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::{
-    IncludeHistory, IntoIpAddr, MatchOptions, MemoryOnlyConfig, Record,
-    RouteStatus, StarCastRib,
-};
+use rotonda_store::test_types::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
@@ -25,10 +26,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         32,
     );
 
-    let threads = (0..16).enumerate().map(|(i, _)| {
-        let tree_bitmap = tree_bitmap.clone();
+    let threads =
+        (0..16).enumerate().map(|(i, _)| {
+            let tree_bitmap = tree_bitmap.clone();
 
-        std::thread::Builder::new()
+            std::thread::Builder::new()
             .name(i.to_string())
             .spawn(
                 move || -> Result<(), Box<dyn std::error::Error + Send>> {
@@ -64,7 +66,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                             let _s_spfx = tree_bitmap.match_prefix(
                         &pfx.unwrap(),
                         &MatchOptions {
-                            match_type: rotonda_store::MatchType::ExactMatch,
+                            match_type: MatchType::ExactMatch,
                             include_withdrawn: true,
                             include_less_specifics: true,
                             include_more_specifics: true,
@@ -90,7 +92,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 },
             )
             .unwrap()
-    });
+        });
 
     threads.for_each(|t| {
         t.thread().unpark();
diff --git a/examples/numbers_treebitmap.rs b/examples/numbers_treebitmap.rs
index 8e667284..ab321e6d 100644
--- a/examples/numbers_treebitmap.rs
+++ b/examples/numbers_treebitmap.rs
@@ -1,8 +1,8 @@
 use inetnum::addr::Prefix;
-use rotonda_store::meta_examples::PrefixAs;
-use rotonda_store::{
-    MemoryOnlyConfig, PrefixRecord, Record, RouteStatus, StarCastRib,
-};
+use rotonda_store::prefix_record::{PrefixRecord, Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::test_types::PrefixAs;
 
 use std::env;
 use std::error::Error;
diff --git a/examples/real_single_thread_24.rs b/examples/real_single_thread_24.rs
index 718540ba..5321101e 100644
--- a/examples/real_single_thread_24.rs
+++ b/examples/real_single_thread_24.rs
@@ -1,22 +1,23 @@
 use inetnum::addr::Prefix;
 use log::trace;
-use rotonda_store::{IntoIpAddr, MemoryOnlyConfig, Record, RouteStatus};
+use rotonda_store::prefix_record::{Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::IntoIpAddr;
 use std::thread;
 use std::time::Duration;
 
 use rand::Rng;
 
-use rotonda_store::meta_examples::PrefixAs;
+use rotonda_store::test_types::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
     env_logger::init();
 
     trace!("Starting one-threaded yolo testing....");
-    let tree_bitmap = rotonda_store::StarCastRib::<
-        PrefixAs,
-        MemoryOnlyConfig,
-    >::try_default()?;
+    let tree_bitmap =
+        StarCastRib::<PrefixAs, MemoryOnlyConfig>::try_default()?;
 
     let mut pfx_int = 0_u32;
 
diff --git a/examples/single_thread_24.rs b/examples/single_thread_24.rs
index 32b70e38..de78c553 100644
--- a/examples/single_thread_24.rs
+++ b/examples/single_thread_24.rs
@@ -1,14 +1,16 @@
 use inetnum::addr::Prefix;
 use log::trace;
-use rotonda_store::{
-    IntoIpAddr, MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
-};
+use rotonda_store::prefix_record::Record;
+use rotonda_store::prefix_record::RouteStatus;
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::IntoIpAddr;
 use std::thread;
 use std::time::Duration;
 
 use rand::Rng;
 
-use rotonda_store::meta_examples::PrefixAs;
+use rotonda_store::test_types::PrefixAs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     #[cfg(feature = "cli")]
diff --git a/examples/treebitmap.rs b/examples/treebitmap.rs
index 80103f8a..feabf3e0 100644
--- a/examples/treebitmap.rs
+++ b/examples/treebitmap.rs
@@ -1,12 +1,19 @@
 use inetnum::addr::Prefix;
-use rotonda_store::{
-    epoch, meta_examples::NoMeta, IncludeHistory, IntoIpAddr, MatchOptions,
-    MatchType, MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
-};
+use rotonda_store::match_options::IncludeHistory;
+use rotonda_store::match_options::MatchOptions;
+use rotonda_store::match_options::MatchType;
+use rotonda_store::prefix_record::Record;
+use rotonda_store::prefix_record::RouteStatus;
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
+use rotonda_store::test_types::NoMeta;
+use rotonda_store::IntoIpAddr;
 
 type Prefix4<'a> = Prefix;
 
-fn main() -> Result<(), Box<dyn std::error::Error>> {
+type Type = Result<(), Box<dyn std::error::Error>>;
+
+fn main() -> Type {
     let tree_bitmap = StarCastRib::<_, MemoryOnlyConfig>::try_default()?;
     let pfxs = vec![
         Prefix::new(
@@ -290,7 +297,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     ] {
         println!("search for: {:?}", spfx);
         // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
-        let guard = &epoch::pin();
+        let guard = &rotonda_store::epoch::pin();
         let s_spfx = tree_bitmap.match_prefix(
             &spfx.unwrap(),
             &MatchOptions {
diff --git a/src/bin/cli.rs b/src/bin/cli.rs
index 0b35eac0..593a573d 100644
--- a/src/bin/cli.rs
+++ b/src/bin/cli.rs
@@ -1,14 +1,14 @@
 #![cfg(feature = "cli")]
 use ansi_term::Colour;
-use rotonda_store::{
-    epoch, IncludeHistory, MatchOptions, MatchType, MemoryOnlyConfig,
-    PrefixRecord, Record, RouteStatus, StarCastRib,
-};
+use rotonda_store::match_options::{IncludeHistory, MatchOptions, MatchType};
+use rotonda_store::prefix_record::{PrefixRecord, Record, RouteStatus};
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 
 use inetnum::addr::Prefix;
-use rotonda_store::meta_examples::PrefixAs;
+use rotonda_store::test_types::PrefixAs;
 use rustyline::history::DefaultHistory;
 
 use std::env;
@@ -91,7 +91,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // tree_bitmap.print_funky_stats();
     // let locks = tree_bitmap.acquire_prefixes_rwlock_read();
-    let guard = &epoch::pin();
+    let guard = &rotonda_store::epoch::pin();
 
     let mut rl = Editor::<(), DefaultHistory>::new()?;
     if rl.load_history("/tmp/rotonda-store-history.txt").is_err() {
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index cb2d83b5..ab8e7617 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -1,15 +1,3 @@
-use rotonda_store::Config;
-use rotonda_store::MemoryOnlyConfig;
-use rotonda_store::PersistHistoryConfig;
-use rotonda_store::PersistOnlyConfig;
-use rotonda_store::PersistStrategy;
-use rotonda_store::Record;
-use rotonda_store::RouteStatus;
-use rotonda_store::StarCastRib;
-use rotonda_store::UpsertReport;
-use rotonda_store::WriteAheadConfig;
-use routecore::bgp::aspath::HopPath;
-use routecore::bgp::message::update_builder::StandardCommunitiesList;
 use std::collections::BTreeSet;
 use std::fmt;
 use std::fs::File;
@@ -19,16 +7,31 @@ use std::time::Instant;
 use clap::Parser;
 use inetnum::addr::Prefix;
 use memmap2::Mmap;
-use rayon::iter::ParallelBridge;
-use rayon::iter::ParallelIterator;
-use rayon::prelude::*;
-use routecore::bgp::message::PduParseInfo;
-use routecore::bgp::path_attributes::OwnedPathAttributes;
-use routecore::mrt::MrtFile;
+use rayon::{
+    iter::{ParallelBridge, ParallelIterator},
+    prelude::*,
+};
+use rotonda_store::{
+    prefix_record::{Meta, Record, RouteStatus},
+    rib::{
+        config::{
+            Config, MemoryOnlyConfig, PersistHistoryConfig,
+            PersistOnlyConfig, PersistStrategy, WriteAheadConfig,
+        },
+        StarCastRib,
+    },
+    stats::UpsertReport,
+};
+use routecore::{
+    bgp::{
+        aspath::HopPath,
+        message::{update_builder::StandardCommunitiesList, PduParseInfo},
+        path_attributes::OwnedPathAttributes,
+    },
+    mrt::{MrtFile, RibEntryIterator, TableDumpIterator},
+};
 
 use rand::seq::SliceRandom;
-use routecore::mrt::RibEntryIterator;
-use routecore::mrt::TableDumpIterator;
 
 #[derive(Clone, Debug)]
 struct PaBytes(Vec<u8>);
@@ -51,7 +54,7 @@ impl From<Vec<u8>> for PaBytes {
     }
 }
 
-impl rotonda_store::Meta for PaBytes {
+impl Meta for PaBytes {
     type Orderable<'a> = u32;
 
     type TBI = u32;
@@ -176,7 +179,7 @@ struct Cli {
 
 type Type = rotonda_store::errors::PrefixStoreError;
 
-fn insert<C: Config, T: rotonda_store::Meta>(
+fn insert<C: Config, T: Meta>(
     store: &StarCastRib<T, C>,
     prefix: &Prefix,
     mui: u32,
diff --git a/src/lib.rs b/src/lib.rs
index 6a3cf622..1f1264cb 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,14 +1,26 @@
-//! A library that provides abstractions for a BGP Routing Information Base (RIB) for different AFI/SAFI types, as a database.
+//! A library that provides abstractions for a BGP Routing Information Base
+//! (RIB) for different AFI/SAFI types, as a database.
 //!
-//! The data structures provides by this crate can be used to store and query routes (or other metadata keyed on IP prefixes, or a comparable bitarray) in memory and on-disk, for both current and historical data.
+//! The data structures provided by this crate can be used to store and query
+//! routes (and route-like data) in memory and on-disk, for both current and
+//! historical data.
 //!
-//! [^1]:[Paper](https://www.cs.cornell.edu/courses/cs419/2005sp/tree-bitmap.pdf).
-//! [^2]: Read more about the data-structure in this [blogpost](https://blog.nlnetlabs.nl/donkeys-mules-horses/).
-
+//! The main data structures that this crate implements are firstly a tree
+//! bitmap, largely as described in this paper[^1] - but with a twist.
+//! There's also a blog post[^2] about the tree bitmap, and similar data
+//! structures. Secondly, this repo implements a concurrent chained hash
+//! table (called `cht` throughout this repo), geared towards keys that are
+//! "prefix-like", i.e. variable-length bitfields, that fit within a
+//! primitive integer type.
+//!
+//! The log-structured merge tree ("lsm_tree") used in this library is
+//! provided by the `lsm_tree` crate - the crate that powers `fjall`.
+//!
+//! [^1]: <https://www.cs.cornell.edu/courses/cs419/2005sp/tree-bitmap.pdf>
+//! [^2]: <https://blog.nlnetlabs.nl/donkeys-mules-horses/>
 mod cht;
 mod lsm_tree;
 mod prefix_cht;
-mod rib;
 mod tree_bitmap;
 mod types;
 
@@ -24,27 +36,29 @@ pub use inetnum::addr;
 
 // Public Interfaces on the root of the crate
 
-pub use rib::starcast::StarCastRib;
-pub use rib::starcast_af::{
-    Config, MemoryOnlyConfig, PersistHistoryConfig, PersistOnlyConfig,
-    PersistStrategy, UpsertReport, WriteAheadConfig,
-};
+/// RIBs for various AFI/SAFI types
+pub mod rib;
+
+/// Types used to create match queries on a RIB
+pub use types::match_options;
+
+/// Record, Record Iterator and related types/traits
+pub use types::prefix_record;
+
+/// Error types returned by a RIB
+pub use types::errors;
+
 pub use types::af::AddressFamily;
 pub use types::af::IPv4;
 pub use types::af::IPv6;
+
+/// Trait that describes the conversion of a u32 or u128 in to a IPv4, or IPV6
+/// respectively.
 pub use types::af::IntoIpAddr;
-pub use types::errors;
-pub use types::match_options::{
-    IncludeHistory, MatchOptions, MatchType, QueryResult,
-};
 
-#[doc(hidden)]
-pub use types::meta_examples;
+/// Statistics and metrics types returned by methods on a RIB
+pub use types::stats;
+
+// Used in tests
 #[doc(hidden)]
 pub use types::test_types;
-
-pub use types::prefix_record::Meta;
-pub use types::prefix_record::PublicPrefixRecord as PrefixRecord;
-pub use types::prefix_record::PublicRecord as Record;
-pub use types::route_status::RouteStatus;
-pub use types::stats;
diff --git a/src/lsm_tree/mod.rs b/src/lsm_tree/mod.rs
index d2301623..f86226d8 100644
--- a/src/lsm_tree/mod.rs
+++ b/src/lsm_tree/mod.rs
@@ -12,12 +12,11 @@ use zerocopy::{
     Unaligned, U32, U64,
 };
 
-use crate::rib::Counters;
+use crate::prefix_record::Meta;
+use crate::stats::Counters;
 use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
-use crate::types::AddressFamily;
-use crate::types::PublicRecord;
+use crate::types::{AddressFamily, Record};
 use crate::types::{PrefixId, RouteStatus};
-use crate::Meta;
 
 pub(crate) trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
     TryFromBytes + KnownLayout + IntoBytes + Unaligned + Immutable
@@ -732,7 +731,7 @@ impl<
     pub(crate) fn persist_record_w_long_key<M: Meta>(
         &self,
         prefix: PrefixId<AF>,
-        record: &PublicRecord<M>,
+        record: &Record<M>,
     ) {
         self.insert(
             LongKey::from((
@@ -749,7 +748,7 @@ impl<
     pub(crate) fn persist_record_w_short_key<M: Meta>(
         &self,
         prefix: PrefixId<AF>,
-        record: &PublicRecord<M>,
+        record: &Record<M>,
     ) {
         trace!("Record to persist {}", record);
         let mut value = ValueHeader {
diff --git a/src/macros.rs b/src/macros.rs
index 12528227..7b1fcf20 100644
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -6,10 +6,11 @@ macro_rules! all_strategies {
         $(
             #[test]
             fn $fn_name() -> Result<(), Box<dyn std::error::Error>> {
-                use rotonda_store::{
+                use rotonda_store::rib::config::{
                         MemoryOnlyConfig,
                         PersistOnlyConfig,
-                        PersistHistoryConfig, WriteAheadConfig
+                        PersistHistoryConfig,
+                        WriteAheadConfig
                     };
 
                 //------- Default (MemoryOnly)
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index c28d84c4..94e04cce 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -10,15 +10,15 @@ use log::{debug, log_enabled, trace};
 use roaring::RoaringBitmap;
 
 use crate::cht::{nodeset_size, prev_node_size};
-use crate::RouteStatus;
+use crate::prefix_record::Meta;
+use crate::stats::UpsertReport;
+use crate::types::RouteStatus;
 use crate::{
     cht::{Cht, OnceBoxSlice, Value},
-    rib::starcast_af::UpsertReport,
     types::{
-        errors::PrefixStoreError, prefix_record::PublicRecord, AddressFamily,
+        errors::PrefixStoreError, prefix_record::Record, AddressFamily,
         PrefixId,
     },
-    Meta,
 };
 
 // ----------- MultiMap ------------------------------------------------------
@@ -61,21 +61,17 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         &self,
         mui: u32,
         include_withdrawn: bool,
-    ) -> Option<PublicRecord<M>> {
+    ) -> Option<Record<M>> {
         let c_map = Arc::clone(&self.0);
         let record_map = c_map.lock().unwrap();
 
-        record_map
-            .get(&mui)
-            .and_then(|r| -> Option<PublicRecord<M>> {
-                if include_withdrawn
-                    || r.route_status() == RouteStatus::Active
-                {
-                    Some(PublicRecord::from((mui, r)))
-                } else {
-                    None
-                }
-            })
+        record_map.get(&mui).and_then(|r| -> Option<Record<M>> {
+            if include_withdrawn || r.route_status() == RouteStatus::Active {
+                Some(Record::from((mui, r)))
+            } else {
+                None
+            }
+        })
     }
 
     pub fn best_backup(&self, tbi: M::TBI) -> (Option<u32>, Option<u32>) {
@@ -94,7 +90,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         mui: u32,
         bmin: &RoaringBitmap,
         rewrite_status: RouteStatus,
-    ) -> Option<PublicRecord<M>> {
+    ) -> Option<Record<M>> {
         let c_map = Arc::clone(&self.0);
         let record_map = c_map.lock().unwrap();
         record_map.get(&mui).map(|r| {
@@ -104,7 +100,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
             if bmin.contains(mui) {
                 r.set_route_status(rewrite_status);
             }
-            PublicRecord::from((mui, &r))
+            Record::from((mui, &r))
         })
     }
 
@@ -113,7 +109,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         mui: u32,
         include_withdrawn: bool,
         bmin: &RoaringBitmap,
-    ) -> Option<PublicRecord<M>> {
+    ) -> Option<Record<M>> {
         match include_withdrawn {
             false => self.get_record_for_mui(mui, include_withdrawn),
             true => self.get_record_for_mui_with_rewritten_status(
@@ -131,7 +127,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         mui: Option<u32>,
         include_withdrawn: bool,
         bmin: &RoaringBitmap,
-    ) -> Option<Vec<PublicRecord<M>>> {
+    ) -> Option<Vec<Record<M>>> {
         if let Some(mui) = mui {
             self.get_filtered_record_for_mui(mui, include_withdrawn, bmin)
                 .map(|r| vec![r])
@@ -168,7 +164,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         &self,
         bmin: &RoaringBitmap,
         rewrite_status: RouteStatus,
-    ) -> Vec<PublicRecord<M>> {
+    ) -> Vec<Record<M>> {
         let c_map = Arc::clone(&self.0);
         let record_map = c_map.lock().unwrap();
         record_map
@@ -178,17 +174,17 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
                 if bmin.contains(*r.0) {
                     rec.set_route_status(rewrite_status);
                 }
-                PublicRecord::from((*r.0, &rec))
+                Record::from((*r.0, &rec))
             })
             .collect::<Vec<_>>()
     }
 
-    pub fn _as_records(&self) -> Vec<PublicRecord<M>> {
+    pub fn _as_records(&self) -> Vec<Record<M>> {
         let c_map = Arc::clone(&self.0);
         let record_map = c_map.lock().unwrap();
         record_map
             .iter()
-            .map(|r| PublicRecord::from((*r.0, r.1)))
+            .map(|r| Record::from((*r.0, r.1)))
             .collect::<Vec<_>>()
     }
 
@@ -198,7 +194,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     pub fn as_active_records_not_in_bmin(
         &self,
         bmin: &RoaringBitmap,
-    ) -> Vec<PublicRecord<M>> {
+    ) -> Vec<Record<M>> {
         let c_map = Arc::clone(&self.0);
         let record_map = c_map.lock().unwrap();
         record_map
@@ -207,7 +203,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
                 if r.1.route_status() == RouteStatus::Active
                     && !bmin.contains(*r.0)
                 {
-                    Some(PublicRecord::from((*r.0, r.1)))
+                    Some(Record::from((*r.0, r.1)))
                 } else {
                     None
                 }
@@ -242,7 +238,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     #[allow(clippy::type_complexity)]
     pub(crate) fn upsert_record(
         &self,
-        new_rec: PublicRecord<M>,
+        new_rec: Record<M>,
     ) -> Result<(Option<(MultiMapValue<M>, usize)>, usize), PrefixStoreError>
     {
         let (mut record_map, retry_count) = self.guard_with_retry(0);
@@ -305,8 +301,8 @@ impl<M: Meta> std::fmt::Display for MultiMapValue<M> {
     }
 }
 
-impl<M: Meta> From<PublicRecord<M>> for MultiMapValue<M> {
-    fn from(value: PublicRecord<M>) -> Self {
+impl<M: Meta> From<Record<M>> for MultiMapValue<M> {
+    fn from(value: Record<M>) -> Self {
         Self {
             ltime: value.ltime,
             route_status: value.status,
@@ -315,7 +311,7 @@ impl<M: Meta> From<PublicRecord<M>> for MultiMapValue<M> {
     }
 }
 
-impl<M: Meta> From<(u32, &MultiMapValue<M>)> for PublicRecord<M> {
+impl<M: Meta> From<(u32, &MultiMapValue<M>)> for Record<M> {
     fn from(value: (u32, &MultiMapValue<M>)) -> Self {
         Self {
             multi_uniq_id: value.0,
@@ -524,7 +520,7 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
         mui: Option<u32>,
         include_withdrawn: bool,
         bmin: &RoaringBitmap,
-    ) -> Option<Vec<PublicRecord<M>>> {
+    ) -> Option<Vec<Record<M>>> {
         let mut prefix_set = self.0.root_for_len(prefix.get_len());
         let mut level: u8 = 0;
         let backoff = Backoff::new();
@@ -569,7 +565,7 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
     pub(crate) fn upsert_prefix(
         &self,
         prefix: PrefixId<AF>,
-        record: PublicRecord<M>,
+        record: Record<M>,
         update_path_selections: Option<M::TBI>,
         guard: &Guard,
     ) -> Result<(UpsertReport, Option<MultiMapValue<M>>), PrefixStoreError>
diff --git a/src/rib/config.rs b/src/rib/config.rs
index c50d768c..d7e031bf 100644
--- a/src/rib/config.rs
+++ b/src/rib/config.rs
@@ -1,9 +1,22 @@
 //------------ Config --------------------------------------------------------
 
 //! Configuration options for a RIB for AFI/SAFIs [IPv4, IPv6] with [Unicast,
-//! Multicast]
-
-/// Defines where records are stored, in-memory and/or persisted (to disk),
+//! Multicast].
+//!
+//! A Configuration is created by picking one of the `*Config` structs in
+//! this module, instantiate it, set some fields on it, and pass it in as an
+//! argument to [new_with_config](super::StarCastRib::new_with_config).
+//!
+//! ```
+//! use rotonda_store::test_types::PrefixAs;
+//! use rotonda_store::rib::StarCastRib;
+//! use rotonda_store::rib::config::PersistOnlyConfig;
+//!
+//! let config = PersistOnlyConfig::default();
+//! let tree_bitmap = StarCastRib::<PrefixAs, _>::new_with_config(config);
+//! ```
+
+/// Defines where records are stored: in-memory and/or persisted (to disk),
 /// and, whether new records for a unique (prefix, mui) pair are overwritten
 /// or persisted.
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
diff --git a/src/rib/mod.rs b/src/rib/mod.rs
index d6a1db80..319943f0 100644
--- a/src/rib/mod.rs
+++ b/src/rib/mod.rs
@@ -1,7 +1,9 @@
+pub mod config;
 pub(crate) mod starcast;
 pub(crate) mod starcast_af;
 pub(crate) mod starcast_af_query;
 
 pub(crate) use starcast::BIT_SPAN_SIZE;
 pub(crate) use starcast::STRIDE_SIZE;
-pub(crate) use starcast_af::Counters;
+
+pub use starcast::StarCastRib;
diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index 21ab4c4d..0142b525 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -4,19 +4,25 @@ use rand::prelude::*;
 
 use crate::{
     epoch,
+    match_options::{MatchOptions, QueryResult},
+    prefix_record::{Meta, PrefixRecord, Record},
+    rib::config::Config,
     types::{errors::PrefixStoreError, PrefixId},
-    AddressFamily, IPv4, IPv6, MatchOptions, Meta, PrefixRecord, QueryResult,
-    Record,
+    AddressFamily, IPv4, IPv6,
 };
 
-use super::starcast_af::{
-    Config, PersistStrategy, StarCastAfRib, StoreStats, UpsertCounters,
-    UpsertReport,
-};
+use super::starcast_af::StarCastAfRib;
+use crate::rib::config::PersistStrategy;
+use crate::stats::{StoreStats, UpsertCounters, UpsertReport};
 
 pub const STRIDE_SIZE: u8 = 4;
 pub const BIT_SPAN_SIZE: u8 = 32;
 
+/// A RIB that stores routes (and/or other data) for [`IPv4`,
+/// `IPv6`]/[`Unicast`, `Multicast`], i.e. AFI/SAFI types `{1,2}/{1,2}`.
+///
+/// Routes can be kept in memory, persisted to disk, or both. Also, historical
+/// records can be persisted.
 pub struct StarCastRib<M: Meta, C: Config> {
     v4: StarCastAfRib<IPv4, M, 9, 33, C, 18>,
     v6: StarCastAfRib<IPv6, M, 33, 129, C, 30>,
@@ -24,12 +30,22 @@ pub struct StarCastRib<M: Meta, C: Config> {
 }
 
 impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
+    /// Create a new RIB with a default configuration. The default
+    /// configuration uses the `MemoryOnly` persistence strategy.
+    ///
+    /// This method is really infallible, but we return a result anyway to be
+    /// in line with the `new_with_config` method.
     pub fn try_default() -> Result<Self, PrefixStoreError> {
         let config = C::default();
         Self::new_with_config(config)
             .map_err(|_| PrefixStoreError::StoreNotReadyError)
     }
 
+    /// Create a new RIB with the specified configuration.
+    ///
+    /// Creation may fail for all strategies that persist to disk, e.g.
+    /// the persistence path does not exist, it doesn't have the correct
+    /// permissions, etc.
     pub fn new_with_config(
         config: C,
     ) -> Result<Self, Box<dyn std::error::Error>> {
diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index 0a9e0bae..f97d6595 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -1,317 +1,26 @@
 use std::path::Path;
-use std::sync::atomic::{AtomicUsize, Ordering};
 
 use inetnum::addr::Prefix;
 use log::{info, trace};
 
-use crossbeam_epoch::{self as epoch};
-use epoch::Guard;
+use crate::prefix_record::Meta;
+use crate::rib::config::PersistStrategy;
+use crate::stats::{Counters, UpsertCounters, UpsertReport};
+use crate::{epoch, Guard};
 use zerocopy::TryFromBytes;
 
 use crate::prefix_cht::cht::PrefixCht;
-use crate::stats::CreatedNodes;
 use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
 use crate::types::{PrefixId, RouteStatus};
 use crate::TreeBitMap;
 use crate::{lsm_tree::LongKey, LsmTree};
-use crate::{
-    types::errors::PrefixStoreError, types::prefix_record::PublicRecord,
-};
+use crate::{types::errors::PrefixStoreError, types::prefix_record::Record};
 
-use crate::{IPv4, IPv6, Meta};
+use crate::{IPv4, IPv6};
 
 use crate::AddressFamily;
 
-//------------ Config --------------------------------------------------------
-
-/// Defines where records are stored, in-memory and/or persisted (to disk),
-/// and, whether new records for a unique (prefix, mui) pair are overwritten
-/// or persisted.
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub enum PersistStrategy {
-    /// Current records are stored both in-memory and persisted. Historical
-    /// records are persisted.
-    WriteAhead,
-    /// Current records are stored in-memory, historical records are
-    /// persisted.
-    PersistHistory,
-    /// Current records are stored in-memory, historical records are discarded
-    /// when newer records appear.
-    MemoryOnly,
-    /// Current records are persisted immediately. No records are stored in
-    /// memory. Historical records are discarded when newer records appear.
-    PersistOnly,
-}
-
-pub trait Config: Clone + Default + std::fmt::Debug {
-    fn persist_strategy(&self) -> PersistStrategy;
-    fn persist_path(&self) -> Option<String>;
-    fn set_persist_path(&mut self, path: String);
-}
-
-//------------ MemoryOnlyConfig ----------------------------------------------
-
-#[derive(Copy, Clone, Debug)]
-pub struct MemoryOnlyConfig;
-
-impl Config for MemoryOnlyConfig {
-    fn persist_strategy(&self) -> PersistStrategy {
-        PersistStrategy::MemoryOnly
-    }
-
-    fn persist_path(&self) -> Option<String> {
-        None
-    }
-
-    fn set_persist_path(&mut self, _: String) {
-        unimplemented!()
-    }
-}
-
-impl Default for MemoryOnlyConfig {
-    fn default() -> Self {
-        Self
-    }
-}
-
-//------------ PeristOnlyConfig ----------------------------------------------
-
-impl Default for PersistOnlyConfig {
-    fn default() -> Self {
-        Self {
-            persist_path: "/tmp/rotonda/".to_string(),
-        }
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct PersistOnlyConfig {
-    persist_path: String,
-}
-
-impl Config for PersistOnlyConfig {
-    fn persist_strategy(&self) -> PersistStrategy {
-        PersistStrategy::PersistOnly
-    }
-
-    fn persist_path(&self) -> Option<String> {
-        Some(self.persist_path.clone())
-    }
-
-    fn set_persist_path(&mut self, path: String) {
-        self.persist_path = path;
-    }
-}
-
-//------------ WriteAheadConfig ----------------------------------------------
-
-impl Default for WriteAheadConfig {
-    fn default() -> Self {
-        Self {
-            persist_path: "/tmp/rotonda/".to_string(),
-        }
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct WriteAheadConfig {
-    persist_path: String,
-}
-
-impl Config for WriteAheadConfig {
-    fn persist_strategy(&self) -> PersistStrategy {
-        PersistStrategy::WriteAhead
-    }
-
-    fn persist_path(&self) -> Option<String> {
-        Some(self.persist_path.clone())
-    }
-
-    fn set_persist_path(&mut self, path: String) {
-        self.persist_path = path;
-    }
-}
-
-//------------ PersistHistoryConfig ------------------------------------------
-
-#[derive(Clone, Debug)]
-pub struct PersistHistoryConfig {
-    persist_path: String,
-}
-
-impl Config for PersistHistoryConfig {
-    fn persist_strategy(&self) -> PersistStrategy {
-        PersistStrategy::PersistHistory
-    }
-
-    fn persist_path(&self) -> Option<String> {
-        Some(self.persist_path.clone())
-    }
-
-    fn set_persist_path(&mut self, path: String) {
-        self.persist_path = path;
-    }
-}
-
-impl Default for PersistHistoryConfig {
-    fn default() -> Self {
-        Self {
-            persist_path: "/tmp/rotonda/".to_string(),
-        }
-    }
-}
-
-//------------ Counters -----------------------------------------------------
-
-#[derive(Debug)]
-pub struct Counters {
-    // number of created nodes in the in-mem tree
-    nodes: AtomicUsize,
-    // number of unique prefixes in the store
-    prefixes: [AtomicUsize; 129],
-    // number of unique (prefix, mui) values inserted in the in-mem tree
-    routes: AtomicUsize,
-}
-
-impl Counters {
-    pub fn get_nodes_count(&self) -> usize {
-        self.nodes.load(Ordering::Relaxed)
-    }
-
-    pub fn inc_nodes_count(&self) {
-        self.nodes.fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn get_prefixes_count(&self) -> Vec<usize> {
-        self.prefixes
-            .iter()
-            .map(|pc| pc.load(Ordering::Relaxed))
-            .collect::<Vec<_>>()
-    }
-
-    pub fn inc_prefixes_count(&self, len: u8) {
-        self.prefixes[len as usize].fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn _dec_prefixes_count(&self, len: u8) {
-        self.prefixes[len as usize].fetch_sub(1, Ordering::Relaxed);
-    }
-
-    pub fn get_prefix_stats(&self) -> Vec<CreatedNodes> {
-        self.prefixes
-            .iter()
-            .enumerate()
-            .filter_map(|(len, count)| {
-                let count = count.load(Ordering::Relaxed);
-                if count != 0 {
-                    Some(CreatedNodes {
-                        depth_level: len as u8,
-                        count,
-                    })
-                } else {
-                    None
-                }
-            })
-            .collect()
-    }
-
-    pub fn inc_routes_count(&self) {
-        self.routes.fetch_add(1, Ordering::Relaxed);
-    }
-}
-
-impl Default for Counters {
-    fn default() -> Self {
-        let mut prefixes: Vec<AtomicUsize> = Vec::with_capacity(129);
-        for _ in 0..=128 {
-            prefixes.push(AtomicUsize::new(0));
-        }
-
-        Self {
-            nodes: AtomicUsize::new(0),
-            prefixes: prefixes.try_into().unwrap(),
-            routes: AtomicUsize::new(0),
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct UpsertCounters {
-    // number of unique inserted prefixes|routes in the in-mem tree
-    in_memory_count: usize,
-    // number of unique persisted prefixes|routes
-    persisted_count: usize,
-    // total number of unique inserted prefixes|routes in the RIB
-    total_count: usize,
-}
-
-impl UpsertCounters {
-    pub fn in_memory(&self) -> usize {
-        self.in_memory_count
-    }
-
-    pub fn persisted(&self) -> usize {
-        self.persisted_count
-    }
-
-    pub fn total(&self) -> usize {
-        self.total_count
-    }
-}
-
-impl std::fmt::Display for UpsertCounters {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "Unique Items in-memory:\t{}", self.in_memory_count)?;
-        write!(f, "Unique persisted Items:\t{}", self.persisted_count)?;
-        write!(f, "Total inserted Items:\t{}", self.total_count)
-    }
-}
-
-impl std::ops::AddAssign for UpsertCounters {
-    fn add_assign(&mut self, rhs: Self) {
-        self.in_memory_count += rhs.in_memory_count;
-        self.persisted_count += rhs.persisted_count;
-        self.total_count += rhs.total_count;
-    }
-}
-
-impl std::ops::Add for UpsertCounters {
-    type Output = UpsertCounters;
-
-    fn add(self, rhs: Self) -> Self::Output {
-        Self {
-            in_memory_count: self.in_memory_count + rhs.in_memory_count,
-            persisted_count: self.persisted_count + rhs.persisted_count,
-            total_count: self.total_count + rhs.total_count,
-        }
-    }
-}
-
-//------------ StoreStats ----------------------------------------------
-
-#[derive(Debug)]
-pub struct StoreStats {
-    pub v4: Vec<CreatedNodes>,
-    pub v6: Vec<CreatedNodes>,
-}
-
-//------------ UpsertReport --------------------------------------------------
-
-#[derive(Debug)]
-pub struct UpsertReport {
-    // Indicates the number of Atomic Compare-and-Swap operations were
-    // necessary to create/update the Record entry. High numbers indicate
-    // contention.
-    pub cas_count: usize,
-    // Indicates whether this was the first mui record for this prefix was
-    // created. So, the prefix did not exist before hand.
-    pub prefix_new: bool,
-    // Indicates whether this mui was new for this prefix. False means an old
-    // value was overwritten.
-    pub mui_new: bool,
-    // The number of mui records for this prefix after the upsert operation.
-    pub mui_count: usize,
-}
+use super::config::Config;
 
 // ----------- Rib -----------------------------------------------------------
 //
@@ -379,7 +88,7 @@ impl<
     pub(crate) fn insert(
         &self,
         prefix: PrefixId<AF>,
-        record: PublicRecord<M>,
+        record: Record<M>,
         update_path_selections: Option<M::TBI>,
     ) -> Result<UpsertReport, PrefixStoreError> {
         trace!("try insertingf {:?}", prefix);
@@ -411,7 +120,7 @@ impl<
     fn upsert_prefix(
         &self,
         prefix: PrefixId<AF>,
-        record: PublicRecord<M>,
+        record: Record<M>,
         update_path_selections: Option<M::TBI>,
         guard: &Guard,
     ) -> Result<UpsertReport, PrefixStoreError> {
@@ -441,7 +150,7 @@ impl<
                         if let Some(persist_tree) = &self.persist_tree {
                             persist_tree.persist_record_w_long_key(
                                 prefix,
-                                &PublicRecord::from((mui, &rec)),
+                                &Record::from((mui, &rec)),
                             );
                         }
                     }
@@ -688,7 +397,7 @@ impl<
     pub fn prefixes_iter<'a>(
         &'a self,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + 'a {
+    ) -> impl Iterator<Item = (Prefix, Vec<Record<M>>)> + 'a {
         self.tree_bitmap.prefixes_iter().map(|p| {
             (
                 p,
@@ -706,7 +415,7 @@ impl<
 
     pub(crate) fn persist_prefixes_iter(
         &self,
-    ) -> impl Iterator<Item = (Prefix, Vec<PublicRecord<M>>)> + '_ {
+    ) -> impl Iterator<Item = (Prefix, Vec<Record<M>>)> + '_ {
         self.persist_tree
             .as_ref()
             .map(|tree| {
@@ -726,14 +435,14 @@ impl<
                                         rec,
                                     )
                                     .unwrap();
-                                PublicRecord {
+                                Record {
                                     multi_uniq_id: rec.multi_uniq_id,
                                     ltime: rec.ltime,
                                     status: rec.status,
                                     meta: rec.meta.to_vec().into(),
                                 }
                             })
-                            .collect::<Vec<PublicRecord<M>>>(),
+                            .collect::<Vec<Record<M>>>(),
                     )
                 })
             })
diff --git a/src/rib/starcast_af_query.rs b/src/rib/starcast_af_query.rs
index b1a8d896..513f4c37 100644
--- a/src/rib/starcast_af_query.rs
+++ b/src/rib/starcast_af_query.rs
@@ -3,19 +3,18 @@ use epoch::Guard;
 use log::trace;
 use zerocopy::TryFromBytes;
 
-use crate::rib::starcast_af::{Config, PersistStrategy, StarCastAfRib};
+use crate::match_options::{MatchOptions, MatchType, QueryResult};
 use crate::types::prefix_record::ZeroCopyRecord;
-use crate::types::PublicRecord;
+use crate::types::Record;
 use crate::AddressFamily;
+use crate::{prefix_record::Meta, rib::starcast_af::StarCastAfRib};
 use inetnum::addr::Prefix;
 
-use crate::{Meta, QueryResult};
-
-use crate::{MatchOptions, MatchType};
-
 use crate::types::errors::PrefixStoreError;
 use crate::types::PrefixId;
 
+use super::config::{Config, PersistStrategy};
+
 //------------ Prefix Matching ----------------------------------------------
 
 impl<
@@ -34,7 +33,7 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> Option<Vec<PublicRecord<M>>> {
+    ) -> Option<Vec<Record<M>>> {
         match self.persist_strategy() {
             PersistStrategy::PersistOnly => {
                 trace!("get value from persist_store for {:?}", prefix_id);
@@ -51,7 +50,7 @@ impl<
                                 let record: &ZeroCopyRecord<AF> =
                                     ZeroCopyRecord::try_ref_from_bytes(bytes)
                                         .unwrap();
-                                PublicRecord::<M> {
+                                Record::<M> {
                                     multi_uniq_id: record.multi_uniq_id,
                                     ltime: record.ltime,
                                     status: record.status,
@@ -148,7 +147,7 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<Record<M>>)> + 'a {
         println!("more_specifics_iter_from fn");
         // If the user wanted a specific mui and not withdrawn prefixes, we
         // may return early if the mui is globally withdrawn.
@@ -198,7 +197,7 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
+    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<Record<M>>)> + 'a {
         self.tree_bitmap
             .less_specific_prefix_iter(prefix_id)
             .filter_map(move |p| {
@@ -280,7 +279,7 @@ impl<
         &'a self,
         search_pfx: PrefixId<AF>,
         guard: &Guard,
-    ) -> Option<Result<PublicRecord<M>, PrefixStoreError>> {
+    ) -> Option<Result<Record<M>, PrefixStoreError>> {
         self.prefix_cht
             .non_recursive_retrieve_prefix(search_pfx)
             .0
@@ -365,13 +364,12 @@ impl<AF: AddressFamily, M: Meta> From<TreeQueryResult<AF>>
     }
 }
 
-pub(crate) type FamilyRecord<AF, M> =
-    Vec<(PrefixId<AF>, Vec<PublicRecord<M>>)>;
+pub(crate) type FamilyRecord<AF, M> = Vec<(PrefixId<AF>, Vec<Record<M>>)>;
 
 pub(crate) struct FamilyQueryResult<AF: AddressFamily, M: Meta> {
     pub match_type: MatchType,
     pub prefix: Option<PrefixId<AF>>,
-    pub prefix_meta: Vec<PublicRecord<M>>,
+    pub prefix_meta: Vec<Record<M>>,
     pub less_specifics: Option<FamilyRecord<AF, M>>,
     pub more_specifics: Option<FamilyRecord<AF, M>>,
 }
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 6ce05c0e..764e2470 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -197,6 +197,7 @@ pub(crate) use tree_bitmap_node::{
 
 use crate::cht::{nodeset_size, prev_node_size, Cht, Value};
 use crate::rib::STRIDE_SIZE;
+use crate::stats::Counters;
 use crate::types::{BitSpan, PrefixId};
 use crossbeam_epoch::{Atomic, Guard, Owned, Shared};
 use log::{debug, error, log_enabled, trace};
@@ -207,7 +208,6 @@ use tree_bitmap_node::NewNodeOrIndex;
 use std::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, Ordering};
 use std::{fmt::Debug, marker::PhantomData};
 
-use crate::rib::Counters;
 use crate::types::AddressFamily;
 use atomic_bitmap::{AtomicBitmap, AtomicPfxBitArr, AtomicPtrBitArr};
 
diff --git a/src/tree_bitmap/tree_bitmap_query.rs b/src/tree_bitmap/tree_bitmap_query.rs
index c78ae45d..ab94c12d 100644
--- a/src/tree_bitmap/tree_bitmap_query.rs
+++ b/src/tree_bitmap/tree_bitmap_query.rs
@@ -1,7 +1,7 @@
+use crate::match_options::{MatchOptions, MatchType};
 use crate::types::AddressFamily;
 
 use crate::rib::starcast_af_query::TreeQueryResult;
-use crate::{MatchOptions, MatchType};
 
 use crate::types::PrefixId;
 use crate::TreeBitMap;
diff --git a/src/types/match_options.rs b/src/types/match_options.rs
index f9a52ce9..2bd13e39 100644
--- a/src/types/match_options.rs
+++ b/src/types/match_options.rs
@@ -15,7 +15,7 @@ use super::prefix_record::Meta;
 /// Note that the `match_type` field may be different from the actual
 /// `MatchType` returned from the result.
 ///
-/// See [MultiThreadedStore::match_prefix] for more details.
+/// See [crate::rib::StarCastRib::match_prefix] for more details.
 #[derive(Debug, Clone)]
 pub struct MatchOptions {
     /// The requested [MatchType]
@@ -82,7 +82,7 @@ pub enum IncludeHistory {
 /// This is the result type of a query. It contains the prefix record that was
 /// found in the store, as well as less- or more-specifics as requested.
 ///
-/// See [MultiThreadedStore::match_prefix] for more details.
+/// See [crate::rib::StarCastRib::match_prefix] for more details.
 
 #[derive(Clone, Debug)]
 pub struct QueryResult<M: Meta> {
diff --git a/tests/best-path.rs b/tests/best-path.rs
index 35daa0cf..7298d307 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -2,13 +2,14 @@ use inetnum::addr::Prefix;
 use inetnum::asn::Asn;
 use log::trace;
 use rotonda_store::errors::PrefixStoreError;
-use rotonda_store::IncludeHistory;
-use rotonda_store::MatchOptions;
-use rotonda_store::MemoryOnlyConfig;
-use rotonda_store::Meta;
-use rotonda_store::Record;
-use rotonda_store::RouteStatus;
-use rotonda_store::StarCastRib;
+use rotonda_store::match_options::IncludeHistory;
+use rotonda_store::match_options::MatchOptions;
+use rotonda_store::match_options::MatchType;
+use rotonda_store::prefix_record::Meta;
+use rotonda_store::prefix_record::Record;
+use rotonda_store::prefix_record::RouteStatus;
+use rotonda_store::rib::config::MemoryOnlyConfig;
+use rotonda_store::rib::StarCastRib;
 use routecore::bgp::aspath::HopPath;
 use routecore::bgp::path_attributes::BgpIdentifier;
 use routecore::bgp::path_attributes::PaMap;
@@ -179,7 +180,7 @@ fn test_best_path_1(// tree_bitmap: MultiThreadedStore<Ipv4Route>,
     let res = tree_bitmap.match_prefix(
         &pfx,
         &MatchOptions {
-            match_type: rotonda_store::MatchType::ExactMatch,
+            match_type: MatchType::ExactMatch,
             include_withdrawn: false,
             include_less_specifics: false,
             include_more_specifics: false,
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index c95ed148..0c3054f4 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -2,8 +2,13 @@ use std::{str::FromStr, sync::atomic::Ordering};
 
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
-    meta_examples::NoMeta, test_types::BeBytesAsn, Config, IncludeHistory,
-    MatchOptions, MemoryOnlyConfig, Record, RouteStatus, StarCastRib,
+    match_options::{IncludeHistory, MatchOptions, MatchType},
+    prefix_record::{Record, RouteStatus},
+    rib::{
+        config::{Config, MemoryOnlyConfig},
+        StarCastRib,
+    },
+    test_types::{BeBytesAsn, NoMeta},
 };
 
 mod common {
@@ -352,7 +357,7 @@ fn test_concurrent_updates_1<C: Config + Sync + Send + 'static>(
     );
 
     let match_options = MatchOptions {
-        match_type: rotonda_store::MatchType::ExactMatch,
+        match_type: MatchType::ExactMatch,
         include_withdrawn: true,
         include_less_specifics: false,
         include_more_specifics: false,
@@ -633,7 +638,7 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
         .collect();
 
     let match_options = MatchOptions {
-        match_type: rotonda_store::MatchType::ExactMatch,
+        match_type: MatchType::ExactMatch,
         include_withdrawn: true,
         include_less_specifics: false,
         include_more_specifics: false,
@@ -658,7 +663,7 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
 
     println!("get all prefixes");
     let match_options = MatchOptions {
-        match_type: rotonda_store::MatchType::EmptyMatch,
+        match_type: MatchType::EmptyMatch,
         include_withdrawn: false,
         include_less_specifics: false,
         include_more_specifics: true,
@@ -736,7 +741,7 @@ fn more_specifics_short_lengths() -> Result<(), Box<dyn std::error::Error>> {
         MemoryOnlyConfig,
     >::try_default()?);
     let match_options = MatchOptions {
-        match_type: rotonda_store::MatchType::EmptyMatch,
+        match_type: MatchType::EmptyMatch,
         include_withdrawn: false,
         include_less_specifics: false,
         include_more_specifics: true,
diff --git a/tests/full-table.rs b/tests/full-table.rs
index 87d230f4..336a49b8 100644
--- a/tests/full-table.rs
+++ b/tests/full-table.rs
@@ -3,10 +3,11 @@
 mod tests {
     use inetnum::addr::Prefix;
     use inetnum::asn::Asn;
-    use rotonda_store::Config;
     use rotonda_store::{
-        epoch, IncludeHistory, MatchOptions, MatchType, Meta, PrefixRecord,
-        Record, RouteStatus, StarCastRib,
+        epoch,
+        match_options::{IncludeHistory, MatchOptions, MatchType},
+        prefix_record::{Meta, PrefixRecord, Record, RouteStatus},
+        rib::{config::Config, StarCastRib},
     };
 
     use std::error::Error;
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index 57e10054..f48d03dd 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -3,10 +3,10 @@
 use std::error::Error;
 
 use inetnum::addr::Prefix;
-use rotonda_store::{
-    meta_examples::PrefixAs, Config, IncludeHistory, MatchOptions, MatchType,
-    Record, RouteStatus, StarCastRib,
-};
+use rotonda_store::match_options::{IncludeHistory, MatchOptions, MatchType};
+use rotonda_store::prefix_record::{Record, RouteStatus};
+use rotonda_store::rib::{config::Config, StarCastRib};
+use rotonda_store::test_types::PrefixAs;
 
 mod common {
     use std::io::Write;
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 12a53099..288120aa 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -1,8 +1,11 @@
 // type Prefix4<'a> = Prefix<u32, PrefixAs>;
 use inetnum::addr::Prefix;
 use rotonda_store::{
-    epoch, meta_examples::PrefixAs, Config, IncludeHistory, MatchOptions,
-    MatchType, Record, RouteStatus, StarCastRib,
+    epoch,
+    match_options::{IncludeHistory, MatchOptions, MatchType},
+    prefix_record::{Record, RouteStatus},
+    rib::{config::Config, StarCastRib},
+    test_types::PrefixAs,
 };
 
 use std::error::Error;
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 4d8f0933..08f50dd7 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -17,9 +17,11 @@ mod tests {
     use log::trace;
     use rotonda_store::{
         epoch,
-        meta_examples::{NoMeta, PrefixAs},
-        Config, IncludeHistory, IntoIpAddr, MatchOptions, MatchType, Record,
-        RouteStatus, StarCastRib,
+        match_options::{IncludeHistory, MatchOptions, MatchType},
+        prefix_record::{Record, RouteStatus},
+        rib::{config::Config, StarCastRib},
+        test_types::{NoMeta, PrefixAs},
+        IntoIpAddr,
     };
 
     rotonda_store::all_strategies![
diff --git a/tests/treebitmap_v6.rs b/tests/treebitmap_v6.rs
index 78f10b7d..13a1e42e 100644
--- a/tests/treebitmap_v6.rs
+++ b/tests/treebitmap_v6.rs
@@ -11,13 +11,17 @@ mod common {
 
 #[cfg(test)]
 mod tests {
-    use inetnum::addr::Prefix;
     use rotonda_store::{
+        addr::Prefix,
         epoch,
-        meta_examples::{NoMeta, PrefixAs},
-        Config, IncludeHistory, IntoIpAddr, MatchOptions, MatchType,
-        MemoryOnlyConfig, PersistOnlyConfig, Record, RouteStatus,
-        StarCastRib,
+        match_options::{IncludeHistory, MatchOptions, MatchType},
+        prefix_record::{Record, RouteStatus},
+        rib::{
+            config::{Config, MemoryOnlyConfig, PersistOnlyConfig},
+            StarCastRib,
+        },
+        test_types::{NoMeta, PrefixAs},
+        IntoIpAddr,
     };
 
     rotonda_store::all_strategies![

From 7527451d67f24c259a636cbc605fdaec0903e89e Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Fri, 14 Mar 2025 08:31:10 +0100
Subject: [PATCH 123/147] public doc draft

---
 src/lib.rs          |   4 +
 src/rib/config.rs   |  47 ++++++++---
 src/rib/starcast.rs | 193 +++++++++++++++++++++++++++++++++++++++++---
 src/types/errors.rs |  20 +++++
 4 files changed, 242 insertions(+), 22 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 1f1264cb..3475cd56 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -48,8 +48,12 @@ pub use types::prefix_record;
 /// Error types returned by a RIB
 pub use types::errors;
 
+/// Trait that defines the AFIs 1 (IPv4) and 2 (IPv6)
 pub use types::af::AddressFamily;
+
+/// The underlying value (u32) and trait impl for AFI 1.
 pub use types::af::IPv4;
+/// The underlying value (u128) and trait impl for AFI 2.
 pub use types::af::IPv6;
 
 /// Trait that describes the conversion of a u32 or u128 in to a IPv4, or IPV6
diff --git a/src/rib/config.rs b/src/rib/config.rs
index d7e031bf..c0f87833 100644
--- a/src/rib/config.rs
+++ b/src/rib/config.rs
@@ -18,7 +18,7 @@
 
 /// Defines where records are stored: in-memory and/or persisted (to disk),
 /// and, whether new records for a unique (prefix, mui) pair are overwritten
-/// or persisted.
+/// or persisted ("historical records").
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum PersistStrategy {
     /// Current records are stored both in-memory and persisted. Historical
@@ -36,13 +36,21 @@ pub enum PersistStrategy {
 }
 
 pub trait Config: Clone + Default + std::fmt::Debug {
+    /// Returns the chosen persist strategy for this configuration
     fn persist_strategy(&self) -> PersistStrategy;
+    /// Returns the path to the directory that is used to store persisted
+    /// records
     fn persist_path(&self) -> Option<String>;
+    /// Set the path to the directory that will be used to persist records to
     fn set_persist_path(&mut self, path: String);
 }
 
 //------------ MemoryOnlyConfig ----------------------------------------------
 
+/// A configuration that sets persist strategy to
+/// `PersistStrategy::MemoryOnly`: Records for unique `(prefix, mui)` pairs
+/// are kept in memory, newer records will overwrite existing records. In
+/// other words, no historical records are preserved.
 #[derive(Copy, Clone, Debug)]
 pub struct MemoryOnlyConfig;
 
@@ -66,16 +74,12 @@ impl Default for MemoryOnlyConfig {
     }
 }
 
-//------------ PeristOnlyConfig ----------------------------------------------
-
-impl Default for PersistOnlyConfig {
-    fn default() -> Self {
-        Self {
-            persist_path: "/tmp/rotonda/".to_string(),
-        }
-    }
-}
+//------------ PersistOnlyConfig ---------------------------------------------
 
+/// A configuration that sets the persist strategy to
+/// `PersistStrategy::PersistOnly`: Records for unique `(prefix, mui)` pairs
+/// are persisted to disk, newer records will overwrite existing records. In
+/// other words, no historical records are preserved.
 #[derive(Clone, Debug)]
 pub struct PersistOnlyConfig {
     persist_path: String,
@@ -95,9 +99,7 @@ impl Config for PersistOnlyConfig {
     }
 }
 
-//------------ WriteAheadConfig ----------------------------------------------
-
-impl Default for WriteAheadConfig {
+impl Default for PersistOnlyConfig {
     fn default() -> Self {
         Self {
             persist_path: "/tmp/rotonda/".to_string(),
@@ -105,6 +107,13 @@ impl Default for WriteAheadConfig {
     }
 }
 
+//------------ WriteAheadConfig ----------------------------------------------
+
+/// A configuration that sets the persist strategy to
+///`PersistStrategy::WriteAhead`: Records for unique `(prefix, mui)` pairs
+///are both kept in memory and persisted to disk, newer records will overwrite
+///existing records in memory, but all records will be kept persisted on disk.
+///In other words, historical records are kept on disk.
 #[derive(Clone, Debug)]
 pub struct WriteAheadConfig {
     persist_path: String,
@@ -124,8 +133,20 @@ impl Config for WriteAheadConfig {
     }
 }
 
+impl Default for WriteAheadConfig {
+    fn default() -> Self {
+        Self {
+            persist_path: "/tmp/rotonda/".to_string(),
+        }
+    }
+}
+
 //------------ PersistHistoryConfig ------------------------------------------
 
+/// A configuration that sets the persist strategy to
+///`PersistStrategy::PersistHistory`: Records for unique `(prefix, mui)` pairs
+///are kept in memory,newer records will replace existing records, but the
+///existing records will be persisted to disk.
 #[derive(Clone, Debug)]
 pub struct PersistHistoryConfig {
     persist_path: String,
diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index 0142b525..72fc6935 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -23,6 +23,37 @@ pub const BIT_SPAN_SIZE: u8 = 32;
 ///
 /// Routes can be kept in memory, persisted to disk, or both. Also, historical
 /// records can be persisted.
+///
+/// A RIB stores "route-like" data. A `route` according to RFC4271 would be
+/// specified as an IP prefix and a set of path attributes. Our StarCastRib,
+/// on the other hand, does not really care whether it stores path attributes,
+/// or any other type of record, for a given IP prefix.
+///
+/// In order to be able to store multiple records for a `(prefix, record)`
+/// pair, however, the store needs to given an extra piece of information in
+/// the key. We are calling this piece of data a `multi uniq id` (called "mui"
+/// throughout this repo), and uses an `u32` as its underlying data type.
+/// This `mui` is completely user-defined, and has no additional semantics
+/// for the  store beyond establishing the uniqueness of the key. The `mui`
+/// was specifically designed for use cases where Rotonda wants to store RIBs
+/// that it receives from multiple peers in one StarCastRib, so that every
+/// peer that Rotonda knows of gets its own, unique `mui`, and our StarCastRib
+/// would store them all without overwriting already stored `(prefix,
+/// record)` pairs. In other words, multiple values can be stored per unique
+/// `(prefix, record)` pair.
+///
+/// Next to creating `(prefix, record)` entries for `mui`, the [RouteStatus]( crate::prefix_record::RouteStatus) of a `mui` can be globally set to
+/// `Withdrawn`or `Active`. A global status of `Withdrawn` overrides the
+/// local status of a prefix for that `mui`. In that case, the local status
+/// can still be changed and will take effect when the global status is set
+/// (back) to `Active`.
+///
+/// The RIB can be conceptually thought of as a MultiMap - a HashMap that can
+/// store multiple values for a given key - that is keyed on `prefix`, and
+/// will store multiple values for a prefix, based on the specified `mui`.
+/// Furthermore, a [persist strategy](crate::rib::config::PersistStrategy),
+/// chosen by the user, for a `StarCastRib` determines what happens with key
+/// collisions in this multi map.
 pub struct StarCastRib<M: Meta, C: Config> {
     v4: StarCastAfRib<IPv4, M, 9, 33, C, 18>,
     v6: StarCastAfRib<IPv6, M, 33, 129, C, 30>,
@@ -30,8 +61,9 @@ pub struct StarCastRib<M: Meta, C: Config> {
 }
 
 impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
-    /// Create a new RIB with a default configuration. The default
-    /// configuration uses the `MemoryOnly` persistence strategy.
+    /// Create a new RIB with a default configuration.
+    ///
+    /// The default configuration uses the `MemoryOnly` persistence strategy.
     ///
     /// This method is really infallible, but we return a result anyway to be
     /// in line with the `new_with_config` method.
@@ -41,7 +73,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
             .map_err(|_| PrefixStoreError::StoreNotReadyError)
     }
 
-    /// Create a new RIB with the specified configuration.
+    /// Create a new RIB with the specified [configuration](
+    /// crate::rib::config).
     ///
     /// Creation may fail for all strategies that persist to disk, e.g.
     /// the persistence path does not exist, it doesn't have the correct
@@ -74,6 +107,16 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         })
     }
 
+    /// Query the RIB for a matching prefix with options.
+    ///
+    /// A reference to a [Guard](crate::Guard) must be passed in to
+    /// assure that the resulting prefixes are time consistent. The guard can
+    /// be re-used for multiple matches to assure time consistency between
+    /// the matches.
+    ///
+    /// Returns a [QueryResult](crate::match_options::QueryResult)
+    ///that may contain one or more prefixes, with or without their associated
+    ///records.
     pub fn match_prefix(
         &'a self,
         search_pfx: &Prefix,
@@ -100,6 +143,9 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         }
     }
 
+    /// Search the RIB for a prefix.
+    ///
+    /// Returns a bool indicating whether the prefix was found. Regardless of the chosen persist strategy
     pub fn contains(&'a self, prefix: &Prefix, mui: Option<u32>) -> bool {
         match prefix.addr() {
             std::net::IpAddr::V4(_addr) => {
@@ -111,6 +157,14 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         }
     }
 
+    /// Return a previously calculated best path for a prefix, if any.
+    ///
+    /// Returns `None` if the prefix was not found
+    ///in the RIB. Returns a [BestPathNotFound](
+    /// crate::errors::PrefixStoreError::BestPathNotFound) error if the
+    /// best path was never calculated, or returns a [StoreNotReadyError](
+    /// crate::errors::PrefixStoreError::StoreNotReadyError) if there is no
+    /// record for the prefix (but the prefix does exist).
     pub fn best_path(
         &'a self,
         search_pfx: &Prefix,
@@ -134,6 +188,17 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         }
     }
 
+    /// Calculate the best path for a prefix.
+    ///
+    /// This method takes all the records for a prefix, i.e. all the records
+    /// for different values of `mui` for this prefix, and calculates the best
+    /// path for them.
+    ///
+    /// Returns the values of `mui` for the best path, and the backup path,
+    /// respectively.
+    /// Returns `None` if the prefix does not exist. Returns a [StoreNotReady]()
+    /// crate::errors::PrefixStoreError::StoreNotReadyError) if there is no
+    /// record for the prefix (but the prefix does exist).
     pub fn calculate_and_store_best_and_backup_path(
         &self,
         search_pfx: &Prefix,
@@ -164,6 +229,12 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         }
     }
 
+    /// Determine if a best path selection is based on stale records.
+    ///
+    /// Returns `Ok(true)` if the records have been updated since the last
+    /// best path selection was performed.
+    /// Returns a [StoreNotReady](crate::errors::PrefixStoreError) if the
+    /// prefix cannot be found in the RIB.
     pub fn is_ps_outdated(
         &self,
         search_pfx: &Prefix,
@@ -187,6 +258,17 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         }
     }
 
+    /// Request all more specific prefixes in the RIB for a certain
+    /// prefix, including the prefix itself.
+    ///
+    /// If a `mui` is specified only prefixes for that particular `mui`
+    /// are returned. If `None` is specified all more specific prefixes,
+    /// regardless of their `mui` will be included in the returned result.
+    ///
+    /// if `include_withdrawn` is set to `true`, all more prefixes that have a
+    /// status  of `Withdrawn` will included in the returned result.
+    ///
+    /// Returns a [QueryResult](crate::match_options::QueryResult).
     pub fn more_specifics_from(
         &'a self,
         search_pfx: &Prefix,
@@ -216,6 +298,17 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         }
     }
 
+    /// Request all less specific prefixes in the RIB for a certain
+    /// prefix, including the prefix itself.
+    ///
+    /// If a `mui` is specified only prefixes for that particular `mui`
+    /// are returned. If `None` is specified all less specific prefixes,
+    /// regardless of their `mui` will be included in the returned result.
+    ///
+    /// if `include_withdrawn` is set to `true`, all more prefixes that have a
+    /// status of `Withdrawn` will included in the returned result.
+    ///
+    /// Returns a [QueryResult](crate::match_options::QueryResult).
     pub fn less_specifics_from(
         &'a self,
         search_pfx: &Prefix,
@@ -245,6 +338,17 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         }
     }
 
+    /// Request an iterator over all less specific prefixes in the RIB for a
+    /// certain prefix, including the prefix itself.
+    ///
+    /// If a `mui` is specified only prefixes for that particular `mui`
+    /// are returned. If `None` is specified all less specific prefixes,
+    /// regardless of their `mui` will be included in the returned result.
+    ///
+    /// if `include_withdrawn` is set to `true`, all more prefixes that have a
+    /// status of `Withdrawn` will included in the returned result.
+    ///
+    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
     pub fn less_specifics_iter_from(
         &'a self,
         search_pfx: &Prefix,
@@ -292,6 +396,17 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
             .chain(right.into_iter().flatten())
     }
 
+    /// Request an iterator over all more specific prefixes in the RIB for a
+    /// certain prefix, including the prefix itself.
+    ///
+    /// If a `mui` is specified only prefixes for that particular `mui`
+    /// are returned. If `None` is specified all more specific prefixes,
+    /// regardless of their `mui` will be included in the returned result.
+    ///
+    /// if `include_withdrawn` is set to `true`, all more prefixes that have a
+    /// status  of `Withdrawn` will included in the returned result.
+    ///
+    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
     pub fn more_specifics_iter_from(
         &'a self,
         search_pfx: &Prefix,
@@ -339,6 +454,13 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
             .chain(right.into_iter().flatten())
     }
 
+    /// Request an iterator over all IPv4 prefixes in the RIB for a certain
+    /// `mui`.
+    ///
+    /// if `include_withdrawn` is set to `true`, all prefixes that have a
+    /// status  of `Withdrawn` will included in the returned result.
+    ///
+    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
     pub fn iter_records_for_mui_v4(
         &'a self,
         mui: u32,
@@ -366,6 +488,13 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         .flatten()
     }
 
+    /// Request an iterator over all IPv6 prefixes in the RIB for a certain
+    /// `mui`.
+    ///
+    /// if `include_withdrawn` is set to `true`, all prefixes that have a
+    /// status  of `Withdrawn` will included in the returned result.
+    ///
+    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
     pub fn iter_records_for_mui_v6(
         &'a self,
         mui: u32,
@@ -393,6 +522,14 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         .flatten()
     }
 
+    /// Insert a Prefix with a [Record](crate::prefix_record::Record) into
+    /// the RIB.
+    ///
+    /// If `update_path_selections` is passed in with the tie breaker info
+    /// then perform a best path selection.
+    ///
+    /// Returns an [UpsertReport](crate::stats::UpsertReport) or
+    /// a PrefixStoreError if the prefix and/or Record cannot be stored.
     pub fn insert(
         &self,
         prefix: &Prefix,
@@ -413,6 +550,9 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         }
     }
 
+    /// Request an iterator over all prefixes in the RIB.
+    ///
+    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
     pub fn prefixes_iter(
         &'a self,
         guard: &'a Guard,
@@ -423,6 +563,9 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
             .chain(self.v6.prefixes_iter(guard).map(PrefixRecord::from))
     }
 
+    /// Request an iterator over all IPv4 prefixes in the RIB.
+    ///
+    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
     pub fn prefixes_iter_v4(
         &'a self,
         guard: &'a Guard,
@@ -430,6 +573,9 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         self.v4.prefixes_iter(guard).map(PrefixRecord::from)
     }
 
+    /// Request an iterator over all IPv6 prefixes in the RIB.
+    ///
+    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
     pub fn prefixes_iter_v6(
         &'a self,
         guard: &'a Guard,
@@ -437,6 +583,9 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         self.v6.prefixes_iter(guard).map(PrefixRecord::from)
     }
 
+    /// Request an iterator over all persisted prefixes.
+    ///
+    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
     pub fn persist_prefixes_iter(
         &'a self,
     ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
@@ -446,28 +595,40 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
             .chain(self.v6.persist_prefixes_iter().map(PrefixRecord::from))
     }
 
+    /// Request an iterator over all persisted IPv4 prefixes.
+    ///
+    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
     pub fn persist_prefixes_iter_v4(
         &'a self,
     ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
         self.v4.persist_prefixes_iter().map(PrefixRecord::from)
     }
 
+    /// Request an iterator over all persisted IPv6 prefixes.
+    ///
+    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
     pub fn persist_prefixes_iter_v6(
         &'a self,
     ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
         self.v6.persist_prefixes_iter().map(PrefixRecord::from)
     }
 
+    /// Request whether the global status of a `mui` is set to `Active` for
+    ///both IPv4 and IPv6 prefixes.
     pub fn is_mui_active(&self, mui: u32) -> bool {
         let guard = &epoch::pin();
         self.v4.is_mui_active(mui, guard) || self.v6.is_mui_active(mui, guard)
     }
 
+    /// Request whether the global status of a `mui` is set to `Active` for
+    ///IPv4 prefixes.
     pub fn is_mui_active_v4(&self, mui: u32) -> bool {
         let guard = &epoch::pin();
         self.v4.is_mui_active(mui, guard)
     }
 
+    /// Request whether the global status of a `mui` is set to `Active` for
+    /// IPv6 prefixes.
     pub fn is_mui_active_v6(&self, mui: u32) -> bool {
         let guard = &epoch::pin();
         self.v6.is_mui_active(mui, guard)
@@ -604,16 +765,16 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         res_v4.and(res_v6)
     }
 
-    // Whether the global status for IPv4 prefixes and the specified
-    // `multi_uniq_id` is set to `Withdrawn`.
+    /// Reques whether the global status for IPv4 prefixes and the specified
+    /// `multi_uniq_id` is set to `Withdrawn`.
     pub fn mui_is_withdrawn_v4(&self, mui: u32) -> bool {
         let guard = &epoch::pin();
 
         self.v4.mui_is_withdrawn(mui, guard)
     }
 
-    // Whether the global status for IPv6 prefixes and the specified
-    // `multi_uniq_id` is set to `Active`.
+    /// Request whether the global status for IPv6 prefixes and the specified
+    /// `multi_uniq_id` is set to `Active`.
     pub fn mui_is_withdrawn_v6(&self, mui: u32) -> bool {
         let guard = &epoch::pin();
 
@@ -712,10 +873,19 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     // Disk Persistence
 
+    /// Request the persist strategy as set in the [configuration](
+    /// crate::rib::config) for this RIB.
     pub fn persist_strategy(&self) -> PersistStrategy {
         self.config.persist_strategy()
     }
 
+    /// Request all records for a prefix.
+    ///
+    /// If `mui` is specified, only the record for that specific `mui` will
+    /// be returned.
+    ///
+    /// if `include_withdrawn` is passed in as `true` records with status
+    /// `Withdrawn` will be returned, as well as records with status `Active`.
     pub fn get_records_for_prefix(
         &self,
         prefix: &Prefix,
@@ -740,8 +910,13 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         }
     }
 
-    /// Persist all the non-unique (prefix, mui, ltime) tuples
-    /// with their values to disk
+    /// Persist all relevant RIB entries to disk.
+    ///
+    /// Records that are marked for persistence are first cached in memory,
+    /// and only written to disk when this method is called.
+    ////
+    /// The specific behaviour is depended on the chosen [persists strategy](
+    /// crate::rib::config::PersistStrategy).
     pub fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
         self.v4.flush_to_disk()?;
         self.v6.flush_to_disk()?;
diff --git a/src/types/errors.rs b/src/types/errors.rs
index 38001254..ce2dd53a 100644
--- a/src/types/errors.rs
+++ b/src/types/errors.rs
@@ -1,15 +1,35 @@
 use std::fmt;
 
+/// Possible errors returned by methods on a RIB
 #[derive(Debug, PartialEq, Eq)]
 pub enum PrefixStoreError {
+    /// There is too much contention while creating a node: the store has
+    /// given up. The method or function returning this error can be safely
+    /// retries.
     NodeCreationMaxRetryError,
+    /// A node that does not exist (yet), maybe due to contention. The
+    ///function or method causing this error can be safely retried.
     NodeNotFound,
+    /// The method returning this error presupposes a condition that has not
+    /// been met, and may never be met. Retrying is safe, but may result in
+    /// the same error. Therefore is should probably be retried only once.
     StoreNotReadyError,
+    /// A best path was requested, but the selection procedure was performed
+    /// on a route set that is now stale. A new best path calculation over the
+    /// set should be performed before retrying.
     PathSelectionOutdated,
+    /// The requested prefix was not found in the store.
     PrefixNotFound,
+    /// A best path was requested, but it was never calculated. Perform a best
+    ///path selection first, before retrying.
     BestPathNotFound,
+    /// A record was specifically requested from the in-memory data structure,
+    /// but the record is not in memory. It may be persisted to disk.
     RecordNotInMemory,
+    /// The method returning this error was trying to persist records to disk
+    /// but failed. Retrying is safe, but may be yield the same result.
     PersistFailed,
+    /// A status for a record was requested, but it was never set.
     StatusUnknown,
 }
 

From f2826d43b27ef3f9244c27ef556b687c5d444997 Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Fri, 14 Mar 2025 14:38:23 +0700
Subject: [PATCH 124/147] corrections

---
 src/rib/starcast.rs | 38 ++++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index 72fc6935..cbdb3d69 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -719,9 +719,10 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     }
 
     /// Change the status of all records for IPv6 prefixes for this
-    /// `multi_uniq_id` globally to Active.  Note that the global
-    /// `Active` status will be overridden by the local status of the
-    /// record.
+    /// `multi_uniq_id` globally to Active.
+    ///
+    /// Note that the global `Active` status will be overridden by the local
+    /// status of the record.
     pub fn mark_mui_as_active_v6(
         &self,
         mui: u32,
@@ -732,11 +733,12 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     }
 
     /// Change the status of all records for IPv6 prefixes for this
-    /// `multi_uniq_id` globally to Withdrawn. A global `Withdrawn`
-    /// status for a `multi_uniq_id` overrides the local status of
-    /// prefixes for this mui. However the local status can still be
-    /// modified. This modification will take effect if the global
-    /// status is changed to `Active`.
+    /// `multi_uniq_id` globally to Withdrawn.
+    ///
+    /// A global `Withdrawn` status for a `multi_uniq_id` overrides the local
+    /// status of prefixes for this mui. However the local status can still be
+    /// modified. This modification will take effect if the global status is
+    /// changed to `Active`.
     pub fn mark_mui_as_withdrawn_v6(
         &self,
         mui: u32,
@@ -765,7 +767,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         res_v4.and(res_v6)
     }
 
-    /// Reques whether the global status for IPv4 prefixes and the specified
+    /// Request whether the global status for IPv4 prefixes and the specified
     /// `multi_uniq_id` is set to `Withdrawn`.
     pub fn mui_is_withdrawn_v4(&self, mui: u32) -> bool {
         let guard = &epoch::pin();
@@ -781,7 +783,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         self.v6.mui_is_withdrawn(mui, guard)
     }
 
-    /// Returns the number of all prefixes in the store.
+    /// Request the number of all prefixes in the store.
     ///
     /// Note that this method will actually traverse the complete
     /// tree.
@@ -789,7 +791,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         self.v4.get_prefixes_count() + self.v6.get_prefixes_count()
     }
 
-    /// Returns the number of all IPv4 prefixes in the store.
+    /// Request the number of all IPv4 prefixes in the store.
     ///
     /// Note that this counter may be lower than the actual
     /// number in the store, due to contention at the time of
@@ -798,7 +800,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         self.v4.get_prefixes_count()
     }
 
-    /// Returns the number of all IPv4 prefixes with the
+    /// Request the number of all IPv4 prefixes with the
     /// supplied prefix length in the store.
     ///
     /// Note that this counter may be lower than the actual
@@ -808,7 +810,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         self.v4.get_prefixes_count_for_len(len)
     }
 
-    /// Returns the number of all IPv6 prefixes in the store.
+    /// Request the number of all IPv6 prefixes in the store.
     ///
     /// Note that this counter may be lower than the actual
     /// number in the store, due to contention at the time of
@@ -827,7 +829,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         self.v6.get_prefixes_count_for_len(len)
     }
 
-    /// Returns the number of nodes in the store.
+    /// Request the number of nodes in the store.
     ///
     /// Note that this counter may be lower than the actual
     /// number in the store, due to contention at the time of
@@ -836,7 +838,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         self.v4.get_nodes_count() + self.v6.get_nodes_count()
     }
 
-    /// Returns the number of IPv4 nodes in the store.
+    /// Request the number of IPv4 nodes in the store.
     ///
     /// Note that this counter may be lower than the actual
     /// number in the store, due to contention at the time of
@@ -845,7 +847,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         self.v4.get_nodes_count()
     }
 
-    /// Returns the number of IPv6 nodes in the store.
+    /// Request the number of IPv6 nodes in the store.
     ///
     /// Note that this counter may be lower than the actual
     /// number in the store, due to contention at the time of
@@ -924,7 +926,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         Ok(())
     }
 
-    /// Return the approximate number of items that are persisted
+    /// Request the approximate number of items that are persisted
     /// to disk, for IPv4 and IPv6 respectively.
     pub fn approx_persisted_items(&self) -> (usize, usize) {
         (
@@ -933,7 +935,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         )
     }
 
-    /// Return an estimation of the disk space currently used by the
+    /// Request an estimation of the disk space currently used by the
     /// store in bytes.
     pub fn disk_space(&self) -> u64 {
         self.v4.disk_space() + self.v6.disk_space()

From fbab2e8626c3ecd0bfc0cc2f33bffc9ac33283e8 Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Sat, 15 Mar 2025 10:05:47 +0700
Subject: [PATCH 125/147] corrections

---
 src/rib/starcast.rs | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index cbdb3d69..155f839f 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -348,7 +348,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// if `include_withdrawn` is set to `true`, all more prefixes that have a
     /// status of `Withdrawn` will included in the returned result.
     ///
-    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord](
+    /// crate::prefix_record::PrefixRecord).
     pub fn less_specifics_iter_from(
         &'a self,
         search_pfx: &Prefix,
@@ -406,7 +407,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// if `include_withdrawn` is set to `true`, all more prefixes that have a
     /// status  of `Withdrawn` will included in the returned result.
     ///
-    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord](
+    /// crate::prefix_record::PrefixRecord).
     pub fn more_specifics_iter_from(
         &'a self,
         search_pfx: &Prefix,
@@ -460,7 +462,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// if `include_withdrawn` is set to `true`, all prefixes that have a
     /// status  of `Withdrawn` will included in the returned result.
     ///
-    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord](
+    /// crate::prefix_record::PrefixRecord).
     pub fn iter_records_for_mui_v4(
         &'a self,
         mui: u32,
@@ -494,7 +497,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// if `include_withdrawn` is set to `true`, all prefixes that have a
     /// status  of `Withdrawn` will included in the returned result.
     ///
-    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord](
+    /// crate::prefix_record::PrefixRecord).
     pub fn iter_records_for_mui_v6(
         &'a self,
         mui: u32,
@@ -528,8 +532,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// If `update_path_selections` is passed in with the tie breaker info
     /// then perform a best path selection.
     ///
-    /// Returns an [UpsertReport](crate::stats::UpsertReport) or
-    /// a PrefixStoreError if the prefix and/or Record cannot be stored.
+    /// Returns an iterator over [PrefixRecord](
+    /// crate::prefix_record::PrefixRecord).
     pub fn insert(
         &self,
         prefix: &Prefix,
@@ -552,7 +556,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     /// Request an iterator over all prefixes in the RIB.
     ///
-    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord](
+    /// crate::prefix_record::PrefixRecord).
     pub fn prefixes_iter(
         &'a self,
         guard: &'a Guard,
@@ -565,7 +570,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     /// Request an iterator over all IPv4 prefixes in the RIB.
     ///
-    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord]
+    /// (crate::prefix_record::PrefixRecord).
     pub fn prefixes_iter_v4(
         &'a self,
         guard: &'a Guard,
@@ -575,7 +581,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     /// Request an iterator over all IPv6 prefixes in the RIB.
     ///
-    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord]
+    /// (crate::prefix_record::PrefixRecord).
     pub fn prefixes_iter_v6(
         &'a self,
         guard: &'a Guard,
@@ -597,7 +604,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     /// Request an iterator over all persisted IPv4 prefixes.
     ///
-    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord](
+    /// crate::prefix_record::PrefixRecord).
     pub fn persist_prefixes_iter_v4(
         &'a self,
     ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
@@ -606,7 +614,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     /// Request an iterator over all persisted IPv6 prefixes.
     ///
-    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord](
+    /// crate::prefix_record::PrefixRecord).
     pub fn persist_prefixes_iter_v6(
         &'a self,
     ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {

From 82f3e8161ce82ec4a20e6dc406e3db425637cd62 Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Sat, 15 Mar 2025 10:13:31 +0700
Subject: [PATCH 126/147] corrections

---
 src/rib/starcast.rs | 34 ++++++++++++----------------------
 1 file changed, 12 insertions(+), 22 deletions(-)

diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index 155f839f..8abd0d20 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -114,9 +114,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// be re-used for multiple matches to assure time consistency between
     /// the matches.
     ///
-    /// Returns a [QueryResult](crate::match_options::QueryResult)
-    ///that may contain one or more prefixes, with or without their associated
-    ///records.
+    /// Returns a [QueryResult] that may contain one or more prefixes, with or
+    /// without their associated records.
     pub fn match_prefix(
         &'a self,
         search_pfx: &Prefix,
@@ -348,8 +347,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// if `include_withdrawn` is set to `true`, all more prefixes that have a
     /// status of `Withdrawn` will included in the returned result.
     ///
-    /// Returns an iterator over [PrefixRecord](
-    /// crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord].
     pub fn less_specifics_iter_from(
         &'a self,
         search_pfx: &Prefix,
@@ -407,8 +405,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// if `include_withdrawn` is set to `true`, all more prefixes that have a
     /// status  of `Withdrawn` will included in the returned result.
     ///
-    /// Returns an iterator over [PrefixRecord](
-    /// crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord].
     pub fn more_specifics_iter_from(
         &'a self,
         search_pfx: &Prefix,
@@ -462,8 +459,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// if `include_withdrawn` is set to `true`, all prefixes that have a
     /// status  of `Withdrawn` will included in the returned result.
     ///
-    /// Returns an iterator over [PrefixRecord](
-    /// crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord].
     pub fn iter_records_for_mui_v4(
         &'a self,
         mui: u32,
@@ -497,8 +493,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// if `include_withdrawn` is set to `true`, all prefixes that have a
     /// status  of `Withdrawn` will included in the returned result.
     ///
-    /// Returns an iterator over [PrefixRecord](
-    /// crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord].
     pub fn iter_records_for_mui_v6(
         &'a self,
         mui: u32,
@@ -532,8 +527,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// If `update_path_selections` is passed in with the tie breaker info
     /// then perform a best path selection.
     ///
-    /// Returns an iterator over [PrefixRecord](
-    /// crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord].
     pub fn insert(
         &self,
         prefix: &Prefix,
@@ -556,8 +550,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     /// Request an iterator over all prefixes in the RIB.
     ///
-    /// Returns an iterator over [PrefixRecord](
-    /// crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord].
     pub fn prefixes_iter(
         &'a self,
         guard: &'a Guard,
@@ -570,8 +563,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     /// Request an iterator over all IPv4 prefixes in the RIB.
     ///
-    /// Returns an iterator over [PrefixRecord]
-    /// (crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord].
     pub fn prefixes_iter_v4(
         &'a self,
         guard: &'a Guard,
@@ -581,8 +573,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     /// Request an iterator over all IPv6 prefixes in the RIB.
     ///
-    /// Returns an iterator over [PrefixRecord]
-    /// (crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord].
     pub fn prefixes_iter_v6(
         &'a self,
         guard: &'a Guard,
@@ -592,7 +583,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     /// Request an iterator over all persisted prefixes.
     ///
-    /// Returns an over [PrefixRecord](crate::prefix_record::PrefixRecord).
+    /// Returns an over [PrefixRecord].
     pub fn persist_prefixes_iter(
         &'a self,
     ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
@@ -614,8 +605,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
 
     /// Request an iterator over all persisted IPv6 prefixes.
     ///
-    /// Returns an iterator over [PrefixRecord](
-    /// crate::prefix_record::PrefixRecord).
+    /// Returns an iterator over [PrefixRecord].
     pub fn persist_prefixes_iter_v6(
         &'a self,
     ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {

From 8e1a06f0f96049c5d46e58bc8629806ee732451c Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Mon, 24 Mar 2025 15:54:39 +0100
Subject: [PATCH 127/147] introduce FatalError to replace unwraps for lsm

---
 src/lsm_tree/mod.rs          | 270 +++++++++++++----------------------
 src/prefix_cht/cht.rs        |  84 +++++++----
 src/rib/starcast.rs          |  23 ++-
 src/rib/starcast_af.rs       | 127 ++++++++--------
 src/rib/starcast_af_query.rs |   6 +-
 src/tree_bitmap/mod.rs       |   3 +-
 src/tree_bitmap/node_cht.rs  |  10 +-
 src/types/errors.rs          |  27 +++-
 src/types/prefix_record.rs   |   8 +-
 9 files changed, 280 insertions(+), 278 deletions(-)

diff --git a/src/lsm_tree/mod.rs b/src/lsm_tree/mod.rs
index f86226d8..52fadeca 100644
--- a/src/lsm_tree/mod.rs
+++ b/src/lsm_tree/mod.rs
@@ -12,6 +12,7 @@ use zerocopy::{
     Unaligned, U32, U64,
 };
 
+use crate::errors::{FatalError, FatalResult};
 use crate::prefix_record::Meta;
 use crate::stats::Counters;
 use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
@@ -33,14 +34,21 @@ pub(crate) trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
     //     Self::try_ref_from_bytes(bytes.as_bytes())
     // }
 
-    fn header(bytes: &[u8]) -> &LongKey<AF> {
-        LongKey::try_ref_from_bytes(bytes.as_bytes()).unwrap()
+    // Try to extract a header from the bytes for reading only. If this
+    // somehow fails, we don't know what to do anymore. Data may be corrupted,
+    // so it probably should not be retried.
+    fn header(bytes: &[u8]) -> Result<&LongKey<AF>, FatalError> {
+        LongKey::try_ref_from_bytes(bytes.as_bytes()).map_err(|_| FatalError)
     }
 
-    fn header_mut(bytes: &mut [u8]) -> &mut LongKey<AF> {
+    // Try to extract a header for writing. If this somehow fails, we most
+    //probably cannot write to it anymore. This is fatal. The application
+    //should exit, data integrity (on disk) should be verified.
+    fn header_mut(bytes: &mut [u8]) -> Result<&mut LongKey<AF>, FatalError> {
         trace!("key size {}", KEY_SIZE);
         trace!("bytes len {}", bytes.len());
-        LongKey::try_mut_from_bytes(bytes.as_mut_bytes()).unwrap()
+        LongKey::try_mut_from_bytes(bytes.as_mut_bytes())
+            .map_err(|_| FatalError)
     }
 
     fn _short_key(bytes: &[u8]) -> &ShortKey<AF> {
@@ -147,12 +155,8 @@ pub struct LsmTree<
     _k: PhantomData<K>,
 }
 
-impl<
-        AF: AddressFamily,
-        K: KeySize<AF, KEY_SIZE>,
-        // const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > LsmTree<AF, K, KEY_SIZE>
+impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
+    LsmTree<AF, K, KEY_SIZE>
 {
     pub fn new(persist_path: &Path) -> LsmTree<AF, K, KEY_SIZE> {
         LsmTree::<AF, K, KEY_SIZE> {
@@ -179,7 +183,7 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         withdrawn_muis_bmin: &RoaringBitmap,
-    ) -> Option<Vec<Vec<u8>>> {
+    ) -> Option<Vec<FatalResult<Vec<u8>>>> {
         match (mui, include_withdrawn) {
             // Specific mui, include withdrawn routes
             (Some(mui), true) => {
@@ -192,24 +196,19 @@ impl<
                         kv.map(|kv| {
                             trace!("mui i persist kv pair found: {:?}", kv);
                             let mut bytes = [kv.0, kv.1].concat();
-                            // let key: &mut LongKey<AF> =
-                            //     LongKey::try_mut_from_bytes(
-                            //         bytes.as_mut_bytes(),
-                            //     )
-                            //     .unwrap();
-                            let key = K::header_mut(&mut bytes[..KEY_SIZE]);
+                            let key = K::header_mut(&mut bytes[..KEY_SIZE])?;
                             // If mui is in the global withdrawn muis table,
                             // then rewrite the routestatus of the record
                             // to withdrawn.
                             if withdrawn_muis_bmin.contains(key.mui.into()) {
                                 key.status = RouteStatus::Withdrawn;
                             }
-                            bytes
+                            Ok(bytes)
                         })
                     })
-                    .collect::<Vec<lsm_tree::Result<Vec<u8>>>>()
+                    .collect::<Vec<lsm_tree::Result<FatalResult<Vec<u8>>>>>()
                     .into_iter()
-                    .collect::<lsm_tree::Result<Vec<Vec<u8>>>>()
+                    .collect::<lsm_tree::Result<Vec<FatalResult<Vec<u8>>>>>()
                     .ok()
                     .and_then(
                         |recs| {
@@ -221,48 +220,33 @@ impl<
                         },
                     )
             }
-            // Al muis, include withdrawn routes
+            // All muis, include withdrawn routes
             (None, true) => {
                 // get all records for this prefix
-                // let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
                 self.tree
                     .prefix(prefix.as_bytes(), None, None)
-                    // .into_iter()
                     .map(|kv| {
                         kv.map(|kv| {
                             trace!("n i persist kv pair found: {:?}", kv);
-                            // let kv = kv.unwrap();
-                            // let (_, r_mui, ltime, mut status) =
-                            //     Self::parse_key(kv.0.as_ref());
 
-                            // If mui is in the global withdrawn muis table, then
-                            // rewrite the routestatus of the record to withdrawn.
+                            // If mui is in the global withdrawn muis table,
+                            // then rewrite the routestatus of the record
+                            // to withdrawn.
                             let mut bytes = [kv.0, kv.1].concat();
                             trace!("bytes {:?}", bytes);
-                            // let key: &mut LongKey<AF> =
-                            //     LongKey::try_mut_from_bytes(
-                            //         bytes.as_mut_bytes(),
-                            //     )
-                            //     .unwrap();
-                            let key = K::header_mut(&mut bytes[..KEY_SIZE]);
+                            let key = K::header_mut(&mut bytes[..KEY_SIZE])?;
                             trace!("key {:?}", key);
                             trace!("wm_bmin {:?}", withdrawn_muis_bmin);
                             if withdrawn_muis_bmin.contains(key.mui.into()) {
                                 trace!("rewrite status");
                                 key.status = RouteStatus::Withdrawn;
                             }
-                            // PublicRecord::new(
-                            //     r_mui,
-                            //     ltime,
-                            //     status,
-                            //     kv.1.as_ref().to_vec().into(),
-                            // )
-                            bytes
+                            Ok(bytes)
                         })
                     })
-                    .collect::<Vec<lsm_tree::Result<Vec<u8>>>>()
+                    .collect::<Vec<lsm_tree::Result<FatalResult<Vec<u8>>>>>()
                     .into_iter()
-                    .collect::<lsm_tree::Result<Vec<Vec<u8>>>>()
+                    .collect::<lsm_tree::Result<Vec<FatalResult<Vec<u8>>>>>()
                     .ok()
                     .and_then(
                         |recs| {
@@ -280,47 +264,49 @@ impl<
                 // let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
                 self.tree
                     .prefix(prefix.as_bytes(), None, None)
-                    .filter_map(|kv| {
-                        kv.map(|kv| {
+                    .filter_map(|r| {
+                        r.map(|kv| {
                             trace!("n f persist kv pair found: {:?}", kv);
                             let mut bytes = [kv.0, kv.1].concat();
-                            // let key: &mut LongKey<AF> =
-                            //     LongKey::try_mut_from_bytes(
-                            //         bytes.as_mut_bytes(),
-                            //     )
-                            //     .unwrap();
-                            let header =
-                                K::header_mut(&mut bytes[..KEY_SIZE]);
-                            // If mui is in the global withdrawn muis table,
-                            // then skip this record
-                            trace!("header {}", Prefix::from(header.prefix));
-                            trace!(
-                                "status {}",
-                                header.status == RouteStatus::Withdrawn
-                            );
-                            if header.status == RouteStatus::Withdrawn
-                                || withdrawn_muis_bmin
-                                    .contains(header.mui.into())
+                            if let Ok(header) =
+                                K::header_mut(&mut bytes[..KEY_SIZE])
                             {
+                                // If mui is in the global withdrawn muis
+                                // table, then skip this record
+                                trace!(
+                                    "header {}",
+                                    Prefix::from(header.prefix)
+                                );
+                                trace!(
+                                    "status {}",
+                                    header.status == RouteStatus::Withdrawn
+                                );
+                                if header.status == RouteStatus::Withdrawn
+                                    || withdrawn_muis_bmin
+                                        .contains(header.mui.into())
+                                {
+                                    trace!(
+                                        "NOT returning {} {}",
+                                        Prefix::from(header.prefix),
+                                        header.mui
+                                    );
+                                    return None;
+                                }
                                 trace!(
-                                    "NOT returning {} {}",
+                                    "RETURNING {} {}",
                                     Prefix::from(header.prefix),
                                     header.mui
                                 );
-                                return None;
+                                Some(Ok(bytes))
+                            } else {
+                                return Some(Err(FatalError));
                             }
-                            trace!(
-                                "RETURNING {} {}",
-                                Prefix::from(header.prefix),
-                                header.mui
-                            );
-                            Some(bytes)
                         })
                         .transpose()
                     })
-                    .collect::<Vec<lsm_tree::Result<Vec<u8>>>>()
+                    .collect::<Vec<lsm_tree::Result<FatalResult<Vec<u8>>>>>()
                     .into_iter()
-                    .collect::<lsm_tree::Result<Vec<Vec<u8>>>>()
+                    .collect::<lsm_tree::Result<Vec<FatalResult<Vec<u8>>>>>()
                     .ok()
                     .and_then(
                         |recs| {
@@ -339,42 +325,29 @@ impl<
                 let prefix_b = ShortKey::<AF>::from((prefix, mui));
                 self.tree
                     .prefix(prefix_b.as_bytes(), None, None)
-                    // .into_iter()
                     .filter_map(|kv| {
-                        // let kv = kv.unwrap();
                         kv.map(|kv| {
                             trace!("mui f persist kv pair found: {:?}", kv);
                             let bytes = [kv.0, kv.1].concat();
-                            // let (_, r_mui, ltime, status) =
-                            //     Self::parse_key(kv.0.as_ref());
-
-                            // let key: &mut LongKey<AF> =
-                            //     LongKey::try_mut_from_bytes(
-                            //         bytes.as_mut_bytes(),
-                            //     )
-                            //     .unwrap();
-                            let key = K::header(&bytes[..KEY_SIZE]);
-                            // If mui is in the global withdrawn muis table, then
-                            // skip this record
-                            if key.status == RouteStatus::Withdrawn
-                                || withdrawn_muis_bmin
-                                    .contains(key.mui.into())
-                            {
-                                return None;
+                            if let Ok(key) = K::header(&bytes[..KEY_SIZE]) {
+                                // If mui is in the global withdrawn muis
+                                // table, then skip this record
+                                if key.status == RouteStatus::Withdrawn
+                                    || withdrawn_muis_bmin
+                                        .contains(key.mui.into())
+                                {
+                                    return None;
+                                }
+                                Some(Ok(bytes))
+                            } else {
+                                Some(Err(FatalError))
                             }
-                            // Some(PublicRecord::new(
-                            //     mui,
-                            //     ltime,
-                            //     status,
-                            //     kv.1.as_ref().to_vec().into(),
-                            // ))
-                            Some(bytes)
                         })
                         .transpose()
                     })
-                    .collect::<Vec<lsm_tree::Result<Vec<u8>>>>()
+                    .collect::<Vec<lsm_tree::Result<FatalResult<Vec<u8>>>>>()
                     .into_iter()
-                    .collect::<lsm_tree::Result<Vec<Vec<u8>>>>()
+                    .collect::<lsm_tree::Result<Vec<FatalResult<Vec<u8>>>>>()
                     .ok()
                     .and_then(
                         |recs| {
@@ -387,74 +360,38 @@ impl<
                     )
             }
         }
-
-        // if let Some(mui) = mui {
-        //     let prefix_b = Self::prefix_mui_persistence_key(prefix, mui);
-
-        //     (*self.tree.prefix(prefix_b))
-        //         .into_iter()
-        //         .filter_map(|kv| {
-        //             let kv = kv.unwrap();
-        //             let (_, mui, ltime, mut status) =
-        //                 Self::parse_key(kv.0.as_ref());
-        //             if include_withdrawn {
-        //                 // If mui is in the global withdrawn muis table, then
-        //                 // rewrite the routestatus of the record to withdrawn.
-        //                 if withdrawn_muis_bmin.contains(mui) {
-        //                     status = RouteStatus::Withdrawn;
-        //                 }
-        //             // If the use does not want withdrawn routes then filter
-        //             // them out here.
-        //             } else if status == RouteStatus::Withdrawn {
-        //                 return None;
-        //             }
-        //             Some(PublicRecord::new(
-        //                 mui,
-        //                 ltime,
-        //                 status,
-        //                 kv.1.as_ref().to_vec().into(),
-        //             ))
-        //         })
-        //         .collect::<Vec<_>>()
-        // } else {
-        //     let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
-
-        //     (*self.tree.prefix(prefix_b))
-        //         .into_iter()
-        //         .map(|kv| {
-        //             let kv = kv.unwrap();
-        //             let (_, mui, ltime, status) =
-        //                 Self::parse_key(kv.0.as_ref());
-        //             if include_withdrawn || status != RouteStatus::Withdrawn {
-        //                 Some(PublicRecord::new(
-        //                     mui,
-        //                     ltime,
-        //                     status,
-        //                     kv.1.as_ref().to_vec().into(),
-        //                 ))
-        //             } else {
-        //                 None
-        //             }
-        //         })
-        //         .collect::<Vec<_>>()
-        // }
     }
 
     pub fn get_most_recent_record_for_prefix_mui(
         &self,
         prefix: PrefixId<AF>,
         mui: u32,
-    ) -> Option<Vec<u8>> {
+    ) -> FatalResult<Option<Vec<u8>>> {
         trace!("get most recent record for prefix mui combo");
         let key_b = ShortKey::from((prefix, mui));
+        let mut res: FatalResult<Vec<u8>> = Err(FatalError);
+
+        for rkv in self.tree.prefix(key_b.as_bytes(), None, None).into_iter()
+        {
+            if let Ok(kvs) = rkv {
+                let kv = [kvs.0, kvs.1].concat();
+                if let Ok(h) = K::header(&kv) {
+                    if let Ok(r) = &res {
+                        if K::header(&r).unwrap().ltime < h.ltime {
+                            res = Ok(kv);
+                        }
+                    } else {
+                        res = Ok(kv);
+                    }
+                } else {
+                    return Err(FatalError);
+                }
+            } else {
+                return Err(FatalError);
+            }
+        }
 
-        (*self.tree.prefix(key_b.as_bytes(), None, None))
-            .into_iter()
-            .map(move |kv| {
-                let kv = kv.unwrap();
-                [kv.0, kv.1].concat()
-            })
-            .max_by(|b0, b1| K::header(b0).ltime.cmp(&K::header(b1).ltime))
+        res.map(|r| Some(r.to_vec()))
     }
 
     pub(crate) fn get_records_with_keys_for_prefix_mui(
@@ -462,9 +399,6 @@ impl<
         prefix: PrefixId<AF>,
         mui: u32,
     ) -> Vec<Vec<u8>> {
-        // let key_b: [u8; KEY_SIZE] =
-        //     ShortKey::from((prefix, mui)).as_key_size_bytes();
-
         let key_b = ShortKey::from((prefix, mui));
 
         (*self.tree.prefix(key_b.as_bytes(), None, None))
@@ -472,16 +406,6 @@ impl<
             .map(|kv| {
                 let kv = kv.unwrap();
                 [kv.0, kv.1].concat()
-                // let (_, mui, ltime, status) = Self::parse_key(kv.0.as_ref());
-                // (
-                //     kv.0.to_vec(),
-                //     PublicRecord::new(
-                //         mui,
-                //         ltime,
-                //         status,
-                //         kv.1.as_ref().to_vec().into(),
-                //     ),
-                // )
             })
             .collect::<Vec<_>>()
     }
@@ -944,7 +868,7 @@ impl<
         };
 
         if let Some(mut r_rec) = rec {
-            let outer_pfx = K::header(&r_rec[0]).prefix;
+            let outer_pfx = K::header(&r_rec[0]).unwrap().prefix;
 
             for (k, v) in self.tree_iter.by_ref().flatten() {
                 // let (pfx, mui, ltime, status) =
@@ -953,7 +877,7 @@ impl<
                 //     );
                 let header = K::header(&k);
 
-                if header.prefix == outer_pfx {
+                if header.unwrap().prefix == outer_pfx {
                     r_rec.push([k, v].concat());
                     // r_rec.1.push(PublicRecord {
                     //     meta: v.to_vec().into(),
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index 94e04cce..c3f33faa 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -35,15 +35,25 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         Self(Arc::new(Mutex::new(record_map)))
     }
 
-    fn guard_with_retry(
+    fn acquire_write_lock(
         &self,
-        mut retry_count: usize,
-    ) -> (MutexGuard<HashMap<u32, MultiMapValue<M>>>, usize) {
+    ) -> Result<
+        (MutexGuard<HashMap<u32, MultiMapValue<M>>>, usize),
+        PrefixStoreError,
+    > {
+        let mut retry_count: usize = 0;
         let backoff = Backoff::new();
 
         loop {
-            if let Ok(guard) = self.0.try_lock() {
-                return (guard, retry_count);
+            // We're using lock(), which returns an Error only if another
+            // thread has panicked while holding the lock. In that situtation
+            // we are certainly not going to write anything.
+            if let Ok(guard) = self
+                .0
+                .lock()
+                .or_else(|_| Err(PrefixStoreError::ExternalError))
+            {
+                return Ok((guard, retry_count));
             }
 
             backoff.spin();
@@ -51,9 +61,24 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         }
     }
 
+    fn acquire_read_guard(
+        &self,
+    ) -> MutexGuard<HashMap<u32, MultiMapValue<M>>> {
+        let backoff = Backoff::new();
+
+        loop {
+            if let Ok(guard) = self.0.try_lock() {
+                return guard;
+            }
+
+            backoff.spin();
+        }
+    }
+
     pub fn _len(&self) -> usize {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
+        // let c_map = Arc::clone(&self.0);
+        // let record_map = c_map.lock().unwrap();
+        let record_map = self.acquire_read_guard();
         record_map.len()
     }
 
@@ -62,8 +87,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         mui: u32,
         include_withdrawn: bool,
     ) -> Option<Record<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
+        // let c_map = Arc::clone(&self.0);
+        // let record_map = c_map.lock().unwrap();
+        let record_map = self.acquire_read_guard();
 
         record_map.get(&mui).and_then(|r| -> Option<Record<M>> {
             if include_withdrawn || r.route_status() == RouteStatus::Active {
@@ -75,8 +101,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     }
 
     pub fn best_backup(&self, tbi: M::TBI) -> (Option<u32>, Option<u32>) {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
+        // let c_map = Arc::clone(&self.0);
+        // let record_map = c_map.lock().unwrap();
+        let record_map = self.acquire_read_guard();
         let ord_routes = record_map
             .iter()
             .map(|r| (r.1.meta().as_orderable(tbi), *r.0));
@@ -91,8 +118,10 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         bmin: &RoaringBitmap,
         rewrite_status: RouteStatus,
     ) -> Option<Record<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
+        // let c_map = Arc::clone(&self.0);
+        // let record_map = c_map.lock().unwrap();
+
+        let record_map = self.acquire_read_guard();
         record_map.get(&mui).map(|r| {
             // We'll return a cloned record: the record in the store remains
             // untouched.
@@ -165,8 +194,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         bmin: &RoaringBitmap,
         rewrite_status: RouteStatus,
     ) -> Vec<Record<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
+        // let c_map = Arc::clone(&self.0);
+        // let record_map = c_map.lock().unwrap();
+        let record_map = self.acquire_read_guard();
         record_map
             .iter()
             .map(move |r| {
@@ -180,8 +210,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     }
 
     pub fn _as_records(&self) -> Vec<Record<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
+        // let c_map = Arc::clone(&self.0);
+        // let record_map = c_map.lock().unwrap();
+        let record_map = self.acquire_read_guard();
         record_map
             .iter()
             .map(|r| Record::from((*r.0, r.1)))
@@ -195,8 +226,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         &self,
         bmin: &RoaringBitmap,
     ) -> Vec<Record<M>> {
-        let c_map = Arc::clone(&self.0);
-        let record_map = c_map.lock().unwrap();
+        // let c_map = Arc::clone(&self.0);
+        // let record_map = c_map.lock().unwrap();
+        let record_map = self.acquire_read_guard();
         record_map
             .iter()
             .filter_map(|r| {
@@ -213,8 +245,9 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
 
     // Change the local status of the record for this mui to Withdrawn.
     pub fn mark_as_withdrawn_for_mui(&self, mui: u32, ltime: u64) {
-        let c_map = Arc::clone(&self.0);
-        let mut record_map = c_map.lock().unwrap();
+        // let c_map = Arc::clone(&self.0);
+        // let mut record_map = c_map.lock().unwrap();
+        let mut record_map = self.acquire_read_guard();
         if let Some(rec) = record_map.get_mut(&mui) {
             rec.set_route_status(RouteStatus::Withdrawn);
             rec.set_logical_time(ltime);
@@ -223,9 +256,10 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
 
     // Change the local status of the record for this mui to Active.
     pub fn mark_as_active_for_mui(&self, mui: u32, ltime: u64) {
-        let record_map = Arc::clone(&self.0);
-        let mut r_map = record_map.lock().unwrap();
-        if let Some(rec) = r_map.get_mut(&mui) {
+        // let record_map = Arc::clone(&self.0);
+        // let mut r_map = record_map.lock().unwrap();
+        let mut record_map = self.acquire_read_guard();
+        if let Some(rec) = record_map.get_mut(&mui) {
             rec.set_route_status(RouteStatus::Active);
             rec.set_logical_time(ltime);
         }
@@ -241,7 +275,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         new_rec: Record<M>,
     ) -> Result<(Option<(MultiMapValue<M>, usize)>, usize), PrefixStoreError>
     {
-        let (mut record_map, retry_count) = self.guard_with_retry(0);
+        let (mut record_map, retry_count) = self.acquire_write_lock()?;
         let key = new_rec.multi_uniq_id;
 
         match record_map.contains_key(&key) {
diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index 8abd0d20..2eb9c1c7 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -4,6 +4,7 @@ use rand::prelude::*;
 
 use crate::{
     epoch,
+    errors::FatalError,
     match_options::{MatchOptions, QueryResult},
     prefix_record::{Meta, PrefixRecord, Record},
     rib::config::Config,
@@ -586,11 +587,15 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// Returns an over [PrefixRecord].
     pub fn persist_prefixes_iter(
         &'a self,
-    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+    ) -> impl Iterator<Item = Result<PrefixRecord<M>, FatalError>> + 'a {
         self.v4
             .persist_prefixes_iter()
-            .map(PrefixRecord::from)
-            .chain(self.v6.persist_prefixes_iter().map(PrefixRecord::from))
+            .map(|rr| rr.map(PrefixRecord::from))
+            .chain(
+                self.v6
+                    .persist_prefixes_iter()
+                    .map(|rr| rr.map(PrefixRecord::from)),
+            )
     }
 
     /// Request an iterator over all persisted IPv4 prefixes.
@@ -599,8 +604,10 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// crate::prefix_record::PrefixRecord).
     pub fn persist_prefixes_iter_v4(
         &'a self,
-    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
-        self.v4.persist_prefixes_iter().map(PrefixRecord::from)
+    ) -> impl Iterator<Item = Result<PrefixRecord<M>, FatalError>> + 'a {
+        self.v4
+            .persist_prefixes_iter()
+            .map(|rr| rr.map(PrefixRecord::from))
     }
 
     /// Request an iterator over all persisted IPv6 prefixes.
@@ -608,8 +615,10 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// Returns an iterator over [PrefixRecord].
     pub fn persist_prefixes_iter_v6(
         &'a self,
-    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
-        self.v6.persist_prefixes_iter().map(PrefixRecord::from)
+    ) -> impl Iterator<Item = Result<PrefixRecord<M>, FatalError>> + 'a {
+        self.v6
+            .persist_prefixes_iter()
+            .map(|rr| rr.map(PrefixRecord::from))
     }
 
     /// Request whether the global status of a `mui` is set to `Active` for
diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index f97d6595..7715e478 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -3,11 +3,11 @@ use std::path::Path;
 use inetnum::addr::Prefix;
 use log::{info, trace};
 
+use crate::errors::FatalError;
 use crate::prefix_record::Meta;
 use crate::rib::config::PersistStrategy;
 use crate::stats::{Counters, UpsertCounters, UpsertReport};
 use crate::{epoch, Guard};
-use zerocopy::TryFromBytes;
 
 use crate::prefix_cht::cht::PrefixCht;
 use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
@@ -68,7 +68,10 @@ impl<
         let persist_tree = match config.persist_strategy() {
             PersistStrategy::MemoryOnly => None,
             _ => {
-                let persist_path = &config.persist_path().unwrap();
+                let persist_path =
+                    &config.persist_path().unwrap_or_else(|| {
+                        "Missing persistence path".to_string()
+                    });
                 let pp_ref = &Path::new(persist_path);
                 Some(LsmTree::new(pp_ref))
             }
@@ -216,16 +219,19 @@ impl<
                     "mark as wd in persist tree {:?} for mui {:?}",
                     prefix, mui
                 );
-                let p_tree = self.persist_tree.as_ref().unwrap();
-                let stored_prefixes =
-                    p_tree.get_records_with_keys_for_prefix_mui(prefix, mui);
-
-                for s in stored_prefixes {
-                    let header = ValueHeader {
-                        ltime,
-                        status: RouteStatus::Withdrawn,
-                    };
-                    p_tree.rewrite_header_for_record(header, &s);
+                if let Some(p_tree) = self.persist_tree.as_ref() {
+                    let stored_prefixes = p_tree
+                        .get_records_with_keys_for_prefix_mui(prefix, mui);
+
+                    for s in stored_prefixes {
+                        let header = ValueHeader {
+                            ltime,
+                            status: RouteStatus::Withdrawn,
+                        };
+                        p_tree.rewrite_header_for_record(header, &s);
+                    }
+                } else {
+                    return Err(PrefixStoreError::StoreNotReadyError);
                 }
             }
             PersistStrategy::PersistHistory => {
@@ -278,16 +284,18 @@ impl<
                 stored_prefix.record_map.mark_as_active_for_mui(mui, ltime);
             }
             PersistStrategy::PersistOnly => {
-                let p_tree = self.persist_tree.as_ref().unwrap();
-
-                if let Some(record_b) =
-                    p_tree.get_most_recent_record_for_prefix_mui(prefix, mui)
-                {
-                    let header = ValueHeader {
-                        ltime,
-                        status: RouteStatus::Active,
-                    };
-                    p_tree.rewrite_header_for_record(header, &record_b);
+                if let Some(p_tree) = self.persist_tree.as_ref() {
+                    if let Ok(Some(record_b)) = p_tree
+                        .get_most_recent_record_for_prefix_mui(prefix, mui)
+                    {
+                        let header = ValueHeader {
+                            ltime,
+                            status: RouteStatus::Active,
+                        };
+                        p_tree.rewrite_header_for_record(header, &record_b);
+                    }
+                } else {
+                    return Err(PrefixStoreError::StoreNotReadyError);
                 }
             }
             PersistStrategy::PersistHistory => {
@@ -348,13 +356,7 @@ impl<
     // functions.
     pub fn mui_is_withdrawn(&self, mui: u32, guard: &Guard) -> bool {
         // unsafe {
-        self.tree_bitmap
-            .withdrawn_muis_bmin(guard)
-            // .load(Ordering::Acquire, guard)
-            // .as_ref()
-            // }
-            // .unwrap()
-            .contains(mui)
+        self.tree_bitmap.withdrawn_muis_bmin(guard).contains(mui)
     }
 
     // Whether this mui is globally active. Note that the local statuses of
@@ -362,14 +364,7 @@ impl<
     // functions.
     pub(crate) fn is_mui_active(&self, mui: u32, guard: &Guard) -> bool {
         // !unsafe {
-        !self
-            .tree_bitmap
-            .withdrawn_muis_bmin(guard)
-            // .load(Ordering::Acquire, guard)
-            // .as_ref()
-            // }
-            // .unwrap()
-            .contains(mui)
+        !self.tree_bitmap.withdrawn_muis_bmin(guard).contains(mui)
     }
 
     pub(crate) fn get_prefixes_count(&self) -> UpsertCounters {
@@ -415,39 +410,39 @@ impl<
 
     pub(crate) fn persist_prefixes_iter(
         &self,
-    ) -> impl Iterator<Item = (Prefix, Vec<Record<M>>)> + '_ {
-        self.persist_tree
-            .as_ref()
-            .map(|tree| {
-                tree.prefixes_iter().map(|recs| {
-                    (
-                        Prefix::from(
-                            ZeroCopyRecord::<AF>::try_ref_from_bytes(
-                                &recs[0],
-                            )
-                            .unwrap()
-                            .prefix,
-                        ),
+    ) -> impl Iterator<Item = Result<(Prefix, Vec<Record<M>>), FatalError>> + '_
+    {
+        if let Some(tree) = &self.persist_tree {
+            Some(tree.prefixes_iter().map(|recs| {
+                if let Ok(pfx) = ZeroCopyRecord::<AF>::from_bytes(&recs[0]) {
+                    Ok((
+                        Prefix::from(pfx.prefix),
                         recs.iter()
-                            .map(|rec| {
-                                let rec =
-                                    ZeroCopyRecord::<AF>::try_ref_from_bytes(
-                                        rec,
-                                    )
-                                    .unwrap();
-                                Record {
-                                    multi_uniq_id: rec.multi_uniq_id,
-                                    ltime: rec.ltime,
-                                    status: rec.status,
-                                    meta: rec.meta.to_vec().into(),
+                            .filter_map(|rec| {
+                                if let Ok(rec) =
+                                    ZeroCopyRecord::<AF>::from_bytes(rec)
+                                {
+                                    Some(Record {
+                                        multi_uniq_id: rec.multi_uniq_id,
+                                        ltime: rec.ltime,
+                                        status: rec.status,
+                                        meta: rec.meta.to_vec().into(),
+                                    })
+                                } else {
+                                    None
                                 }
                             })
                             .collect::<Vec<Record<M>>>(),
-                    )
-                })
-            })
-            .into_iter()
-            .flatten()
+                    ))
+                } else {
+                    Err(FatalError)
+                }
+            }))
+        } else {
+            None
+        }
+        .into_iter()
+        .flatten()
     }
 
     pub(crate) fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
diff --git a/src/rib/starcast_af_query.rs b/src/rib/starcast_af_query.rs
index 513f4c37..64056883 100644
--- a/src/rib/starcast_af_query.rs
+++ b/src/rib/starcast_af_query.rs
@@ -48,8 +48,10 @@ impl<
                         v.iter()
                             .map(|bytes| {
                                 let record: &ZeroCopyRecord<AF> =
-                                    ZeroCopyRecord::try_ref_from_bytes(bytes)
-                                        .unwrap();
+                                    ZeroCopyRecord::try_ref_from_bytes(
+                                        bytes.as_ref().unwrap(),
+                                    )
+                                    .unwrap();
                                 Record::<M> {
                                     multi_uniq_id: record.multi_uniq_id,
                                     ltime: record.ltime,
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 764e2470..7559aafa 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -820,7 +820,8 @@ Giving up this node. This shouldn't happen!",
             }
         }
     }
-    pub fn retrieve_node_for_mui(
+
+    pub(crate) fn retrieve_node_for_mui(
         &self,
         id: StrideNodeId<AF>,
         mui: u32,
diff --git a/src/tree_bitmap/node_cht.rs b/src/tree_bitmap/node_cht.rs
index 7c64de74..cc739119 100644
--- a/src/tree_bitmap/node_cht.rs
+++ b/src/tree_bitmap/node_cht.rs
@@ -47,7 +47,10 @@ impl<AF: AddressFamily> NodeSet<AF> {
         AF: crate::types::AddressFamily,
     {
         let try_count = 0;
-        let mut rbm = self.1.write().unwrap();
+        let mut rbm = self
+            .1
+            .write()
+            .map_err(|_| PrefixStoreError::StoreNotReadyError)?;
         let absent = rbm.insert(multi_uniq_id);
 
         Ok((try_count, !absent))
@@ -63,7 +66,10 @@ impl<AF: AddressFamily> NodeSet<AF> {
     {
         let try_count = 0;
 
-        let mut rbm = self.1.write().unwrap();
+        let mut rbm = self
+            .1
+            .write()
+            .map_err(|_| PrefixStoreError::StoreNotReadyError)?;
         rbm.remove(multi_uniq_id);
 
         Ok(try_count)
diff --git a/src/types/errors.rs b/src/types/errors.rs
index ce2dd53a..58a01d2d 100644
--- a/src/types/errors.rs
+++ b/src/types/errors.rs
@@ -12,7 +12,7 @@ pub enum PrefixStoreError {
     NodeNotFound,
     /// The method returning this error presupposes a condition that has not
     /// been met, and may never be met. Retrying is safe, but may result in
-    /// the same error. Therefore is should probably be retried only once.
+    /// the same error. Therefore it should probably be retried only once.
     StoreNotReadyError,
     /// A best path was requested, but the selection procedure was performed
     /// on a route set that is now stale. A new best path calculation over the
@@ -26,6 +26,9 @@ pub enum PrefixStoreError {
     /// A record was specifically requested from the in-memory data structure,
     /// but the record is not in memory. It may be persisted to disk.
     RecordNotInMemory,
+    /// An error external to to our execution happened, most notably another
+    ///thread panicking while trying to acquire a lock.
+    ExternalError,
     /// The method returning this error was trying to persist records to disk
     /// but failed. Retrying is safe, but may be yield the same result.
     PersistFailed,
@@ -78,6 +81,13 @@ impl fmt::Display for PrefixStoreError {
                     persisted."
                 )
             }
+            PrefixStoreError::ExternalError => {
+                write!(
+                f,
+                "Error: An action could not be completed, due to another \
+                thread panicking."
+            )
+            }
             PrefixStoreError::StatusUnknown => {
                 write!(
                     f,
@@ -88,3 +98,18 @@ impl fmt::Display for PrefixStoreError {
         }
     }
 }
+
+#[derive(Debug)]
+pub struct FatalError;
+
+impl std::fmt::Display for FatalError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "Error: A Fatal error has occurred. The store must be considered \
+             corrupted. The application should terminate."
+        )
+    }
+}
+
+pub type FatalResult<T> = Result<T, FatalError>;
diff --git a/src/types/prefix_record.rs b/src/types/prefix_record.rs
index 109421ad..156ab000 100644
--- a/src/types/prefix_record.rs
+++ b/src/types/prefix_record.rs
@@ -1,7 +1,7 @@
 use std::fmt;
 use std::fmt::Debug;
 
-use crate::types::AddressFamily;
+use crate::{errors::FatalError, types::AddressFamily};
 use inetnum::addr::Prefix;
 use zerocopy::{Immutable, IntoBytes, KnownLayout, TryFromBytes, Unaligned};
 
@@ -77,6 +77,12 @@ pub(crate) struct ZeroCopyRecord<AF: AddressFamily> {
     pub meta: [u8],
 }
 
+impl<AF: AddressFamily> ZeroCopyRecord<AF> {
+    pub(crate) fn from_bytes(b: &[u8]) -> Result<&Self, FatalError> {
+        Self::try_ref_from_bytes(b).or_else(|_| Err(FatalError))
+    }
+}
+
 impl<AF: AddressFamily + std::fmt::Display> std::fmt::Display
     for ZeroCopyRecord<AF>
 {

From 57124362c2fd54310cf71705e11a28fc08a800ef Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 09:26:11 +0100
Subject: [PATCH 128/147] FatalResult everywhere

---
 src/bin/cli.rs               |  15 ++-
 src/bin/load_mrt.rs          |   2 +
 src/lsm_tree/mod.rs          | 150 ++++++++--------------
 src/prefix_cht/cht.rs        |   6 +-
 src/prefix_cht/iterators.rs  |   7 ++
 src/rib/starcast.rs          |  54 ++++----
 src/rib/starcast_af.rs       | 135 ++++++++++++--------
 src/rib/starcast_af_query.rs | 235 +++++++++++++++++++++--------------
 src/tree_bitmap/mod.rs       |  37 +++---
 src/types/af.rs              |  28 ++---
 src/types/errors.rs          |  16 ++-
 src/types/prefix_id.rs       |  16 +--
 src/types/prefix_record.rs   |   2 +-
 13 files changed, 389 insertions(+), 314 deletions(-)

diff --git a/src/bin/cli.rs b/src/bin/cli.rs
index 593a573d..a376826e 100644
--- a/src/bin/cli.rs
+++ b/src/bin/cli.rs
@@ -111,6 +111,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                     tree_bitmap
                                         .prefixes_iter_v4(guard)
                                         .for_each(|pfx| {
+                                            let pfx = pfx.unwrap();
                                             println!(
                                                 "{} {}",
                                                 pfx.prefix, pfx.meta[0]
@@ -125,6 +126,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                     tree_bitmap
                                         .prefixes_iter_v6(guard)
                                         .for_each(|pfx| {
+                                            let pfx = pfx.unwrap();
                                             println!(
                                                 "{} {}",
                                                 pfx.prefix, pfx.meta[0]
@@ -147,6 +149,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                     tree_bitmap
                                         .prefixes_iter(guard)
                                         .for_each(|pfx| {
+                                            let pfx = pfx.unwrap();
                                             println!(
                                                 "{} {}",
                                                 pfx.prefix, pfx.meta[0]
@@ -235,7 +238,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                         include_history: IncludeHistory::None,
                                     },
                                     guard,
-                                );
+                                )?;
                                 println!("start query result");
                                 println!("{}", query_result);
                                 println!("end query result");
@@ -264,7 +267,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                             None,
                                             false,
                                             guard,
-                                        )
+                                        )?
                                         .more_specifics
                                         .map_or("None".to_string(), |x| x
                                             .to_string())
@@ -278,7 +281,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                             None,
                                             false,
                                             guard,
-                                        )
+                                        )?
                                         .less_specifics
                                         .map_or("None".to_string(), |x| x
                                             .to_string())
@@ -300,7 +303,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                                 IncludeHistory::None
                                         },
                                         guard
-                                    )
+                                    )?
                                 );
                                 println!("--- numatch");
                                 println!("more specifics");
@@ -312,7 +315,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                             None,
                                             false,
                                             guard,
-                                        )
+                                        )?
                                         .more_specifics
                                         .map_or("None".to_string(), |x| x
                                             .to_string())
@@ -326,7 +329,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                             None,
                                             false,
                                             guard
-                                        )
+                                        )?
                                         .less_specifics
                                         .map_or("None".to_string(), |x| x
                                             .to_string())
diff --git a/src/bin/load_mrt.rs b/src/bin/load_mrt.rs
index ab8e7617..beb285e9 100644
--- a/src/bin/load_mrt.rs
+++ b/src/bin/load_mrt.rs
@@ -593,6 +593,7 @@ fn exec_for_store<'a, C: Config + Sync>(
             let values = store
                 .unwrap()
                 .get_records_for_prefix(&pfx, None, false)
+                .unwrap()
                 .unwrap();
             if values.is_empty() {
                 eprintln!("Found empty prefix on disk");
@@ -604,6 +605,7 @@ fn exec_for_store<'a, C: Config + Sync>(
                 let recs = store
                     .unwrap()
                     .get_records_for_prefix(&pfx, None, false)
+                    .unwrap()
                     .unwrap();
                 println!("LEN {} prefix: {}", max_len, pfx);
                 for rec in recs {
diff --git a/src/lsm_tree/mod.rs b/src/lsm_tree/mod.rs
index 52fadeca..e173bc9f 100644
--- a/src/lsm_tree/mod.rs
+++ b/src/lsm_tree/mod.rs
@@ -51,12 +51,12 @@ pub(crate) trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
             .map_err(|_| FatalError)
     }
 
-    fn _short_key(bytes: &[u8]) -> &ShortKey<AF> {
-        trace!("short key from bytes {:?}", bytes);
-        let s_b = &bytes[..(AF::BITS as usize / 8) + 6];
-        trace!("short key {:?}", s_b);
-        ShortKey::try_ref_from_prefix(bytes).unwrap().0
-    }
+    // fn _short_key(bytes: &[u8]) -> &ShortKey<AF> {
+    //     trace!("short key from bytes {:?}", bytes);
+    //     let s_b = &bytes[..(AF::BITS as usize / 8) + 6];
+    //     trace!("short key {:?}", s_b);
+    //     ShortKey::try_ref_from_prefix(bytes).unwrap().0
+    // }
 }
 
 #[derive(Debug, KnownLayout, Immutable, FromBytes, Unaligned, IntoBytes)]
@@ -87,17 +87,6 @@ pub struct LongKey<AF: AddressFamily> {
 impl<AF: AddressFamily, const KEY_SIZE: usize> KeySize<AF, KEY_SIZE>
     for ShortKey<AF>
 {
-    // fn new_write_key(
-    //     prefix: PrefixId<AF>,
-    //     mui: u32,
-    //     _ltime: u64,
-    //     _status: RouteStatus,
-    // ) -> [u8; KEY_SIZE] {
-    //     *Self::from((prefix, mui))
-    //         .as_bytes()
-    //         .first_chunk::<KEY_SIZE>()
-    //         .unwrap()
-    // }
 }
 
 impl<AF: AddressFamily> From<(PrefixId<AF>, u32)> for ShortKey<AF> {
@@ -112,17 +101,6 @@ impl<AF: AddressFamily> From<(PrefixId<AF>, u32)> for ShortKey<AF> {
 impl<AF: AddressFamily, const KEY_SIZE: usize> KeySize<AF, KEY_SIZE>
     for LongKey<AF>
 {
-    // fn new_write_key(
-    //     prefix: PrefixId<AF>,
-    //     mui: u32,
-    //     ltime: u64,
-    //     status: RouteStatus,
-    // ) -> [u8; KEY_SIZE] {
-    //     *Self::from((prefix, mui, ltime, status))
-    //         .as_bytes()
-    //         .first_chunk::<KEY_SIZE>()
-    //         .unwrap()
-    // }
 }
 
 impl<AF: AddressFamily> From<(PrefixId<AF>, u32, u64, RouteStatus)>
@@ -158,12 +136,16 @@ pub struct LsmTree<
 impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
     LsmTree<AF, K, KEY_SIZE>
 {
-    pub fn new(persist_path: &Path) -> LsmTree<AF, K, KEY_SIZE> {
-        LsmTree::<AF, K, KEY_SIZE> {
-            tree: lsm_tree::Config::new(persist_path).open().unwrap(),
-            counters: Counters::default(),
-            _af: PhantomData,
-            _k: PhantomData,
+    pub fn new(persist_path: &Path) -> FatalResult<LsmTree<AF, K, KEY_SIZE>> {
+        if let Ok(tree) = lsm_tree::Config::new(persist_path).open() {
+            Ok(LsmTree::<AF, K, KEY_SIZE> {
+                tree,
+                counters: Counters::default(),
+                _af: PhantomData,
+                _k: PhantomData,
+            })
+        } else {
+            Err(FatalError)
         }
     }
 
@@ -299,7 +281,7 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
                                 );
                                 Some(Ok(bytes))
                             } else {
-                                return Some(Err(FatalError));
+                                Some(Err(FatalError))
                             }
                         })
                         .transpose()
@@ -371,14 +353,15 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
         let key_b = ShortKey::from((prefix, mui));
         let mut res: FatalResult<Vec<u8>> = Err(FatalError);
 
-        for rkv in self.tree.prefix(key_b.as_bytes(), None, None).into_iter()
-        {
+        for rkv in self.tree.prefix(key_b.as_bytes(), None, None) {
             if let Ok(kvs) = rkv {
                 let kv = [kvs.0, kvs.1].concat();
                 if let Ok(h) = K::header(&kv) {
                     if let Ok(r) = &res {
-                        if K::header(&r).unwrap().ltime < h.ltime {
-                            res = Ok(kv);
+                        if let Ok(h_res) = K::header(r) {
+                            if h_res.ltime < h.ltime {
+                                res = Ok(kv);
+                            }
                         }
                     } else {
                         res = Ok(kv);
@@ -398,14 +381,17 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
         &self,
         prefix: PrefixId<AF>,
         mui: u32,
-    ) -> Vec<Vec<u8>> {
+    ) -> Vec<FatalResult<Vec<u8>>> {
         let key_b = ShortKey::from((prefix, mui));
 
         (*self.tree.prefix(key_b.as_bytes(), None, None))
             .into_iter()
-            .map(|kv| {
-                let kv = kv.unwrap();
-                [kv.0, kv.1].concat()
+            .map(|rkv| {
+                if let Ok(kv) = rkv {
+                    Ok([kv.0, kv.1].concat())
+                } else {
+                    Err(FatalError)
+                }
             })
             .collect::<Vec<_>>()
     }
@@ -698,9 +684,9 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
         &self,
         header: ValueHeader,
         record_b: &[u8],
-    ) {
+    ) -> FatalResult<()> {
         let record = ZeroCopyRecord::<AF>::try_ref_from_prefix(record_b)
-            .unwrap()
+            .map_err(|_| FatalError)?
             .0;
         let key = ShortKey::from((record.prefix, record.multi_uniq_id));
         trace!("insert key {:?}", key);
@@ -711,6 +697,8 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
             .extend_from_slice(record.meta.as_ref());
 
         self.insert(key.as_bytes(), header.as_bytes());
+
+        Ok(())
     }
 
     pub(crate) fn insert_empty_record(
@@ -728,7 +716,7 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
 
     pub(crate) fn prefixes_iter(
         &self,
-    ) -> impl Iterator<Item = Vec<Vec<u8>>> + '_ {
+    ) -> impl Iterator<Item = Vec<FatalResult<Vec<u8>>>> + '_ {
         PersistedPrefixIter::<AF, K, KEY_SIZE> {
             tree_iter: self.tree.iter(None, None),
             cur_rec: None,
@@ -805,27 +793,19 @@ impl<
 pub(crate) struct PersistedPrefixIter<
     AF: AddressFamily,
     K: KeySize<AF, KEY_SIZE>,
-    // M: Meta,
-    // const PREFIX_SIZE: usize,
     const KEY_SIZE: usize,
 > {
-    cur_rec: Option<Vec<Vec<u8>>>,
+    cur_rec: Option<Vec<FatalResult<Vec<u8>>>>,
     tree_iter:
         Box<dyn DoubleEndedIterator<Item = Result<KvPair, lsm_tree::Error>>>,
     _af: PhantomData<AF>,
-    // _m: PhantomData<M>,
     _k: PhantomData<K>,
 }
 
-impl<
-        AF: AddressFamily,
-        K: KeySize<AF, KEY_SIZE>,
-        // M: Meta,
-        // const PREFIX_SIZE: usize,
-        const KEY_SIZE: usize,
-    > Iterator for PersistedPrefixIter<AF, K, KEY_SIZE>
+impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
+    Iterator for PersistedPrefixIter<AF, K, KEY_SIZE>
 {
-    type Item = Vec<Vec<u8>>;
+    type Item = Vec<FatalResult<Vec<u8>>>;
     fn next(&mut self) -> Option<Self::Item> {
         let rec;
 
@@ -843,20 +823,7 @@ impl<
                     return None;
                 }
                 Some(Ok((k, v))) => {
-                    // let p_k =
-                    //     PersistTree::<AF, K, PREFIX_SIZE, KEY_SIZE>::parse_key(
-                    //         k.as_ref(),
-                    //     );
-                    // rec = Some((
-                    //     p_k.0,
-                    //     vec![PublicRecord::<M> {
-                    //         multi_uniq_id: p_k.1,
-                    //         ltime: p_k.2,
-                    //         status: p_k.3,
-                    //         meta: v.to_vec().into(),
-                    //     }],
-                    // ));
-                    rec = Some(vec![[k, v].concat()]);
+                    rec = Some(vec![Ok([k, v].concat())]);
                 }
                 Some(Err(_)) => {
                     // This is NOT GOOD. Both that it happens, and that we are
@@ -868,35 +835,26 @@ impl<
         };
 
         if let Some(mut r_rec) = rec {
-            let outer_pfx = K::header(&r_rec[0]).unwrap().prefix;
+            let outer_pfx = if let Some(Ok(Ok(rr))) =
+                r_rec.first().map(|v| v.as_ref().map(|h| K::header(h)))
+            {
+                rr.prefix
+            } else {
+                return Some(vec![Err(FatalError)]);
+            };
 
             for (k, v) in self.tree_iter.by_ref().flatten() {
-                // let (pfx, mui, ltime, status) =
-                //     PersistTree::<AF, PREFIX_SIZE, KEY_SIZE>::parse_key(
-                //         k.as_ref(),
-                //     );
                 let header = K::header(&k);
 
-                if header.unwrap().prefix == outer_pfx {
-                    r_rec.push([k, v].concat());
-                    // r_rec.1.push(PublicRecord {
-                    //     meta: v.to_vec().into(),
-                    //     multi_uniq_id: header.mui.into(),
-                    //     ltime: header.ltime.into(),
-                    //     status: header.status,
-                    // });
+                if let Ok(h) = header {
+                    if h.prefix == outer_pfx {
+                        r_rec.push(Ok([k, v].concat()));
+                    } else {
+                        self.cur_rec = Some(vec![Ok([k, v].concat())]);
+                        break;
+                    }
                 } else {
-                    self.cur_rec = Some(vec![[k, v].concat()]);
-                    // self.cur_rec = Some((
-                    //     header.prefix,
-                    //     vec![PublicRecord {
-                    //         meta: v.to_vec().into(),
-                    //         multi_uniq_id: header.mui.into(),
-                    //         ltime: header.ltime.into(),
-                    //         status: header.status.into(),
-                    //     }],
-                    // ));
-                    break;
+                    r_rec.push(Err(FatalError));
                 }
             }
 
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index c3f33faa..0d49eb13 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -48,10 +48,8 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
             // We're using lock(), which returns an Error only if another
             // thread has panicked while holding the lock. In that situtation
             // we are certainly not going to write anything.
-            if let Ok(guard) = self
-                .0
-                .lock()
-                .or_else(|_| Err(PrefixStoreError::ExternalError))
+            if let Ok(guard) =
+                self.0.lock().map_err(|_| PrefixStoreError::ExternalError)
             {
                 return Ok((guard, retry_count));
             }
diff --git a/src/prefix_cht/iterators.rs b/src/prefix_cht/iterators.rs
index 4298c24c..e28748de 100644
--- a/src/prefix_cht/iterators.rs
+++ b/src/prefix_cht/iterators.rs
@@ -7,6 +7,13 @@ use crate::{
     TreeBitMap,
 };
 
+// This iterator is unused right now: all iterators go over the in-memory
+// treebitmap, and retreive metadata based on the persist_strategy per prefix
+// from the relevant tree.
+//
+// However this tree may ultimately be more efficient for the MemoryOnly
+// strategy.
+
 pub(crate) struct _MoreSpecificPrefixIter<
     'a,
     AF: AddressFamily,
diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index 2eb9c1c7..fa7f1591 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -4,7 +4,7 @@ use rand::prelude::*;
 
 use crate::{
     epoch,
-    errors::FatalError,
+    errors::{FatalError, FatalResult},
     match_options::{MatchOptions, QueryResult},
     prefix_record::{Meta, PrefixRecord, Record},
     rib::config::Config,
@@ -122,7 +122,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         search_pfx: &Prefix,
         options: &MatchOptions,
         guard: &'a Guard,
-    ) -> QueryResult<M> {
+    ) -> FatalResult<QueryResult<M>> {
         match search_pfx.addr() {
             std::net::IpAddr::V4(addr) => self.v4.match_prefix(
                 PrefixId::<IPv4>::new(
@@ -275,7 +275,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> QueryResult<M> {
+    ) -> FatalResult<QueryResult<M>> {
         match search_pfx.addr() {
             std::net::IpAddr::V4(addr) => self.v4.more_specifics_from(
                 PrefixId::<IPv4>::new(
@@ -315,7 +315,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> QueryResult<M> {
+    ) -> FatalResult<QueryResult<M>> {
         match search_pfx.addr() {
             std::net::IpAddr::V4(addr) => self.v4.less_specifics_from(
                 PrefixId::<IPv4>::new(
@@ -355,7 +355,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+    ) -> impl Iterator<Item = FatalResult<PrefixRecord<M>>> + 'a {
         let (left, right) = match search_pfx.addr() {
             std::net::IpAddr::V4(addr) => (
                 Some(
@@ -369,7 +369,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
                             include_withdrawn,
                             guard,
                         )
-                        .map(PrefixRecord::from),
+                        .map(|r| r.map(PrefixRecord::from)),
                 ),
                 None,
             ),
@@ -386,7 +386,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
                             include_withdrawn,
                             guard,
                         )
-                        .map(PrefixRecord::from),
+                        .map(|r| r.map(PrefixRecord::from)),
                 ),
             ),
         };
@@ -413,7 +413,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+    ) -> impl Iterator<Item = FatalResult<PrefixRecord<M>>> + 'a {
         let (left, right) = match search_pfx.addr() {
             std::net::IpAddr::V4(addr) => (
                 Some(
@@ -427,7 +427,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
                             include_withdrawn,
                             guard,
                         )
-                        .map(PrefixRecord::from),
+                        .map(|r| r.map(PrefixRecord::from)),
                 ),
                 None,
             ),
@@ -444,7 +444,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
                             include_withdrawn,
                             guard,
                         )
-                        .map(PrefixRecord::from),
+                        .map(|r| r.map(PrefixRecord::from)),
                 ),
             ),
         };
@@ -466,7 +466,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         mui: u32,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+    ) -> impl Iterator<Item = FatalResult<PrefixRecord<M>>> + 'a {
         if self.v4.mui_is_withdrawn(mui, guard) && !include_withdrawn {
             None
         } else {
@@ -481,7 +481,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
                         include_withdrawn,
                         guard,
                     )
-                    .map(PrefixRecord::from),
+                    .map(|r| r.map(PrefixRecord::from)),
             )
         }
         .into_iter()
@@ -500,7 +500,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         mui: u32,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+    ) -> impl Iterator<Item = FatalResult<PrefixRecord<M>>> + 'a {
         if self.v6.mui_is_withdrawn(mui, guard) && !include_withdrawn {
             None
         } else {
@@ -515,7 +515,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
                         include_withdrawn,
                         guard,
                     )
-                    .map(PrefixRecord::from),
+                    .map(|r| r.map(PrefixRecord::from)),
             )
         }
         .into_iter()
@@ -555,11 +555,15 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     pub fn prefixes_iter(
         &'a self,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
+    ) -> impl Iterator<Item = FatalResult<PrefixRecord<M>>> + 'a {
         self.v4
             .prefixes_iter(guard)
-            .map(PrefixRecord::from)
-            .chain(self.v6.prefixes_iter(guard).map(PrefixRecord::from))
+            .map(|r| r.map(PrefixRecord::from))
+            .chain(
+                self.v6
+                    .prefixes_iter(guard)
+                    .map(|r| r.map(PrefixRecord::from)),
+            )
     }
 
     /// Request an iterator over all IPv4 prefixes in the RIB.
@@ -568,8 +572,10 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     pub fn prefixes_iter_v4(
         &'a self,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
-        self.v4.prefixes_iter(guard).map(PrefixRecord::from)
+    ) -> impl Iterator<Item = FatalResult<PrefixRecord<M>>> + 'a {
+        self.v4
+            .prefixes_iter(guard)
+            .map(|r| r.map(PrefixRecord::from))
     }
 
     /// Request an iterator over all IPv6 prefixes in the RIB.
@@ -578,8 +584,10 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     pub fn prefixes_iter_v6(
         &'a self,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = PrefixRecord<M>> + 'a {
-        self.v6.prefixes_iter(guard).map(PrefixRecord::from)
+    ) -> impl Iterator<Item = FatalResult<PrefixRecord<M>>> + 'a {
+        self.v6
+            .prefixes_iter(guard)
+            .map(|r| r.map(PrefixRecord::from))
     }
 
     /// Request an iterator over all persisted prefixes.
@@ -679,7 +687,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         prefix: &Prefix,
         mui: u32,
         ltime: u64,
-    ) -> Result<(), PrefixStoreError> {
+    ) -> FatalResult<()> {
         match prefix.addr() {
             std::net::IpAddr::V4(_addr) => {
                 self.v4.mark_mui_as_active_for_prefix(
@@ -901,7 +909,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         prefix: &Prefix,
         mui: Option<u32>,
         include_withdrawn: bool,
-    ) -> Option<Vec<Record<M>>> {
+    ) -> FatalResult<Option<Vec<Record<M>>>> {
         let guard = &epoch::pin();
 
         match prefix.is_v4() {
diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index 7715e478..7b21e58e 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -3,12 +3,12 @@ use std::path::Path;
 use inetnum::addr::Prefix;
 use log::{info, trace};
 
-use crate::errors::FatalError;
 use crate::prefix_record::Meta;
 use crate::rib::config::PersistStrategy;
 use crate::stats::{Counters, UpsertCounters, UpsertReport};
 use crate::{epoch, Guard};
 
+use crate::errors::{FatalError, FatalResult};
 use crate::prefix_cht::cht::PrefixCht;
 use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
 use crate::types::{PrefixId, RouteStatus};
@@ -68,12 +68,22 @@ impl<
         let persist_tree = match config.persist_strategy() {
             PersistStrategy::MemoryOnly => None,
             _ => {
-                let persist_path =
-                    &config.persist_path().unwrap_or_else(|| {
-                        "Missing persistence path".to_string()
-                    });
-                let pp_ref = &Path::new(persist_path);
-                Some(LsmTree::new(pp_ref))
+                let persist_path = if let Some(pp) = config.persist_path() {
+                    pp
+                } else {
+                    return Err(std::io::Error::new(
+                        std::io::ErrorKind::Other,
+                        "Missing persistence path".to_string(),
+                    )
+                    .into());
+                };
+                let pp_ref = &Path::new(&persist_path);
+                Some(LsmTree::new(pp_ref).map_err(|_| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::Other,
+                        "Cannot create persistence store",
+                    )
+                })?)
             }
         };
 
@@ -223,12 +233,20 @@ impl<
                     let stored_prefixes = p_tree
                         .get_records_with_keys_for_prefix_mui(prefix, mui);
 
-                    for s in stored_prefixes {
-                        let header = ValueHeader {
-                            ltime,
-                            status: RouteStatus::Withdrawn,
-                        };
-                        p_tree.rewrite_header_for_record(header, &s);
+                    for rkv in stored_prefixes {
+                        if let Ok(r) = rkv {
+                            let header = ValueHeader {
+                                ltime,
+                                status: RouteStatus::Withdrawn,
+                            };
+                            p_tree
+                                .rewrite_header_for_record(header, &r)
+                                .map_err(|_| {
+                                    PrefixStoreError::StoreNotReadyError
+                                })?;
+                        } else {
+                            return Err(PrefixStoreError::StoreNotReadyError);
+                        }
                     }
                 } else {
                     return Err(PrefixStoreError::StoreNotReadyError);
@@ -272,14 +290,14 @@ impl<
         prefix: PrefixId<AF>,
         mui: u32,
         ltime: u64,
-    ) -> Result<(), PrefixStoreError> {
+    ) -> FatalResult<()> {
         match self.persist_strategy() {
             PersistStrategy::WriteAhead | PersistStrategy::MemoryOnly => {
                 let (stored_prefix, exists) =
                     self.prefix_cht.non_recursive_retrieve_prefix_mut(prefix);
 
                 if !exists {
-                    return Err(PrefixStoreError::PrefixNotFound);
+                    return Err(FatalError);
                 }
                 stored_prefix.record_map.mark_as_active_for_mui(mui, ltime);
             }
@@ -292,10 +310,11 @@ impl<
                             ltime,
                             status: RouteStatus::Active,
                         };
-                        p_tree.rewrite_header_for_record(header, &record_b);
+                        p_tree
+                            .rewrite_header_for_record(header, &record_b)?;
                     }
                 } else {
-                    return Err(PrefixStoreError::StoreNotReadyError);
+                    return Err(FatalError);
                 }
             }
             PersistStrategy::PersistHistory => {
@@ -304,7 +323,7 @@ impl<
                     self.prefix_cht.non_recursive_retrieve_prefix_mut(prefix);
 
                 if !exists {
-                    return Err(PrefixStoreError::PrefixNotFound);
+                    return Err(FatalError);
                 }
                 stored_prefix.record_map.mark_as_active_for_mui(mui, ltime);
 
@@ -316,7 +335,7 @@ impl<
                         if let Some(p_tree) = self.persist_tree.as_ref() {
                             p_tree
                         } else {
-                            return Err(PrefixStoreError::StoreNotReadyError);
+                            return Err(FatalError);
                         };
 
                     // Here we are keeping persisted history, so no removal of
@@ -392,13 +411,14 @@ impl<
     pub fn prefixes_iter<'a>(
         &'a self,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = (Prefix, Vec<Record<M>>)> + 'a {
+    ) -> impl Iterator<Item = FatalResult<(Prefix, Vec<Record<M>>)>> + 'a
+    {
         self.tree_bitmap.prefixes_iter().map(|p| {
-            (
-                p,
-                self.get_value(p.into(), None, true, guard)
-                    .unwrap_or_default(),
-            )
+            if let Ok(r) = self.get_value(p.into(), None, true, guard) {
+                Ok((p, r.unwrap_or_default()))
+            } else {
+                Err(FatalError)
+            }
         })
     }
 
@@ -410,39 +430,44 @@ impl<
 
     pub(crate) fn persist_prefixes_iter(
         &self,
-    ) -> impl Iterator<Item = Result<(Prefix, Vec<Record<M>>), FatalError>> + '_
+    ) -> impl Iterator<Item = FatalResult<(Prefix, Vec<Record<M>>)>> + '_
     {
-        if let Some(tree) = &self.persist_tree {
-            Some(tree.prefixes_iter().map(|recs| {
-                if let Ok(pfx) = ZeroCopyRecord::<AF>::from_bytes(&recs[0]) {
-                    Ok((
-                        Prefix::from(pfx.prefix),
-                        recs.iter()
-                            .filter_map(|rec| {
-                                if let Ok(rec) =
-                                    ZeroCopyRecord::<AF>::from_bytes(rec)
-                                {
-                                    Some(Record {
-                                        multi_uniq_id: rec.multi_uniq_id,
-                                        ltime: rec.ltime,
-                                        status: rec.status,
-                                        meta: rec.meta.to_vec().into(),
-                                    })
+        self.persist_tree
+            .as_ref()
+            .map(|tree| {
+                tree.prefixes_iter().map(|recs| {
+                    if let Some(Ok(first_rec)) = recs.first() {
+                        if let Ok(pfx) =
+                            ZeroCopyRecord::<AF>::from_bytes(first_rec)
+                        {
+                            let mut rec_vec: Vec<Record<M>> = vec![];
+                            for res_rec in recs.iter() {
+                                if let Ok(rec) = res_rec {
+                                    if let Ok(rec) =
+                                        ZeroCopyRecord::<AF>::from_bytes(rec)
+                                    {
+                                        rec_vec.push(Record {
+                                            multi_uniq_id: rec.multi_uniq_id,
+                                            ltime: rec.ltime,
+                                            status: rec.status,
+                                            meta: rec.meta.to_vec().into(),
+                                        });
+                                    }
                                 } else {
-                                    None
+                                    return Err(FatalError);
                                 }
-                            })
-                            .collect::<Vec<Record<M>>>(),
-                    ))
-                } else {
-                    Err(FatalError)
-                }
-            }))
-        } else {
-            None
-        }
-        .into_iter()
-        .flatten()
+                            }
+                            Ok((Prefix::from(pfx.prefix), rec_vec))
+                        } else {
+                            Err(FatalError)
+                        }
+                    } else {
+                        Err(FatalError)
+                    }
+                })
+            })
+            .into_iter()
+            .flatten()
     }
 
     pub(crate) fn flush_to_disk(&self) -> Result<(), PrefixStoreError> {
diff --git a/src/rib/starcast_af_query.rs b/src/rib/starcast_af_query.rs
index 64056883..ccfd5bf9 100644
--- a/src/rib/starcast_af_query.rs
+++ b/src/rib/starcast_af_query.rs
@@ -3,7 +3,9 @@ use epoch::Guard;
 use log::trace;
 use zerocopy::TryFromBytes;
 
+use crate::errors::{FatalError, FatalResult};
 use crate::match_options::{MatchOptions, MatchType, QueryResult};
+use crate::prefix_record::{PrefixRecord, RecordSet};
 use crate::types::prefix_record::ZeroCopyRecord;
 use crate::types::Record;
 use crate::AddressFamily;
@@ -33,45 +35,70 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> Option<Vec<Record<M>>> {
+    ) -> FatalResult<Option<Vec<Record<M>>>> {
         match self.persist_strategy() {
             PersistStrategy::PersistOnly => {
                 trace!("get value from persist_store for {:?}", prefix_id);
-                self.persist_tree.as_ref().and_then(|tree| {
-                    tree.get_records_for_prefix(
-                        prefix_id,
-                        mui,
-                        include_withdrawn,
-                        self.tree_bitmap.withdrawn_muis_bmin(guard),
-                    )
-                    .map(|v| {
-                        v.iter()
-                            .map(|bytes| {
-                                let record: &ZeroCopyRecord<AF> =
-                                    ZeroCopyRecord::try_ref_from_bytes(
-                                        bytes.as_ref().unwrap(),
-                                    )
-                                    .unwrap();
-                                Record::<M> {
-                                    multi_uniq_id: record.multi_uniq_id,
-                                    ltime: record.ltime,
-                                    status: record.status,
-                                    meta: <Vec<u8>>::from(
-                                        record.meta.as_ref(),
-                                    )
-                                    .into(),
-                                }
-                            })
-                            .collect::<Vec<_>>()
+                self.persist_tree
+                    .as_ref()
+                    .and_then(|tree| {
+                        tree.get_records_for_prefix(
+                            prefix_id,
+                            mui,
+                            include_withdrawn,
+                            self.tree_bitmap.withdrawn_muis_bmin(guard),
+                        )
+                        .map(|v| {
+                            v.iter()
+                                .map(|bytes| {
+                                    if let Ok(b) = bytes.as_ref() {
+                                        let record: &ZeroCopyRecord<AF> =
+                                        ZeroCopyRecord::try_ref_from_bytes(b)
+                                            .map_err(|_| FatalError)?;
+                                        Ok(Record::<M> {
+                                            multi_uniq_id: record
+                                                .multi_uniq_id,
+                                            ltime: record.ltime,
+                                            status: record.status,
+                                            meta: <Vec<u8>>::from(
+                                                record.meta.as_ref(),
+                                            )
+                                            .into(),
+                                        })
+                                    } else {
+                                        Err(FatalError)
+                                    }
+                                })
+                                .collect::<FatalResult<Vec<_>>>()
+                            // let record: FatalResult<&ZeroCopyRecord<AF>> =
+                            //     ZeroCopyRecord::try_ref_from_bytes(
+                            //         bytes
+                            //             .as_ref()
+                            //             .map_err(|_| FatalError)
+                            //             .unwrap(),
+                            //     )
+                            //     .map_err(|_| FatalError);
+                            // Record::<M> {
+                            //     multi_uniq_id: record.multi_uniq_id,
+                            //     ltime: record.ltime,
+                            //     status: record.status,
+                            //     meta: <Vec<u8>>::from(
+                            //         record.meta.as_ref(),
+                            //     )
+                            //     .into(),
+                            // }
+                            // })
+                            // .collect::<Vec<_>>()
+                        })
                     })
-                })
+                    .transpose()
             }
-            _ => self.prefix_cht.get_records_for_prefix(
+            _ => Ok(self.prefix_cht.get_records_for_prefix(
                 prefix_id,
                 mui,
                 include_withdrawn,
                 self.tree_bitmap.withdrawn_muis_bmin(guard),
-            ),
+            )),
         }
     }
 
@@ -81,33 +108,38 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> QueryResult<M> {
+    ) -> FatalResult<QueryResult<M>> {
         let prefix = if !self.contains(prefix_id, mui) {
             Some(Prefix::from(prefix_id))
         } else {
             None
         };
+
+        let prefix_meta = self
+            .get_value(prefix_id, mui, include_withdrawn, guard)?
+            .unwrap_or_default();
+
         let more_specifics = self
             .tree_bitmap
             .more_specific_prefix_iter_from(prefix_id)
             .map(|p| {
                 self.get_value(prefix_id, mui, include_withdrawn, guard)
-                    .map(|v| (p, v))
+                    .map(|res| res.map(|v| (p, v)))
             })
-            .collect();
+            .collect::<FatalResult<Option<RecordSet<M>>>>()?;
 
-        QueryResult {
+        Ok(QueryResult {
             prefix,
-            prefix_meta: prefix
-                .map(|_pfx| {
-                    self.get_value(prefix_id, mui, include_withdrawn, guard)
-                        .unwrap_or_default()
-                })
-                .unwrap_or(vec![]),
+            prefix_meta,
+            // prefix.map(|_pfx| {
+            //     self.get_value(prefix_id, mui, include_withdrawn, guard)?
+            //         .unwrap_or_default()
+            // })
+            // .unwrap_or(vec![]),
             match_type: MatchType::EmptyMatch,
             less_specifics: None,
             more_specifics,
-        }
+        })
     }
 
     pub(crate) fn less_specifics_from(
@@ -116,31 +148,32 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> QueryResult<M> {
+    ) -> FatalResult<QueryResult<M>> {
         let prefix = if !self.contains(prefix_id, mui) {
             Some(Prefix::from(prefix_id))
         } else {
             None
         };
+        let prefix_meta = self
+            .get_value(prefix_id, mui, include_withdrawn, guard)?
+            .unwrap_or_default();
 
         let less_specifics = self
             .tree_bitmap
             .less_specific_prefix_iter(prefix_id)
             .map(|p| {
                 self.get_value(prefix_id, mui, include_withdrawn, guard)
-                    .map(|v| (p, v))
+                    .map(|res| res.map(|v| (p, v)))
             })
-            .collect();
+            .collect::<FatalResult<Option<RecordSet<M>>>>()?;
 
-        QueryResult {
+        Ok(QueryResult {
             prefix,
-            prefix_meta: self
-                .get_value(prefix_id, mui, include_withdrawn, guard)
-                .unwrap_or_default(),
+            prefix_meta,
             match_type: MatchType::EmptyMatch,
             less_specifics,
             more_specifics: None,
-        }
+        })
     }
 
     pub(crate) fn more_specifics_iter_from(
@@ -149,7 +182,8 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<Record<M>>)> + 'a {
+    ) -> impl Iterator<Item = FatalResult<(PrefixId<AF>, Vec<Record<M>>)>> + 'a
+    {
         println!("more_specifics_iter_from fn");
         // If the user wanted a specific mui and not withdrawn prefixes, we
         // may return early if the mui is globally withdrawn.
@@ -163,7 +197,8 @@ impl<
                     .more_specific_prefix_iter_from(prefix_id)
                     .filter_map(move |p| {
                         self.get_value(p, mui, include_withdrawn, guard)
-                            .map(|v| (p, v))
+                            .map(|res| res.map(|v| (p, v)))
+                            .transpose()
                     }),
             )
         })
@@ -199,12 +234,14 @@ impl<
         mui: Option<u32>,
         include_withdrawn: bool,
         guard: &'a Guard,
-    ) -> impl Iterator<Item = (PrefixId<AF>, Vec<Record<M>>)> + 'a {
+    ) -> impl Iterator<Item = FatalResult<(PrefixId<AF>, Vec<Record<M>>)>> + 'a
+    {
         self.tree_bitmap
             .less_specific_prefix_iter(prefix_id)
             .filter_map(move |p| {
                 self.get_value(p, mui, include_withdrawn, guard)
-                    .map(|v| (p, v))
+                    .map(|res| res.map(|v| (p, v)))
+                    .transpose()
             })
     }
 
@@ -213,21 +250,23 @@ impl<
         search_pfx: PrefixId<AF>,
         options: &MatchOptions,
         guard: &'a Guard,
-    ) -> QueryResult<M> {
+    ) -> FatalResult<QueryResult<M>> {
         trace!("match_prefix rib {:?} {:?}", search_pfx, options);
         let res = self.tree_bitmap.match_prefix(search_pfx, options);
 
         trace!("res {:?}", res);
         let mut res = QueryResult::from(res);
 
-        if let Some(Some(m)) = res.prefix.map(|p| {
+        if let Some(Ok(Some(m))) = res.prefix.map(|p| {
             self.get_value(
                 p.into(),
                 options.mui,
                 options.include_withdrawn,
                 guard,
             )
-            .and_then(|v| if v.is_empty() { None } else { Some(v) })
+            .map(|res| {
+                res.and_then(|v| if v.is_empty() { None } else { Some(v) })
+            })
         }) {
             res.prefix_meta = m;
         } else {
@@ -236,45 +275,59 @@ impl<
         }
 
         if options.include_more_specifics {
-            res.more_specifics = res.more_specifics.map(|p| {
-                p.iter()
-                    .filter_map(|mut r| {
-                        if let Some(m) = self.get_value(
-                            r.prefix.into(),
-                            options.mui,
-                            options.include_withdrawn,
-                            guard,
-                        ) {
-                            r.meta = m;
-                            Some(r)
-                        } else {
-                            None
-                        }
-                    })
-                    .collect()
-            });
+            res.more_specifics = res
+                .more_specifics
+                .map(|p| {
+                    p.iter()
+                        .filter_map(|mut r| {
+                            if let Ok(mm) = self.get_value(
+                                r.prefix.into(),
+                                options.mui,
+                                options.include_withdrawn,
+                                guard,
+                            ) {
+                                if let Some(m) = mm {
+                                    r.meta = m;
+                                    Some(Ok(r))
+                                } else {
+                                    None
+                                }
+                            } else {
+                                Some(Err(FatalError))
+                            }
+                        })
+                        .collect::<FatalResult<RecordSet<M>>>()
+                })
+                .transpose()?;
         }
         if options.include_less_specifics {
-            res.less_specifics = res.less_specifics.map(|p| {
-                p.iter()
-                    .filter_map(|mut r| {
-                        if let Some(m) = self.get_value(
-                            r.prefix.into(),
-                            options.mui,
-                            options.include_withdrawn,
-                            guard,
-                        ) {
-                            r.meta = m;
-                            Some(r)
-                        } else {
-                            None
-                        }
-                    })
-                    .collect()
-            });
+            res.less_specifics = res
+                .less_specifics
+                .map(|p| {
+                    p.iter()
+                        .filter_map(|mut r| {
+                            if let Ok(mm) = self.get_value(
+                                r.prefix.into(),
+                                options.mui,
+                                options.include_withdrawn,
+                                guard,
+                            ) {
+                                if let Some(m) = mm {
+                                    r.meta = m;
+                                    Some(Ok(r))
+                                } else {
+                                    None
+                                }
+                            } else {
+                                Some(Err(FatalError))
+                            }
+                        })
+                        .collect::<FatalResult<RecordSet<M>>>()
+                })
+                .transpose()?;
         }
 
-        res
+        Ok(res)
     }
 
     pub(crate) fn best_path(
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 7559aafa..1db26c90 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -235,15 +235,17 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
             default_route_exists: AtomicBool::new(false),
         };
 
-        let _retry_count = tree_bitmap.store_node(
-            StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0),
-            0_u32,
-            TreeBitMapNode {
-                ptrbitarr: AtomicPtrBitArr(AtomicU16::new(0)),
-                pfxbitarr: AtomicPfxBitArr(AtomicU32::new(0)),
-                _af: PhantomData,
-            },
-        )?;
+        let _retry_count = tree_bitmap
+            .store_node(
+                StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0),
+                0_u32,
+                TreeBitMapNode {
+                    ptrbitarr: AtomicPtrBitArr(AtomicU16::new(0)),
+                    pfxbitarr: AtomicPfxBitArr(AtomicU32::new(0)),
+                    _af: PhantomData,
+                },
+            )
+            .map_err(|_| "Cannot create root for in memory tree")?;
 
         Ok(tree_bitmap)
     }
@@ -498,9 +500,11 @@ Giving up this node. This shouldn't happen!",
     ) -> Result<(), PrefixStoreError> {
         let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
 
-        let mut new = unsafe { current.as_ref() }.unwrap().clone();
-        new.remove(mui);
+        let mut new = unsafe { current.as_ref() }
+            .ok_or(PrefixStoreError::StoreNotReadyError)?
+            .clone();
 
+        new.remove(mui);
         self.update_withdrawn_muis_bmin(current, new, guard)
     }
 
@@ -511,7 +515,10 @@ Giving up this node. This shouldn't happen!",
     ) -> Result<(), PrefixStoreError> {
         let current = self.withdrawn_muis_bmin.load(Ordering::Acquire, guard);
 
-        let mut new = unsafe { current.as_ref() }.unwrap().clone();
+        let mut new = unsafe { current.as_ref() }
+            .ok_or(PrefixStoreError::StoreNotReadyError)?
+            .clone();
+
         new.insert(mui);
 
         self.update_withdrawn_muis_bmin(current, new, guard)
@@ -533,8 +540,9 @@ Giving up this node. This shouldn't happen!",
             ) {
                 Ok(_) => return Ok(()),
                 Err(updated) => {
-                    new =
-                        unsafe { updated.current.as_ref() }.unwrap().clone();
+                    new = unsafe { updated.current.as_ref() }
+                        .ok_or(PrefixStoreError::StoreNotReadyError)?
+                        .clone();
                 }
             }
         }
@@ -844,7 +852,6 @@ Giving up this node. This shouldn't happen!",
                     return None;
                 }
                 Some(this_node) => {
-                    // node = this_node;
                     // early return if the mui is not in the index
                     // stored in this node, meaning the mui does not
                     // appear anywhere in the sub-tree formed from
diff --git a/src/types/af.rs b/src/types/af.rs
index ac43108d..ce9eede4 100644
--- a/src/types/af.rs
+++ b/src/types/af.rs
@@ -1,5 +1,5 @@
 use log::trace;
-use zerocopy::{IntoBytes, NetworkEndian, U128, U32};
+use zerocopy::{NetworkEndian, U128, U32};
 
 use crate::types::BitSpan;
 
@@ -78,7 +78,7 @@ pub trait AddressFamily:
     // finding node_ids (always zero for 0/0).
     fn checked_shr_or_zero(self, rhs: u32) -> Self;
 
-    fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE];
+    // fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE];
 }
 
 //-------------- Ipv4 Type --------------------------------------------------
@@ -216,12 +216,12 @@ impl AddressFamily for IPv4 {
         self >> U32::<NetworkEndian>::from(rhs)
     }
 
-    fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
-        *self.as_bytes().first_chunk::<PREFIX_SIZE>().unwrap()
-        // *u32::to_be_bytes(*self)
-        //     .first_chunk::<PREFIX_SIZE>()
-        //     .unwrap()
-    }
+    // fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
+    //     *self.as_bytes().first_chunk::<PREFIX_SIZE>().unwrap()
+    //     // *u32::to_be_bytes(*self)
+    //     //     .first_chunk::<PREFIX_SIZE>()
+    //     //     .unwrap()
+    // }
 }
 
 //-------------- Ipv6 Type --------------------------------------------------
@@ -360,12 +360,12 @@ impl AddressFamily for IPv6 {
         self >> U128::from(rhs as u128)
     }
 
-    fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
-        // *u128::to_be_bytes(*self)
-        //     .first_chunk::<PREFIX_SIZE>()
-        //     .unwrap()
-        *self.as_bytes().first_chunk::<PREFIX_SIZE>().unwrap()
-    }
+    // fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
+    //     // *u128::to_be_bytes(*self)
+    //     //     .first_chunk::<PREFIX_SIZE>()
+    //     //     .unwrap()
+    //     *self.as_bytes().first_chunk::<PREFIX_SIZE>().unwrap()
+    // }
 
     fn from_u8(value: u8) -> Self {
         IPv6::from([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, value])
diff --git a/src/types/errors.rs b/src/types/errors.rs
index 58a01d2d..f76fcd74 100644
--- a/src/types/errors.rs
+++ b/src/types/errors.rs
@@ -99,7 +99,7 @@ impl fmt::Display for PrefixStoreError {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Copy, Clone)]
 pub struct FatalError;
 
 impl std::fmt::Display for FatalError {
@@ -113,3 +113,17 @@ impl std::fmt::Display for FatalError {
 }
 
 pub type FatalResult<T> = Result<T, FatalError>;
+
+impl std::error::Error for FatalError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        None
+    }
+
+    fn description(&self) -> &str {
+        "description() is deprecated; use Display"
+    }
+
+    fn cause(&self) -> Option<&dyn std::error::Error> {
+        self.source()
+    }
+}
diff --git a/src/types/prefix_id.rs b/src/types/prefix_id.rs
index a8f463a9..efbc7c7b 100644
--- a/src/types/prefix_id.rs
+++ b/src/types/prefix_id.rs
@@ -67,14 +67,14 @@ impl<AF: AddressFamily> PrefixId<AF> {
     // The lsm tree, used for persistence, stores the prefix in the key with
     // len first, so that key range lookups can be made for more-specifics in
     // each prefix length.
-    pub fn len_first_bytes<const PREFIX_SIZE: usize>(
-        &self,
-    ) -> [u8; PREFIX_SIZE] {
-        let bytes = &mut [0_u8; PREFIX_SIZE];
-        *bytes.last_chunk_mut::<4>().unwrap() = self.net.to_be_bytes();
-        bytes[0] = self.len;
-        *bytes
-    }
+    // pub fn len_first_bytes<const PREFIX_SIZE: usize>(
+    //     &self,
+    // ) -> [u8; PREFIX_SIZE] {
+    //     let bytes = &mut [0_u8; PREFIX_SIZE];
+    //     *bytes.last_chunk_mut::<4>().unwrap() = self.net.to_be_bytes();
+    //     bytes[0] = self.len;
+    //     *bytes
+    // }
 }
 
 impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
diff --git a/src/types/prefix_record.rs b/src/types/prefix_record.rs
index 156ab000..75e03b52 100644
--- a/src/types/prefix_record.rs
+++ b/src/types/prefix_record.rs
@@ -79,7 +79,7 @@ pub(crate) struct ZeroCopyRecord<AF: AddressFamily> {
 
 impl<AF: AddressFamily> ZeroCopyRecord<AF> {
     pub(crate) fn from_bytes(b: &[u8]) -> Result<&Self, FatalError> {
-        Self::try_ref_from_bytes(b).or_else(|_| Err(FatalError))
+        Self::try_ref_from_bytes(b).map_err(|_| FatalError)
     }
 }
 

From cc1c3f737922e3dcb89403aadbbbec7d54e9df39 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 12:55:15 +0100
Subject: [PATCH 129/147] ? all the tests

---
 src/lib.rs                          |  2 ++
 src/tree_bitmap/mod.rs              | 24 +++++++++++++++++++++---
 src/tree_bitmap/tree_bitmap_node.rs |  4 +---
 src/types/af.rs                     |  2 +-
 src/types/prefix_id.rs              |  8 ++++----
 5 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 3475cd56..3fc89319 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,3 +1,5 @@
+#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
+
 //! A library that provides abstractions for a BGP Routing Information Base
 //! (RIB) for different AFI/SAFI types, as a database.
 //!
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 1db26c90..1916e112 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -9,6 +9,7 @@ pub(crate) use tree_bitmap_node::{
     NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter, StrideNodeId,
     TreeBitMapNode,
 };
+use zerocopy::FromZeros;
 
 // ----------- THE STORE ----------------------------------------------------
 //
@@ -237,7 +238,10 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
 
         let _retry_count = tree_bitmap
             .store_node(
-                StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0),
+                StrideNodeId::dangerously_new_with_id_as_is(
+                    <AF as FromZeros>::new_zeroed(),
+                    0,
+                ),
                 0_u32,
                 TreeBitMapNode {
                     ptrbitarr: AtomicPtrBitArr(AtomicU16::new(0)),
@@ -857,7 +861,18 @@ Giving up this node. This shouldn't happen!",
                     // appear anywhere in the sub-tree formed from
                     // this node.
                     node = this_node;
-                    let bmin = this_node.node_set.rbm().read().unwrap();
+
+                    let bmin = match this_node.node_set.rbm().read() {
+                        Ok(bmin) => bmin,
+                        // if this lock is poisened, we are still going to
+                        // work with the bmin. The data in the bmin may be
+                        // stale, because of the lock poisoning, but this may
+                        // also happen because of delays in other parts of the
+                        // store in normal circumstances. We are counting on a
+                        // future call to a write method to actually propagate
+                        // a FatalError to the user of the store.
+                        Err(bmin) => bmin.into_inner(),
+                    };
                     if !bmin.contains(mui) {
                         return None;
                     }
@@ -882,7 +897,10 @@ Giving up this node. This shouldn't happen!",
     }
 
     pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
-        StrideNodeId::dangerously_new_with_id_as_is(AF::zero(), 0)
+        StrideNodeId::dangerously_new_with_id_as_is(
+            <AF as FromZeros>::new_zeroed(),
+            0,
+        )
     }
 
     pub fn get_nodes_count(&self) -> usize {
diff --git a/src/tree_bitmap/tree_bitmap_node.rs b/src/tree_bitmap/tree_bitmap_node.rs
index 50dee3d0..2b0655c3 100644
--- a/src/tree_bitmap/tree_bitmap_node.rs
+++ b/src/tree_bitmap/tree_bitmap_node.rs
@@ -501,9 +501,7 @@ pub(crate) fn ptr_range(ptrbitarr: u16, bs: BitSpan) -> (u16, u8) {
     let stop: u8 = start + (1 << (4 - bs.len));
     let mask: u16 = (((1_u32 << (stop as u32 - start as u32)) - 1)
         .rotate_right(stop as u32)
-        >> 16)
-        .try_into()
-        .unwrap();
+        >> 16) as u16;
     if log_enabled!(log::Level::Trace) {
         trace!("- mask      {:032b}", mask);
         trace!("- ptrbitarr {:032b}", ptrbitarr);
diff --git a/src/types/af.rs b/src/types/af.rs
index ce9eede4..c983bbef 100644
--- a/src/types/af.rs
+++ b/src/types/af.rs
@@ -28,7 +28,7 @@ pub trait AddressFamily:
     // + Zero
     + Copy
     + Ord
-    + zerocopy::TryFromBytes
+    + zerocopy::FromBytes
     + zerocopy::IntoBytes
     + zerocopy::KnownLayout
     + zerocopy::Immutable
diff --git a/src/types/prefix_id.rs b/src/types/prefix_id.rs
index efbc7c7b..123a9b32 100644
--- a/src/types/prefix_id.rs
+++ b/src/types/prefix_id.rs
@@ -1,4 +1,4 @@
-use zerocopy::TryFromBytes;
+use zerocopy::{FromBytes, TryFromBytes};
 
 use crate::AddressFamily;
 
@@ -10,7 +10,7 @@ use crate::AddressFamily;
     Debug,
     Copy,
     Clone,
-    zerocopy::TryFromBytes,
+    zerocopy::FromBytes,
     zerocopy::IntoBytes,
     zerocopy::KnownLayout,
     zerocopy::Immutable,
@@ -104,7 +104,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<[u8; PREFIX_SIZE]>
 {
     fn from(value: [u8; PREFIX_SIZE]) -> Self {
         Self {
-            net: *AF::try_ref_from_bytes(&value.as_slice()[1..]).unwrap(),
+            net: *AF::ref_from_bytes(&value.as_slice()[1..]).unwrap(),
             len: value[0],
         }
     }
@@ -114,6 +114,6 @@ impl<'a, AF: AddressFamily, const PREFIX_SIZE: usize>
     From<&'a [u8; PREFIX_SIZE]> for &'a PrefixId<AF>
 {
     fn from(value: &'a [u8; PREFIX_SIZE]) -> Self {
-        PrefixId::try_ref_from_bytes(value.as_slice()).unwrap()
+        PrefixId::ref_from_bytes(value.as_slice()).unwrap()
     }
 }

From f6915569020b744d0c2289a8074667ebbe84c44d Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 12:55:36 +0100
Subject: [PATCH 130/147] ? all the tests

---
 examples/full_table_multiple_trees_json.rs |   2 +-
 examples/multi_no_thread.rs                |   2 +-
 examples/multi_thread_1.rs                 |   2 +-
 examples/multi_thread_2.rs                 |   2 +-
 examples/multi_thread_3.rs                 |   2 +-
 examples/multi_thread_4.rs                 |   2 +-
 examples/multi_thread_multi_prefix.rs      |   1 +
 examples/multi_thread_single_prefix.rs     |   2 +-
 tests/best-path.rs                         |   2 +-
 tests/concurrency.rs                       | 367 +++++++--------------
 tests/full-table.rs                        |   9 +-
 tests/more-more-specifics.rs               |   5 +-
 tests/more-specifics.rs                    |   6 +-
 tests/treebitmap.rs                        |  67 ++--
 tests/treebitmap_v6.rs                     |  28 +-
 15 files changed, 195 insertions(+), 304 deletions(-)

diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index 9987180b..6660e6db 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -99,7 +99,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                                     include_history: IncludeHistory::None,
                                 },
                                 guard,
-                            );
+                            )?;
                         }
                     }
                 }
diff --git a/examples/multi_no_thread.rs b/examples/multi_no_thread.rs
index d490484c..d2a8317f 100644
--- a/examples/multi_no_thread.rs
+++ b/examples/multi_no_thread.rs
@@ -67,7 +67,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_history: IncludeHistory::None,
         },
         guard,
-    );
+    )?;
     println!("query result");
     println!("{}", s_spfx);
     println!("{}", s_spfx.more_specifics.unwrap());
diff --git a/examples/multi_thread_1.rs b/examples/multi_thread_1.rs
index 3460e0f5..9331def9 100644
--- a/examples/multi_thread_1.rs
+++ b/examples/multi_thread_1.rs
@@ -65,7 +65,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         println!("query result");
         println!("{}", s_spfx);
         println!("{}", s_spfx.more_specifics.unwrap());
diff --git a/examples/multi_thread_2.rs b/examples/multi_thread_2.rs
index 9ed7e0cb..e18d7d44 100644
--- a/examples/multi_thread_2.rs
+++ b/examples/multi_thread_2.rs
@@ -76,7 +76,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_history: IncludeHistory::None,
         },
         guard,
-    );
+    )?;
     println!("query result");
     println!("{}", s_spfx);
     println!("{}", s_spfx.more_specifics.unwrap());
diff --git a/examples/multi_thread_3.rs b/examples/multi_thread_3.rs
index eb0a3eb4..167dfb61 100644
--- a/examples/multi_thread_3.rs
+++ b/examples/multi_thread_3.rs
@@ -103,7 +103,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_history: IncludeHistory::None,
         },
         guard,
-    );
+    )?;
     println!("query result");
     println!("{}", s_spfx);
     println!("{}", s_spfx.more_specifics.unwrap());
diff --git a/examples/multi_thread_4.rs b/examples/multi_thread_4.rs
index 4de0929c..10e62cd2 100644
--- a/examples/multi_thread_4.rs
+++ b/examples/multi_thread_4.rs
@@ -139,7 +139,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             include_history: IncludeHistory::None,
         },
         guard,
-    );
+    )?;
     println!("query result");
     println!("{}", s_spfx);
     println!("{}", s_spfx.more_specifics.unwrap());
diff --git a/examples/multi_thread_multi_prefix.rs b/examples/multi_thread_multi_prefix.rs
index a132172d..e63098fa 100644
--- a/examples/multi_thread_multi_prefix.rs
+++ b/examples/multi_thread_multi_prefix.rs
@@ -87,6 +87,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                     },
                                     guard,
                                 )
+                                .unwrap()
                                 .prefix_meta;
                             x += 1;
                         }
diff --git a/examples/multi_thread_single_prefix.rs b/examples/multi_thread_single_prefix.rs
index f56eefd4..562e25dd 100644
--- a/examples/multi_thread_single_prefix.rs
+++ b/examples/multi_thread_single_prefix.rs
@@ -74,7 +74,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                             include_history: IncludeHistory::None,
                         },
                         guard,
-                    ).prefix_meta;
+                    ).unwrap().prefix_meta;
                             x += 1;
                         }
 
diff --git a/tests/best-path.rs b/tests/best-path.rs
index 7298d307..562331e1 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -190,7 +190,7 @@ fn test_best_path_1(// tree_bitmap: MultiThreadedStore<Ipv4Route>,
         &rotonda_store::epoch::pin(),
     );
 
-    println!("{:?}", res.prefix_meta);
+    println!("{:?}", res.as_ref().unwrap().prefix_meta);
     let best_path = tree_bitmap.best_path(&pfx, &rotonda_store::epoch::pin());
     println!(
         "ps outdated? {}",
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index 0c3054f4..d22cc312 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -2,8 +2,9 @@ use std::{str::FromStr, sync::atomic::Ordering};
 
 use inetnum::{addr::Prefix, asn::Asn};
 use rotonda_store::{
+    errors::FatalResult,
     match_options::{IncludeHistory, MatchOptions, MatchType},
-    prefix_record::{Record, RouteStatus},
+    prefix_record::{PrefixRecord, Record, RouteStatus},
     rib::{
         config::{Config, MemoryOnlyConfig},
         StarCastRib,
@@ -28,6 +29,49 @@ rotonda_store::all_strategies![
     BeBytesAsn
 ];
 
+fn iter(
+    pfxs_iter: &[FatalResult<PrefixRecord<BeBytesAsn>>],
+    pfx: Prefix,
+) -> impl Iterator<Item = &Record<BeBytesAsn>> + '_ {
+    pfxs_iter
+        .iter()
+        .find(|p| p.as_ref().unwrap().prefix == pfx)
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .meta
+        .iter()
+}
+
+fn iter_len(
+    pfxs_iter: &[FatalResult<PrefixRecord<BeBytesAsn>>],
+    pfx: Prefix,
+) -> usize {
+    pfxs_iter
+        .iter()
+        .find(|p| p.as_ref().unwrap().prefix == pfx)
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .meta
+        .len()
+}
+
+fn first_meta(
+    pfxs_iter: &[FatalResult<PrefixRecord<BeBytesAsn>>],
+    pfx: Prefix,
+) -> BeBytesAsn {
+    pfxs_iter
+        .iter()
+        .find(|p| p.as_ref().unwrap().prefix == pfx)
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .meta[0]
+        .meta
+        .clone()
+}
+
 fn test_concurrent_updates_1<C: Config + Sync + Send + 'static>(
     tree_bitmap: StarCastRib<BeBytesAsn, C>,
 ) -> Result<(), Box<dyn std::error::Error>> {
@@ -138,188 +182,89 @@ fn test_concurrent_updates_1<C: Config + Sync + Send + 'static>(
     assert!(tree_bitmap.contains(&pfx, Some(1)));
     assert!(tree_bitmap.contains(&pfx, Some(2)));
 
-    assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
         .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+        .any(|p| p.as_ref().unwrap().prefix == pfx));
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 1 && m.meta == 65501.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 2 && m.meta == 65502.into()));
 
     let pfx = Prefix::from_str("185.34.10.0/24").unwrap();
-    assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
         .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+        .any(|p| p.as_ref().unwrap().prefix == pfx));
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 1 && m.meta == 65501.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 2 && m.meta == 65502.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 3 && m.meta == 65503.into()));
 
     let pfx = Prefix::from_str("185.34.11.0/24").unwrap();
-    assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
         .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+        .any(|p| pfx == p.as_ref().unwrap().prefix));
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 1 && m.meta == 65501.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 2 || m.meta == 65502.into())));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 3 || m.meta == 65503.into())));
 
     let pfx = Prefix::from_str("185.34.11.0/24").unwrap();
-    assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
         .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+        .any(|p| p.as_ref().unwrap().prefix == pfx));
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 1 && m.meta == 65501.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 2 || m.meta == 65502.into())));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 3 && m.meta == 65503.into())));
 
     let pfx = Prefix::from_str("185.34.12.0/24").unwrap();
-    assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
         .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+        .any(|p| p.as_ref().unwrap().prefix == pfx));
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 2 && m.meta == 65502.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 3 && m.meta == 65503.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 1 || m.meta == 65501.into())));
 
     let pfx = Prefix::from_str("183.0.0.0/8")?;
-    assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
         .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+        .any(|p| p.as_ref().unwrap().prefix == pfx));
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 1 || m.meta == 65501.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 2 && m.meta == 65502.into())));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 3 && m.meta == 65503.into())));
 
     let pfx = Prefix::from_str("186.0.0.0/8")?;
-    assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
         .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+        .any(|p| p.as_ref().unwrap().prefix == pfx));
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 2 && m.meta == 65502.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 1 || m.meta == 65501.into())));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 3 && m.meta == 65503.into())));
 
     let pfx = Prefix::from_str("187.0.0.0/8")?;
-    assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
         .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+        .any(|p| p.as_ref().unwrap().prefix == pfx));
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 3 && m.meta == 65503.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 2 && m.meta == 65502.into())));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .all(|m| !(m.multi_uniq_id == 1 || m.meta == 65501.into())));
 
     // Create Withdrawals
@@ -367,7 +312,7 @@ fn test_concurrent_updates_1<C: Config + Sync + Send + 'static>(
 
     for pfx in pfx_vec_2 {
         let guard = rotonda_store::epoch::pin();
-        let res = tree_bitmap.match_prefix(&pfx, &match_options, &guard);
+        let res = tree_bitmap.match_prefix(&pfx, &match_options, &guard)?;
         assert_eq!(res.prefix, Some(pfx));
         println!("strategy {:?}", tree_bitmap.persist_strategy());
         println!("PFX {}", res);
@@ -472,145 +417,57 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
     let all_pfxs_iter = tree_bitmap.prefixes_iter(guard).collect::<Vec<_>>();
 
     let pfx = Prefix::from_str("185.33.0.0/16").unwrap();
-    assert!(all_pfxs_iter.iter().any(|p| p.prefix == pfx));
     assert!(all_pfxs_iter
         .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+        .any(|p| p.as_ref().unwrap().prefix == pfx));
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 1 && m.meta == 65501.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 2 && m.meta == 65502.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 3 && m.meta == 65503.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 4 && m.meta == 65504.into()));
 
     let pfx = Prefix::from_str("185.34.0.0/16").unwrap();
-    assert_eq!(
-        all_pfxs_iter
-            .iter()
-            .find(|p| p.prefix == pfx)
-            .unwrap()
-            .meta
-            .len(),
-        2
-    );
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert_eq!(iter_len(&all_pfxs_iter, pfx), 2);
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 1 && m.meta == 65501.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 2 && m.meta == 65502.into()));
 
     let pfx = Prefix::from_str("185.34.14.0/24").unwrap();
+    assert_eq!(iter_len(&all_pfxs_iter, pfx), 1);
     assert_eq!(
         all_pfxs_iter
             .iter()
-            .find(|p| p.prefix == pfx)
+            .find(|p| p.as_ref().unwrap().prefix == pfx)
             .unwrap()
-            .meta
-            .len(),
-        1
-    );
-    assert_eq!(
-        all_pfxs_iter.iter().find(|p| p.prefix == pfx).unwrap().meta[0].meta,
+            .as_ref()
+            .unwrap()
+            .meta[0]
+            .meta,
         Asn::from_u32(65503).into()
     );
 
     let pfx = Prefix::from_str("187.0.0.0/8").unwrap();
-    assert_eq!(
-        all_pfxs_iter
-            .iter()
-            .find(|p| p.prefix == pfx)
-            .unwrap()
-            .meta
-            .len(),
-        1
-    );
-    assert_eq!(
-        all_pfxs_iter.iter().find(|p| p.prefix == pfx).unwrap().meta[0].meta,
-        Asn::from_u32(65504).into()
-    );
+    assert_eq!(iter_len(&all_pfxs_iter, pfx), 1);
+    assert_eq!(first_meta(&all_pfxs_iter, pfx), Asn::from_u32(65504).into());
 
     let pfx = Prefix::from_str("185.35.0.0/16").unwrap();
-    assert_eq!(
-        all_pfxs_iter
-            .iter()
-            .find(|p| p.prefix == pfx)
-            .unwrap()
-            .meta
-            .len(),
-        1
-    );
-    assert_eq!(
-        all_pfxs_iter.iter().find(|p| p.prefix == pfx).unwrap().meta[0].meta,
-        Asn::from_u32(65501).into()
-    );
+    assert_eq!(iter_len(&all_pfxs_iter, pfx), 1);
+    assert_eq!(first_meta(&all_pfxs_iter, pfx), Asn::from_u32(65501).into());
 
     let pfx = Prefix::from_str("185.34.15.0/24").unwrap();
-    assert_eq!(
-        all_pfxs_iter
-            .iter()
-            .find(|p| p.prefix == pfx)
-            .unwrap()
-            .meta
-            .len(),
-        2
-    );
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert_eq!(iter_len(&all_pfxs_iter, pfx), 2);
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 2 && m.meta == 65502.into()));
-    assert!(all_pfxs_iter
-        .iter()
-        .find(|p| p.prefix == pfx)
-        .unwrap()
-        .meta
-        .iter()
+    assert!(iter(&all_pfxs_iter, pfx)
         .any(|m| m.multi_uniq_id == 3 && m.meta == 65503.into()));
 
     let pfx = Prefix::from_str("188.0.0.0/8").unwrap();
-    assert_eq!(
-        all_pfxs_iter
-            .iter()
-            .find(|p| p.prefix == pfx)
-            .unwrap()
-            .meta
-            .len(),
-        1
-    );
-    assert_eq!(
-        all_pfxs_iter.iter().find(|p| p.prefix == pfx).unwrap().meta[0].meta,
-        Asn::from_u32(65504).into()
-    );
+    assert_eq!(iter_len(&all_pfxs_iter, pfx), 1);
+    assert_eq!(first_meta(&all_pfxs_iter, pfx), Asn::from_u32(65504).into());
 
     // Create Withdrawals
     let wd_pfxs = [pfx_vec_1[1], pfx_vec_2[1], pfx_vec_3[1]];
@@ -649,10 +506,11 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
     for pfx in wd_pfxs {
         let guard = rotonda_store::epoch::pin();
         let res = tree_bitmap.match_prefix(&pfx, &match_options, &guard);
-        assert_eq!(res.prefix, Some(pfx));
+        assert_eq!(res.as_ref().unwrap().prefix, Some(pfx));
         println!("RES {:#?}", res);
         assert_eq!(
-            res.prefix_meta
+            res.unwrap()
+                .prefix_meta
                 .iter()
                 .find(|m| m.multi_uniq_id == 2)
                 .unwrap()
@@ -706,6 +564,7 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
             &match_options,
             &guard,
         )
+        .unwrap()
         .more_specifics
         .unwrap();
     println!("0/2 {}", mp02);
@@ -721,12 +580,18 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
 
     let active_len = all_pfxs_iter
         .iter()
-        .filter(|p| p.meta.iter().all(|m| m.status == RouteStatus::Active))
+        .filter(|p| {
+            p.as_ref()
+                .unwrap()
+                .meta
+                .iter()
+                .all(|m| m.status == RouteStatus::Active)
+        })
         .collect::<Vec<_>>()
         .len();
     assert_eq!(active_len, all_pfxs_iter.len());
     // let len_2 = res0.more_specifics.unwrap().v4.len()
-    assert_eq!(active_len, res128.more_specifics.unwrap().v4.len());
+    assert_eq!(active_len, res128?.more_specifics.unwrap().v4.len());
 
     Ok(())
 }
@@ -803,18 +668,18 @@ fn more_specifics_short_lengths() -> Result<(), Box<dyn std::error::Error>> {
 
     println!(
         "more specifics#0: {}",
-        m.more_specifics.as_ref().unwrap()[0]
+        m.as_ref().unwrap().more_specifics.as_ref().unwrap()[0]
     );
     println!(
         "more specifics#1: {}",
-        m.more_specifics.as_ref().unwrap()[1]
+        m.as_ref().unwrap().more_specifics.as_ref().unwrap()[1]
     );
     println!(
         "more specifics#2: {}",
-        m.more_specifics.as_ref().unwrap()[2]
+        m.as_ref().unwrap().more_specifics.as_ref().unwrap()[2]
     );
 
-    assert_eq!(m.more_specifics.map(|mp| mp.len()), Some(3));
+    assert_eq!(m.unwrap().more_specifics.map(|mp| mp.len()), Some(3));
 
     Ok(())
 }
diff --git a/tests/full-table.rs b/tests/full-table.rs
index 336a49b8..417879d8 100644
--- a/tests/full-table.rs
+++ b/tests/full-table.rs
@@ -150,10 +150,13 @@ mod tests {
                     guard,
                 );
 
-                if query.prefix.is_none() {
+                if query.as_ref().unwrap().prefix.is_none() {
                     panic!("STOPSTOPSTOPST");
                 } else {
-                    assert_eq!(query.prefix.unwrap(), pfx.prefix);
+                    assert_eq!(
+                        query.as_ref().unwrap().prefix.unwrap(),
+                        pfx.prefix
+                    );
                 }
             }
 
@@ -188,7 +191,7 @@ mod tests {
                             },
                             guard,
                         );
-                        if let Some(_pfx) = res.prefix {
+                        if let Some(_pfx) = res.as_ref().unwrap().prefix {
                             // println!("_pfx {:?}", _pfx);
                             // println!("pfx {:?}", pfx);
                             // println!("{:#?}", res);
diff --git a/tests/more-more-specifics.rs b/tests/more-more-specifics.rs
index f48d03dd..7f221824 100644
--- a/tests/more-more-specifics.rs
+++ b/tests/more-more-specifics.rs
@@ -96,11 +96,12 @@ fn test_more_specifics_without_less_specifics<C: Config>(
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         println!("em/m-s: {:#?}", found_result);
 
         let more_specifics = found_result
             .more_specifics
+            .as_ref()
             .unwrap()
             .iter()
             .filter(|p| p.prefix != spfx.0.unwrap())
@@ -194,7 +195,7 @@ fn test_more_specifics_with_less_specifics<C: Config>(
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         println!("em/m-s: {}", found_result);
 
         let more_specifics = found_result
diff --git a/tests/more-specifics.rs b/tests/more-specifics.rs
index 288120aa..9d494775 100644
--- a/tests/more-specifics.rs
+++ b/tests/more-specifics.rs
@@ -189,7 +189,7 @@ fn test_more_specifics<C: Config>(
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         // println!("em/m-s: {:#?}", found_result);
         //
         println!("search prefix: {}", spfx.0.unwrap());
@@ -217,7 +217,7 @@ fn test_more_specifics<C: Config>(
         for (i, p) in tree_bitmap
             .prefixes_iter_v4(guard)
             .enumerate()
-            .map(|(i, p)| (i, p.prefix))
+            .map(|(i, p)| (i, p.as_ref().unwrap().prefix))
         {
             println!("ms {}: {}", i, p);
         }
@@ -303,7 +303,7 @@ fn test_brunos_more_specifics<C: Config>(
             include_history: IncludeHistory::None,
         },
         guard,
-    );
+    )?;
 
     assert!(found_result.more_specifics.unwrap().is_empty());
     Ok(())
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index 08f50dd7..c9b8b2a0 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -62,7 +62,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         println!("prefix: {:?}", &expect_pfx);
         println!("result: {:#?}", &res);
         assert!(res.prefix.is_some());
@@ -96,7 +96,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         assert!(res.prefix.is_some());
         assert_eq!(res.prefix, Some(expect_pfx?));
         Ok(())
@@ -361,8 +361,9 @@ mod tests {
         let guard = &epoch::pin();
         for pfx in tree_bitmap.prefixes_iter(guard) {
             // let pfx_nm = pfx.strip_meta();
+            let pfx = pfx.unwrap().prefix;
             let res = tree_bitmap.match_prefix(
-                &pfx.prefix,
+                &pfx,
                 &MatchOptions {
                     match_type: MatchType::LongestMatch,
                     include_withdrawn: false,
@@ -372,10 +373,10 @@ mod tests {
                     include_history: IncludeHistory::None,
                 },
                 guard,
-            );
+            )?;
             println!("PFX {}", pfx);
             println!("RES {}", res);
-            assert_eq!(res.prefix.unwrap(), pfx.prefix);
+            assert_eq!(res.prefix.unwrap(), pfx);
         }
 
         let res = tree_bitmap.match_prefix(
@@ -389,7 +390,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         println!("prefix {:?}", res.prefix);
         println!("res: {}", &res);
 
@@ -481,7 +482,7 @@ mod tests {
                             include_history: IncludeHistory::None,
                         },
                         guard,
-                    );
+                    )?;
                     println!("{:?}", pfx);
 
                     assert_eq!(res.prefix.unwrap(), res_pfx?);
@@ -575,6 +576,7 @@ mod tests {
             .iter_records_for_mui_v4(5, false, guard)
             .collect::<Vec<_>>()
         {
+            let rec = rec.unwrap();
             println!("{}", rec);
 
             assert_eq!(rec.meta.len(), 1);
@@ -585,7 +587,7 @@ mod tests {
             .iter_records_for_mui_v4(1, false, guard)
             .collect::<Vec<_>>()
         {
-            println!("{}", rec);
+            println!("{}", rec.unwrap());
         }
 
         // println!("all records");
@@ -607,7 +609,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         print!(".pfx {:#?}.", all_recs_for_pfx);
         assert_eq!(all_recs_for_pfx.prefix_meta.len(), 5);
         let wd_rec = all_recs_for_pfx
@@ -629,7 +631,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         assert_eq!(active_recs_for_pfx.prefix_meta.len(), 4);
         assert!(!active_recs_for_pfx
             .prefix_meta
@@ -644,11 +646,13 @@ mod tests {
         let all_recs = tree_bitmap.prefixes_iter(guard);
 
         for rec in tree_bitmap.prefixes_iter(guard).collect::<Vec<_>>() {
+            let rec = rec.unwrap();
             println!("{}", rec);
         }
 
-        let mui_2_recs =
-            all_recs.filter_map(|r| r.get_record_for_mui(2).cloned());
+        let mui_2_recs = all_recs.filter_map(|r| {
+            r.as_ref().unwrap().get_record_for_mui(2).cloned()
+        });
         let wd_2_rec = mui_2_recs
             .filter(|r| r.status == RouteStatus::Withdrawn)
             .collect::<Vec<_>>();
@@ -656,14 +660,22 @@ mod tests {
         assert_eq!(wd_2_rec[0].multi_uniq_id, 2);
 
         let mui_2_recs = tree_bitmap.prefixes_iter(guard).filter_map(|r| {
-            r.get_record_for_mui(2).cloned().map(|rec| (r.prefix, rec))
+            r.as_ref()
+                .unwrap()
+                .get_record_for_mui(2)
+                .cloned()
+                .map(|rec| (r.as_ref().unwrap().prefix, rec))
         });
         println!("mui_2_recs prefixes_iter");
         for rec in mui_2_recs {
             println!("{} {:#?}", rec.0, rec.1);
         }
         let mui_2_recs = tree_bitmap.prefixes_iter(guard).filter_map(|r| {
-            r.get_record_for_mui(2).cloned().map(|rec| (r.prefix, rec))
+            r.as_ref()
+                .unwrap()
+                .get_record_for_mui(2)
+                .cloned()
+                .map(|rec| (r.as_ref().unwrap().prefix, rec))
         });
 
         let active_2_rec = mui_2_recs
@@ -675,6 +687,7 @@ mod tests {
         let mui_2_recs = tree_bitmap.iter_records_for_mui_v4(2, false, guard);
         println!("mui_2_recs iter_records_for_mui_v4");
         for rec in mui_2_recs {
+            let rec = rec.unwrap();
             println!("{} {:#?}", rec.prefix, rec.meta);
         }
 
@@ -692,7 +705,11 @@ mod tests {
         assert_eq!(mui_1_recs.len(), 4);
         println!("mui_1_recs iter_records_for_mui_v4 w/ withdrawn");
         for rec in mui_1_recs {
-            assert_eq!(rec.meta[0].status, RouteStatus::Withdrawn);
+            let rec = rec.unwrap();
+            assert_eq!(
+                rec.meta.first().unwrap().status,
+                RouteStatus::Withdrawn
+            );
         }
 
         //--------------
@@ -708,12 +725,13 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
 
         println!("more_specifics match {} w/ withdrawn", more_specifics);
 
         let guard = &rotonda_store::epoch::pin();
         for p in tree_bitmap.prefixes_iter_v4(guard) {
+            let p = p.unwrap();
             println!("{}", p);
         }
 
@@ -754,7 +772,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
 
         println!("more_specifics match {} w/o withdrawn", more_specifics);
         let more_specifics = more_specifics.more_specifics.unwrap();
@@ -800,7 +818,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
 
         println!("more_specifics match w/o withdrawn #2 {}", more_specifics);
         // We withdrew mui 1 for the requested prefix itself, since mui 2 was
@@ -858,10 +876,11 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         println!("more_specifics match w/o withdrawn #3 {}", more_specifics);
 
-        // This prefix should not be found, since we withdrew all records for it.
+        // This prefix should not be found, since we withdrew all records
+        // for it.
         assert!(more_specifics.prefix_meta.is_empty());
 
         // ..as a result, its resulting match_type should be EmptyMatch
@@ -914,7 +933,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
 
         trace!("{:#?}", query);
 
@@ -931,7 +950,9 @@ mod tests {
         println!("less_specifics match w/o withdrawn #5");
 
         trace!("mark {} as active", wd_pfx);
-        tree_bitmap.mark_mui_as_active_for_prefix(&wd_pfx, 5, 1)?;
+        tree_bitmap
+            .mark_mui_as_active_for_prefix(&wd_pfx, 5, 1)
+            .unwrap();
 
         let less_specifics = tree_bitmap.match_prefix(
             &Prefix::from_str("1.0.0.0/17")?,
@@ -944,7 +965,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         let less_specifics = less_specifics.less_specifics.unwrap();
         println!("{:#?}", less_specifics);
 
diff --git a/tests/treebitmap_v6.rs b/tests/treebitmap_v6.rs
index 13a1e42e..9ecd5b7d 100644
--- a/tests/treebitmap_v6.rs
+++ b/tests/treebitmap_v6.rs
@@ -63,7 +63,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         println!("prefix: {:?}", &expect_pfx);
         println!("result: {:#?}", &res);
         assert!(res.prefix.is_some());
@@ -114,7 +114,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         println!("prefix: {}", &expect_pfx.unwrap());
         println!("result: {}", &res);
         assert!(res.prefix.is_some());
@@ -157,7 +157,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         assert!(res.prefix.is_some());
         assert_eq!(res.prefix, Some(expect_pfx?));
         Ok(())
@@ -330,9 +330,9 @@ mod tests {
 
         let guard = &epoch::pin();
         for pfx in tree_bitmap.prefixes_iter(guard) {
-            // let pfx_nm = pfx.strip_meta();
+            let pfx = pfx.as_ref().unwrap().prefix;
             let res = tree_bitmap.match_prefix(
-                &pfx.prefix,
+                &pfx,
                 &MatchOptions {
                     match_type: MatchType::LongestMatch,
                     include_withdrawn: false,
@@ -342,9 +342,9 @@ mod tests {
                     include_history: IncludeHistory::None,
                 },
                 guard,
-            );
+            )?;
             println!("{}", pfx);
-            assert_eq!(res.prefix.unwrap(), pfx.prefix);
+            assert_eq!(res.prefix.unwrap(), pfx);
         }
 
         Ok(())
@@ -611,9 +611,9 @@ mod tests {
 
         let guard = &epoch::pin();
         for pfx in tree_bitmap.prefixes_iter(guard) {
-            // let pfx_nm = pfx.strip_meta();
+            let pfx = pfx.unwrap().prefix;
             let res = tree_bitmap.match_prefix(
-                &pfx.prefix,
+                &pfx,
                 &MatchOptions {
                     match_type: MatchType::LongestMatch,
                     include_withdrawn: false,
@@ -623,9 +623,9 @@ mod tests {
                     include_history: IncludeHistory::None,
                 },
                 guard,
-            );
+            )?;
             println!("{}", pfx);
-            assert_eq!(res.prefix.unwrap(), pfx.prefix);
+            assert_eq!(res.prefix.unwrap(), pfx);
         }
 
         let res = tree_bitmap.match_prefix(
@@ -642,7 +642,7 @@ mod tests {
                 include_history: IncludeHistory::None,
             },
             guard,
-        );
+        )?;
         println!("prefix {:?}", res.prefix);
         println!("res: {:#?}", &res);
 
@@ -731,7 +731,7 @@ mod tests {
                             include_history: IncludeHistory::None,
                         },
                         guard,
-                    );
+                    )?;
                     println!("{:?}", pfx);
 
                     assert_eq!(res.prefix.unwrap(), res_pfx?);
@@ -793,7 +793,7 @@ mod tests {
                             include_history: IncludeHistory::None,
                         },
                         guard,
-                    );
+                    )?;
                     println!("{:?}", pfx);
 
                     assert_eq!(res.prefix.unwrap(), res_pfx?);

From 604122b698cfa029a2044972a6a929f2bcfc214c Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 14:39:01 +0100
Subject: [PATCH 131/147] verify all slice indexing

---
 src/cht/mod.rs                      |  4 ++
 src/cht/oncebox.rs                  |  4 ++
 src/lib.rs                          |  7 ++-
 src/lsm_tree/mod.rs                 | 22 ++++++-
 src/prefix_cht/cht.rs               |  6 +-
 src/prefix_cht/iterators.rs         |  7 +--
 src/rib/starcast.rs                 | 10 +++-
 src/rib/starcast_af.rs              | 26 +++++---
 src/rib/starcast_af_query.rs        |  2 +-
 src/tree_bitmap/mod.rs              | 43 +++++++++----
 src/tree_bitmap/node_cht.rs         |  8 +--
 src/tree_bitmap/tree_bitmap_node.rs |  1 +
 src/types/af.rs                     | 93 ++++-------------------------
 src/types/errors.rs                 | 19 ++++++
 src/types/prefix_id.rs              | 30 ++++------
 src/types/prefix_record.rs          |  2 +
 src/types/stats.rs                  | 26 +++++---
 src/types/test_types.rs             |  2 +
 18 files changed, 168 insertions(+), 144 deletions(-)

diff --git a/src/cht/mod.rs b/src/cht/mod.rs
index 930075c7..ea32b46f 100644
--- a/src/cht/mod.rs
+++ b/src/cht/mod.rs
@@ -24,6 +24,10 @@ impl<V: Value, const ROOT_SIZE: usize, const STRIDES_PER_BUCKET: usize>
         }))
     }
 
+    // There cannot be a root node for a prefix length that has NO slots,
+    // STRIDES_PER_BICKET (a instance wide const) should always be bigger
+    // than 0.
+    #[allow(clippy::indexing_slicing)]
     pub(crate) fn root_for_len(&self, len: u8) -> &V {
         &self.0[len as usize / STRIDES_PER_BUCKET]
     }
diff --git a/src/cht/oncebox.rs b/src/cht/oncebox.rs
index 76c4d063..96d058fa 100644
--- a/src/cht/oncebox.rs
+++ b/src/cht/oncebox.rs
@@ -89,6 +89,10 @@ impl<T> OnceBoxSlice<T> {
         }
     }
 
+    // This is a bit tricky: the caller of this method should make sure that
+    // the slice has enough elements. For performance reasons we are NOT
+    // checking that here.
+    #[allow(clippy::indexing_slicing)]
     pub fn get_or_init(
         &self,
         idx: usize,
diff --git a/src/lib.rs b/src/lib.rs
index 3fc89319..32e79aa7 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,4 +1,9 @@
-#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
+#![deny(
+    clippy::unwrap_used,
+    clippy::expect_used,
+    clippy::panic,
+    clippy::indexing_slicing
+)]
 
 //! A library that provides abstractions for a BGP Routing Information Base
 //! (RIB) for different AFI/SAFI types, as a database.
diff --git a/src/lsm_tree/mod.rs b/src/lsm_tree/mod.rs
index e173bc9f..ee76088d 100644
--- a/src/lsm_tree/mod.rs
+++ b/src/lsm_tree/mod.rs
@@ -12,7 +12,7 @@ use zerocopy::{
     Unaligned, U32, U64,
 };
 
-use crate::errors::{FatalError, FatalResult};
+use crate::errors::{FatalError, FatalResult, PrefixStoreError};
 use crate::prefix_record::Meta;
 use crate::stats::Counters;
 use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
@@ -153,12 +153,19 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
         self.tree.insert::<&[u8], &[u8]>(key, value, 0)
     }
 
+    // This is not production code yet. To be re-evaluated if it does become
+    // production code.
+    #[allow(clippy::indexing_slicing)]
     pub fn _remove(&self, key: &[u8]) {
         self.tree.remove_weak(key, 0);
         // the first byte of the prefix holds the length of the prefix.
         self.counters._dec_prefixes_count(key[0]);
     }
 
+    // Based on the properties of the lsm_tree we can assume that the key and
+    // value concatenated in this method always has a lenght of greater than
+    // KEYS_SIZE, a global constant for the store per AF.
+    #[allow(clippy::indexing_slicing)]
     pub fn get_records_for_prefix(
         &self,
         prefix: PrefixId<AF>,
@@ -634,8 +641,17 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
         self.counters.get_prefixes_count().iter().sum()
     }
 
-    pub fn get_prefixes_count_for_len(&self, len: u8) -> usize {
-        self.counters.get_prefixes_count()[len as usize]
+    //
+    #[allow(clippy::indexing_slicing)]
+    pub fn get_prefixes_count_for_len(
+        &self,
+        len: u8,
+    ) -> Result<usize, PrefixStoreError> {
+        if len <= AF::BITS {
+            Ok(self.counters.get_prefixes_count()[len as usize])
+        } else {
+            Err(PrefixStoreError::StoreNotReadyError)
+        }
     }
 
     pub(crate) fn persist_record_w_long_key<M: Meta>(
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index 0d49eb13..11b93227 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -35,6 +35,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         Self(Arc::new(Mutex::new(record_map)))
     }
 
+    #[allow(clippy::type_complexity)]
     fn acquire_write_lock(
         &self,
     ) -> Result<
@@ -756,7 +757,10 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
     // This function is used by the match_prefix, and [more|less]_specifics
     // public methods on the TreeBitMap (indirectly).
     #[allow(clippy::type_complexity)]
-    pub fn non_recursive_retrieve_prefix(
+    // This method can never run out of levels, since it only continues if it
+    // finds occupied child slots. The indexing can therefore not crash.
+    #[allow(clippy::indexing_slicing)]
+    pub(crate) fn non_recursive_retrieve_prefix(
         &self,
         id: PrefixId<AF>,
     ) -> (
diff --git a/src/prefix_cht/iterators.rs b/src/prefix_cht/iterators.rs
index e28748de..945dffc0 100644
--- a/src/prefix_cht/iterators.rs
+++ b/src/prefix_cht/iterators.rs
@@ -1,11 +1,10 @@
-use log::trace;
-use roaring::RoaringBitmap;
-
 use crate::{
     tree_bitmap::{NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter},
     types::{AddressFamily, BitSpan, PrefixId},
     TreeBitMap,
 };
+use log::trace;
+use roaring::RoaringBitmap;
 
 // This iterator is unused right now: all iterators go over the in-memory
 // treebitmap, and retreive metadata based on the persist_strategy per prefix
@@ -13,7 +12,6 @@ use crate::{
 //
 // However this tree may ultimately be more efficient for the MemoryOnly
 // strategy.
-
 pub(crate) struct _MoreSpecificPrefixIter<
     'a,
     AF: AddressFamily,
@@ -32,6 +30,7 @@ pub(crate) struct _MoreSpecificPrefixIter<
     include_withdrawn: bool,
 }
 
+#[allow(clippy::unwrap_used)]
 impl<'a, AF: AddressFamily + 'a, const ROOT_SIZE: usize> Iterator
     for _MoreSpecificPrefixIter<'a, AF, ROOT_SIZE>
 {
diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index fa7f1591..79269b52 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -822,7 +822,10 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// Note that this counter may be lower than the actual
     /// number in the store, due to contention at the time of
     /// reading the value.
-    pub fn prefixes_v4_count_for_len(&self, len: u8) -> UpsertCounters {
+    pub fn prefixes_v4_count_for_len(
+        &self,
+        len: u8,
+    ) -> Result<UpsertCounters, PrefixStoreError> {
         self.v4.get_prefixes_count_for_len(len)
     }
 
@@ -841,7 +844,10 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// Note that this counter may be lower than the actual
     /// number in the store, due to contention at the time of
     /// reading the value.
-    pub fn prefixes_v6_count_for_len(&self, len: u8) -> UpsertCounters {
+    pub fn prefixes_v6_count_for_len(
+        &self,
+        len: u8,
+    ) -> Result<UpsertCounters, PrefixStoreError> {
         self.v6.get_prefixes_count_for_len(len)
     }
 
diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index 7b21e58e..1aaa7af6 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -397,14 +397,24 @@ impl<
         }
     }
 
-    pub fn get_prefixes_count_for_len(&self, len: u8) -> UpsertCounters {
-        UpsertCounters {
-            in_memory_count: self.tree_bitmap.get_prefixes_count_for_len(len),
-            persisted_count: self
-                .persist_tree
-                .as_ref()
-                .map_or(0, |p| p.get_prefixes_count_for_len(len)),
-            total_count: self.counters.get_prefixes_count()[len as usize],
+    // the len check does it all.
+    #[allow(clippy::indexing_slicing, clippy::unwrap_used)]
+    pub fn get_prefixes_count_for_len(
+        &self,
+        len: u8,
+    ) -> Result<UpsertCounters, PrefixStoreError> {
+        if len <= AF::BITS {
+            Ok(UpsertCounters {
+                in_memory_count: self
+                    .tree_bitmap
+                    .get_prefixes_count_for_len(len)?,
+                persisted_count: self.persist_tree.as_ref().map_or(0, |p| {
+                    p.get_prefixes_count_for_len(len).unwrap()
+                }),
+                total_count: self.counters.get_prefixes_count()[len as usize],
+            })
+        } else {
+            Err(PrefixStoreError::PrefixLengthInvalid)
         }
     }
 
diff --git a/src/rib/starcast_af_query.rs b/src/rib/starcast_af_query.rs
index ccfd5bf9..989a135c 100644
--- a/src/rib/starcast_af_query.rs
+++ b/src/rib/starcast_af_query.rs
@@ -5,7 +5,7 @@ use zerocopy::TryFromBytes;
 
 use crate::errors::{FatalError, FatalResult};
 use crate::match_options::{MatchOptions, MatchType, QueryResult};
-use crate::prefix_record::{PrefixRecord, RecordSet};
+use crate::prefix_record::RecordSet;
 use crate::types::prefix_record::ZeroCopyRecord;
 use crate::types::Record;
 use crate::AddressFamily;
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 1916e112..eabda825 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -197,6 +197,7 @@ use zerocopy::FromZeros;
 // produce a fix for it).
 
 use crate::cht::{nodeset_size, prev_node_size, Cht, Value};
+use crate::errors::{FatalError, FatalResult};
 use crate::rib::STRIDE_SIZE;
 use crate::stats::Counters;
 use crate::types::{BitSpan, PrefixId};
@@ -270,7 +271,8 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
                 !self.default_route_exists.swap(true, Ordering::Acquire);
             return self
                 .update_default_route_prefix_meta(mui)
-                .map(|(rc, _mui_exists)| (rc, !prefix_new));
+                .map(|(rc, _mui_exists)| (rc, !prefix_new))
+                .map_err(|_| PrefixStoreError::StoreNotReadyError);
         }
 
         let mut stride_end: u8 = 0;
@@ -472,7 +474,7 @@ Giving up this node. This shouldn't happen!",
     fn update_default_route_prefix_meta(
         &self,
         mui: u32,
-    ) -> Result<(u32, bool), PrefixStoreError> {
+    ) -> FatalResult<(u32, bool)> {
         trace!("Updating the default route...");
 
         if let Some(_root_node) =
@@ -482,7 +484,7 @@ Giving up this node. This shouldn't happen!",
                 .root_for_len(self.get_root_node_id().len())
                 .update_rbm_index(mui)
         } else {
-            Err(PrefixStoreError::StoreNotReadyError)
+            Err(FatalError)
         }
     }
 
@@ -557,7 +559,7 @@ Giving up this node. This shouldn't happen!",
         id: StrideNodeId<AF>,
         multi_uniq_id: u32,
         new_node: TreeBitMapNode<AF>,
-    ) -> Result<(StrideNodeId<AF>, u32), PrefixStoreError> {
+    ) -> Result<(StrideNodeId<AF>, u32), FatalError> {
         if log_enabled!(log::Level::Trace) {
             debug!(
                 "{} store: Store node {}: {:?} mui {}",
@@ -697,11 +699,13 @@ Giving up this node. This shouldn't happen!",
                             next_bit_shift if next_bit_shift > 0 => {
                                 nodes = &stored_node.node_set;
                             }
-                            // There's no next level!
-                            _ => panic!(
-                                "out of storage levels, current level is {}",
-                                level
-                            ),
+                            // There's no next level anymore, we ran out of
+                            // the maximum number of levels for this AF. This
+                            // should happen under no circumstance, there's a
+                            // serious logic error here somewhere.
+                            _ => {
+                                return Err(FatalError);
+                            }
                         }
                     }
                 }
@@ -911,8 +915,17 @@ Giving up this node. This shouldn't happen!",
         self.counters.get_prefixes_count().iter().sum()
     }
 
-    pub fn get_prefixes_count_for_len(&self, len: u8) -> usize {
-        self.counters.get_prefixes_count()[len as usize]
+    // len checking does it all
+    #[allow(clippy::indexing_slicing)]
+    pub fn get_prefixes_count_for_len(
+        &self,
+        len: u8,
+    ) -> Result<usize, PrefixStoreError> {
+        if len <= AF::BITS {
+            Ok(self.counters.get_prefixes_count()[len as usize])
+        } else {
+            Err(PrefixStoreError::PrefixLengthInvalid)
+        }
     }
 
     // Stride related methods
@@ -1121,8 +1134,12 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> std::fmt::Display
                 _f,
                 "{}",
                 Colour::Green.paint(
-                    bars[((prefix_count as u32 % SCALE) / (SCALE / 7))
-                        as usize]
+                    *bars
+                        .get(
+                            ((prefix_count as u32 % SCALE) / (SCALE / 7))
+                                as usize
+                        )
+                        .unwrap_or(&"NaN")
                 ) //  = scale / 7
             )?;
 
diff --git a/src/tree_bitmap/node_cht.rs b/src/tree_bitmap/node_cht.rs
index cc739119..8b9f3280 100644
--- a/src/tree_bitmap/node_cht.rs
+++ b/src/tree_bitmap/node_cht.rs
@@ -5,6 +5,7 @@ use log::{debug, log_enabled};
 use roaring::RoaringBitmap;
 
 use crate::cht::{Cht, OnceBoxSlice, Value};
+use crate::errors::{FatalError, FatalResult};
 use crate::types::AddressFamily;
 
 use super::tree_bitmap_node::{StrideNodeId, TreeBitMapNode};
@@ -42,15 +43,12 @@ impl<AF: AddressFamily> NodeSet<AF> {
     pub(crate) fn update_rbm_index(
         &self,
         multi_uniq_id: u32,
-    ) -> Result<(u32, bool), PrefixStoreError>
+    ) -> FatalResult<(u32, bool)>
     where
         AF: crate::types::AddressFamily,
     {
         let try_count = 0;
-        let mut rbm = self
-            .1
-            .write()
-            .map_err(|_| PrefixStoreError::StoreNotReadyError)?;
+        let mut rbm = self.1.write().map_err(|_| FatalError)?;
         let absent = rbm.insert(multi_uniq_id);
 
         Ok((try_count, !absent))
diff --git a/src/tree_bitmap/tree_bitmap_node.rs b/src/tree_bitmap/tree_bitmap_node.rs
index 2b0655c3..24a7edab 100644
--- a/src/tree_bitmap/tree_bitmap_node.rs
+++ b/src/tree_bitmap/tree_bitmap_node.rs
@@ -440,6 +440,7 @@ impl<AF: AddressFamily> std::iter::Iterator
 //     _s: PhantomData<S>,
 // }
 
+#[allow(clippy::indexing_slicing)]
 pub const fn ms_prefix_mask_arr(bs: BitSpan) -> u32 {
     [
         0b_01111111111111111111111111111110, // bits = 0, len = 0
diff --git a/src/types/af.rs b/src/types/af.rs
index c983bbef..d42190e2 100644
--- a/src/types/af.rs
+++ b/src/types/af.rs
@@ -15,9 +15,6 @@ pub trait AddressFamily:
     + std::fmt::Debug
     + std::hash::Hash
     + std::fmt::Display
-    // + From<u32>
-    // + From<u16>
-    // + From<u8>
     + Eq
     + std::ops::BitAnd<Output = Self>
     + std::ops::BitOr<Output = Self>
@@ -25,7 +22,6 @@ pub trait AddressFamily:
     + std::ops::Shl<Output = Self>
     + std::ops::Shl<Self, Output = Self>
     + std::ops::Sub<Output = Self>
-    // + Zero
     + Copy
     + Ord
     + zerocopy::FromBytes
@@ -34,8 +30,6 @@ pub trait AddressFamily:
     + zerocopy::Immutable
     + zerocopy::Unaligned
 {
-    /// The byte representation of the family filled with 1s.
-    // const BITMASK: Self;
     /// The number of bits in the byte representation of the family.
     const BITS: u8;
     type Inner: Into<Self> + From<u32> + From<u8>;
@@ -62,23 +56,16 @@ pub trait AddressFamily:
 
     fn truncate_to_len(self, len: u8) -> Self;
 
-    // fn from_ipaddr(net: std::net::IpAddr) -> Self;
-
     fn into_ipaddr(self) -> std::net::IpAddr;
 
     // temporary function, this will botch IPv6 completely.
     fn dangerously_truncate_to_u32(self) -> u32;
 
-    // temporary function, this will botch IPv6 completely.
-    // fn dangerously_truncate_to_usize(self) -> usize;
-
     // For the sake of searching for 0/0, check the the right shift, since
     // since shifting with MAXLEN (32 in Ipv4, or 128 in IPv6) will panic
     // in debug mode. A failed check will simply retutrn zero. Used in
     // finding node_ids (always zero for 0/0).
     fn checked_shr_or_zero(self, rhs: u32) -> Self;
-
-    // fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE];
 }
 
 //-------------- Ipv4 Type --------------------------------------------------
@@ -87,7 +74,6 @@ pub trait AddressFamily:
 pub type IPv4 = zerocopy::U32<NetworkEndian>;
 
 impl AddressFamily for IPv4 {
-    // const BITMASK: u32 = 0x1u32.rotate_right(1);
     const BITS: u8 = 32;
     type Inner = u32;
     type InnerIpAddr = std::net::Ipv4Addr;
@@ -170,18 +156,6 @@ impl AddressFamily for IPv4 {
         (res, len + bs.len)
     }
 
-    // fn from_ipaddr(addr: std::net::IpAddr) -> U32<NetworkEndian> {
-    //     // Well, this is awkward.
-    //     if let std::net::IpAddr::V4(addr) = addr {
-    //         (addr.octets()[0] as u32) << 24
-    //             | (addr.octets()[1] as u32) << 16
-    //             | (addr.octets()[2] as u32) << 8
-    //             | (addr.octets()[3] as u32)
-    //     } else {
-    //         panic!("Can't convert IPv6 to IPv4");
-    //     }
-    // }
-
     fn into_ipaddr(self) -> std::net::IpAddr {
         std::net::IpAddr::V4(std::net::Ipv4Addr::from(u32::from(self)))
     }
@@ -191,11 +165,12 @@ impl AddressFamily for IPv4 {
         self.into()
     }
 
-    // fn dangerously_truncate_to_usize(self) -> usize {
-    //     // not dangerous at all.
-    //     <usize>::from(self)
-    // }
-
+    // We are totally allowing panic here: the panicking arm holds an
+    // invariant that's a super basic assumption of this whole store. If this
+    // panics than this whole library should not be used, and be checked for
+    // logic errors everywhere. For performance reasons we are leaving out the
+    // FatalResult wrapper.
+    #[allow(clippy::panic)]
     fn truncate_to_len(self, len: u8) -> Self {
         match len {
             0 => U32::new(0),
@@ -215,13 +190,6 @@ impl AddressFamily for IPv4 {
         }
         self >> U32::<NetworkEndian>::from(rhs)
     }
-
-    // fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
-    //     *self.as_bytes().first_chunk::<PREFIX_SIZE>().unwrap()
-    //     // *u32::to_be_bytes(*self)
-    //     //     .first_chunk::<PREFIX_SIZE>()
-    //     //     .unwrap()
-    // }
 }
 
 //-------------- Ipv6 Type --------------------------------------------------
@@ -295,6 +263,12 @@ impl AddressFamily for IPv6 {
         (res, len + bs.len)
     }
 
+    // We are totally allowing panic here: the panicking arm holds an
+    // invariant that's a super basic assumption of this whole store. If this
+    // panics than this whole library should not be used, and be checked for
+    // logic errors everywhere. For performance reasons we are leaving out the
+    // FatalResult wrapper.
+    #[allow(clippy::panic)]
     fn truncate_to_len(self, len: u8) -> Self {
         match len {
             0 => U128::new(0),
@@ -307,37 +281,6 @@ impl AddressFamily for IPv6 {
         }
     }
 
-    // fn truncate_to_len(self, len: u8) -> Self {
-    //     if (128 - len) == 0 {
-    //         0
-    //     } else {
-    //         (self >> (128 - len)) << (128 - len)
-    //     }
-    // }
-
-    // fn from_ipaddr(net: std::net::IpAddr) -> u128 {
-    //     if let std::net::IpAddr::V6(addr) = net {
-    //         addr.octets()[15] as u128
-    //             | (addr.octets()[14] as u128) << 8
-    //             | (addr.octets()[13] as u128) << 16
-    //             | (addr.octets()[12] as u128) << 24
-    //             | (addr.octets()[11] as u128) << 32
-    //             | (addr.octets()[10] as u128) << 40
-    //             | (addr.octets()[9] as u128) << 48
-    //             | (addr.octets()[8] as u128) << 56
-    //             | (addr.octets()[7] as u128) << 64
-    //             | (addr.octets()[6] as u128) << 72
-    //             | (addr.octets()[5] as u128) << 80
-    //             | (addr.octets()[4] as u128) << 88
-    //             | (addr.octets()[3] as u128) << 96
-    //             | (addr.octets()[2] as u128) << 104
-    //             | (addr.octets()[1] as u128) << 112
-    //             | (addr.octets()[0] as u128) << 120
-    //     } else {
-    //         panic!("Can't convert IPv4 to IPv6");
-    //     }
-    // }
-
     fn into_ipaddr(self) -> std::net::IpAddr {
         std::net::IpAddr::V6(std::net::Ipv6Addr::from(u128::from(self)))
     }
@@ -347,11 +290,6 @@ impl AddressFamily for IPv6 {
         u128::from(self) as u32
     }
 
-    // fn dangerously_truncate_to_usize(self) -> usize {
-    //     // this will chop off the high bits.
-    //     self as usize
-    // }
-
     fn checked_shr_or_zero(self, rhs: u32) -> Self {
         if rhs == 0 || rhs == 128 {
             return U128::from(0);
@@ -360,13 +298,6 @@ impl AddressFamily for IPv6 {
         self >> U128::from(rhs as u128)
     }
 
-    // fn to_be_bytes<const PREFIX_SIZE: usize>(&self) -> [u8; PREFIX_SIZE] {
-    //     // *u128::to_be_bytes(*self)
-    //     //     .first_chunk::<PREFIX_SIZE>()
-    //     //     .unwrap()
-    //     *self.as_bytes().first_chunk::<PREFIX_SIZE>().unwrap()
-    // }
-
     fn from_u8(value: u8) -> Self {
         IPv6::from([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, value])
     }
diff --git a/src/types/errors.rs b/src/types/errors.rs
index f76fcd74..9f540a71 100644
--- a/src/types/errors.rs
+++ b/src/types/errors.rs
@@ -14,12 +14,20 @@ pub enum PrefixStoreError {
     /// been met, and may never be met. Retrying is safe, but may result in
     /// the same error. Therefore it should probably be retried only once.
     StoreNotReadyError,
+    /// An unrecoverable error occurred, most probably during disk IO, or a
+    /// poisoned lock while writing. The store is probably corrupt. The caller
+    /// should terminate the store, and probably also terminate itself. This
+    /// error variant is the same as the `FatalError` type, but is used as a
+    /// return for methods that can also return non-fatal errors.
+    FatalError,
     /// A best path was requested, but the selection procedure was performed
     /// on a route set that is now stale. A new best path calculation over the
     /// set should be performed before retrying.
     PathSelectionOutdated,
     /// The requested prefix was not found in the store.
     PrefixNotFound,
+    /// The requested prefix length cannot exist.
+    PrefixLengthInvalid,
     /// A best path was requested, but it was never calculated. Perform a best
     ///path selection first, before retrying.
     BestPathNotFound,
@@ -61,6 +69,9 @@ impl fmt::Display for PrefixStoreError {
             PrefixStoreError::PrefixNotFound => {
                 write!(f, "Error: The Prefix cannot be found.")
             }
+            PrefixStoreError::PrefixLengthInvalid => {
+                write!(f, "Error: The specified Prefix length is invalid.")
+            }
             PrefixStoreError::BestPathNotFound => {
                 write!(
                     f,
@@ -95,6 +106,14 @@ impl fmt::Display for PrefixStoreError {
                     counters cannot be reported for persist only strategy."
                 )
             }
+            PrefixStoreError::FatalError => {
+                write!(
+                    f,
+                    "FATAL: An unrecoverable error occurred during disk I/O \
+                    or writing memory. All data in the store should be \
+                    considered corrupy and the application should terminate."
+                )
+            }
         }
     }
 }
diff --git a/src/types/prefix_id.rs b/src/types/prefix_id.rs
index 123a9b32..33bf7cfe 100644
--- a/src/types/prefix_id.rs
+++ b/src/types/prefix_id.rs
@@ -1,4 +1,4 @@
-use zerocopy::{FromBytes, TryFromBytes};
+use zerocopy::FromBytes;
 
 use crate::AddressFamily;
 
@@ -23,30 +23,19 @@ pub struct PrefixId<AF: AddressFamily> {
 }
 
 impl<AF: AddressFamily> PrefixId<AF> {
-    pub fn new(net: AF, len: u8) -> Self {
+    pub(crate) fn new(net: AF, len: u8) -> Self {
         PrefixId { len, net }
     }
 
-    pub fn get_net(&self) -> AF {
+    pub(crate) fn get_net(&self) -> AF {
         self.net
     }
 
-    pub fn get_len(&self) -> u8 {
+    pub(crate) fn get_len(&self) -> u8 {
         self.len
     }
 
-    // Increment the length of the prefix without changing the bits part.
-    // This is used to iterate over more-specific prefixes for this prefix,
-    // since the more specifics iterator includes the requested `base_prefix`
-    // itself.
-    pub fn inc_len(self) -> Self {
-        Self {
-            net: self.net,
-            len: self.len + 1,
-        }
-    }
-
-    pub fn truncate_to_len(self, len: u8) -> Self {
+    pub(crate) fn truncate_to_len(self, len: u8) -> Self {
         // trace!("orig {:032b}", self.net);
         // trace!(
         //     "new  {:032b}",
@@ -58,7 +47,6 @@ impl<AF: AddressFamily> PrefixId<AF> {
         //     len
         // );
         Self {
-            // net: (self.net >> (AF::BITS - len)) << (AF::BITS - len),
             net: self.net.truncate_to_len(len),
             len,
         }
@@ -77,6 +65,7 @@ impl<AF: AddressFamily> PrefixId<AF> {
     // }
 }
 
+#[allow(clippy::unwrap_used)]
 impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
     fn from(value: inetnum::addr::Prefix) -> Self {
         Self {
@@ -93,27 +82,34 @@ impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
     }
 }
 
+// There is no reasonable way for this to panic, PrefixId and inetnum's Prefix
+// represent the same data in slightly different ways.
+#[allow(clippy::unwrap_used)]
 impl<AF: AddressFamily> From<PrefixId<AF>> for inetnum::addr::Prefix {
     fn from(value: PrefixId<AF>) -> Self {
         Self::new(value.get_net().into_ipaddr(), value.get_len()).unwrap()
     }
 }
 
+#[allow(clippy::unwrap_used, clippy::indexing_slicing)]
 impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<[u8; PREFIX_SIZE]>
     for PrefixId<AF>
 {
     fn from(value: [u8; PREFIX_SIZE]) -> Self {
         Self {
+            // This cannot panic for values of PREFIX_SIZE greater than 1
             net: *AF::ref_from_bytes(&value.as_slice()[1..]).unwrap(),
             len: value[0],
         }
     }
 }
 
+#[allow(clippy::unwrap_used)]
 impl<'a, AF: AddressFamily, const PREFIX_SIZE: usize>
     From<&'a [u8; PREFIX_SIZE]> for &'a PrefixId<AF>
 {
     fn from(value: &'a [u8; PREFIX_SIZE]) -> Self {
+        // This cannot panic for values of PREFIX_SIZE greater than 1
         PrefixId::ref_from_bytes(value.as_slice()).unwrap()
     }
 }
diff --git a/src/types/prefix_record.rs b/src/types/prefix_record.rs
index 75e03b52..f7abd18c 100644
--- a/src/types/prefix_record.rs
+++ b/src/types/prefix_record.rs
@@ -306,6 +306,8 @@ impl<'a, M: Meta + 'a> std::iter::FromIterator<&'a PrefixRecord<M>>
 impl<M: Meta> std::ops::Index<usize> for RecordSet<M> {
     type Output = PrefixRecord<M>;
 
+    // This does not change the behaviour of the Index trait
+    #[allow(clippy::indexing_slicing)]
     fn index(&self, index: usize) -> &Self::Output {
         if index < self.v4.len() {
             &self.v4[index]
diff --git a/src/types/stats.rs b/src/types/stats.rs
index 5ed8e4ca..e0e924c1 100644
--- a/src/types/stats.rs
+++ b/src/types/stats.rs
@@ -10,7 +10,7 @@ use crate::{rib::STRIDE_SIZE, types::AddressFamily};
 
 pub(crate) struct StrideStats<AF: AddressFamily> {
     pub(crate) created_nodes: Vec<CreatedNodes>,
-    pub(crate) prefixes_num: Vec<CreatedNodes>,
+    pub(crate) _prefixes_num: Vec<CreatedNodes>,
     _af: PhantomData<AF>,
 }
 
@@ -18,7 +18,7 @@ impl<AF: AddressFamily> StrideStats<AF> {
     pub fn new() -> Self {
         Self {
             created_nodes: Self::nodes_vec(AF::BITS / STRIDE_SIZE),
-            prefixes_num: Self::nodes_vec(AF::BITS / STRIDE_SIZE),
+            _prefixes_num: Self::nodes_vec(AF::BITS / STRIDE_SIZE),
             _af: PhantomData,
         }
     }
@@ -42,12 +42,16 @@ impl<AF: AddressFamily> StrideStats<AF> {
         vec
     }
 
-    pub fn inc(&mut self, depth_level: u8) {
-        self.created_nodes[depth_level as usize].count += 1;
+    pub fn _inc(&mut self, depth_level: u8) {
+        if let Some(n) = self.created_nodes.get_mut(depth_level as usize) {
+            n.count += 1
+        }
     }
 
-    pub fn inc_prefix_count(&mut self, depth_level: u8) {
-        self.prefixes_num[depth_level as usize].count += 1;
+    pub fn _inc_prefix_count(&mut self, depth_level: u8) {
+        if let Some(p) = self._prefixes_num.get_mut(depth_level as usize) {
+            p.count += 1;
+        }
     }
 }
 
@@ -134,11 +138,15 @@ impl Counters {
     }
 
     pub fn inc_prefixes_count(&self, len: u8) {
-        self.prefixes[len as usize].fetch_add(1, Ordering::Relaxed);
+        if let Some(p) = self.prefixes.get(len as usize) {
+            p.fetch_add(1, Ordering::Relaxed);
+        }
     }
 
     pub fn _dec_prefixes_count(&self, len: u8) {
-        self.prefixes[len as usize].fetch_sub(1, Ordering::Relaxed);
+        if let Some(p) = self.prefixes.get(len as usize) {
+            p.fetch_sub(1, Ordering::Relaxed);
+        }
     }
 
     pub fn get_prefix_stats(&self) -> Vec<CreatedNodes> {
@@ -164,6 +172,8 @@ impl Counters {
     }
 }
 
+// How can this unwrap in here ever fail?
+#[allow(clippy::unwrap_used)]
 impl Default for Counters {
     fn default() -> Self {
         let mut prefixes: Vec<AtomicUsize> = Vec::with_capacity(129);
diff --git a/src/types/test_types.rs b/src/types/test_types.rs
index 7c40df00..227d06bc 100644
--- a/src/types/test_types.rs
+++ b/src/types/test_types.rs
@@ -118,6 +118,8 @@ impl AsRef<[u8]> for PrefixAs {
     }
 }
 
+// This is no production code, crash all you want
+#[allow(clippy::unwrap_used)]
 impl From<Vec<u8>> for PrefixAs {
     fn from(value: Vec<u8>) -> Self {
         Self(*value.first_chunk::<4>().unwrap())

From b9a60149c5abb4a6bdf878babe553d124ef7fa38 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 15:24:19 +0100
Subject: [PATCH 132/147] doc strings

---
 src/types/af.rs        | 100 +++++------------------------------------
 src/types/bit_span.rs  |  13 ++++++
 src/types/prefix_id.rs |  28 +++---------
 3 files changed, 31 insertions(+), 110 deletions(-)

diff --git a/src/types/af.rs b/src/types/af.rs
index d42190e2..ebcd7e10 100644
--- a/src/types/af.rs
+++ b/src/types/af.rs
@@ -32,7 +32,13 @@ pub trait AddressFamily:
 {
     /// The number of bits in the byte representation of the family.
     const BITS: u8;
+
+    /// The type actually holding the value, u32 for IPv4, and u128 for IPv6.
     type Inner: Into<Self> + From<u32> + From<u8>;
+
+    /// The std::net that the value of self belongs to. So,
+    /// [std::net::Ipv4Addr], and [std::net::Ipv6Addr] for IPv4, and IPv6
+    /// respectively.
     type InnerIpAddr;
 
     fn new(value: Self::Inner) -> Self {
@@ -46,19 +52,22 @@ pub trait AddressFamily:
 
     fn zero() -> Self;
 
-    fn fmt_net(net: Self) -> String;
     // returns the specified nibble from `start_bit` to (and including)
     // `start_bit + len` and shifted to the right.
     fn into_bit_span(net: Self, start_bit: u8, len: u8) -> BitSpan;
 
-    /// Treat self as a prefix and append the given nibble to it.
+    /// Treat self as a prefix and append the given bitspan to it.
     fn add_bit_span(self, len: u8, bs: BitSpan) -> (Self, u8);
 
+    /// fill the bits after the specified len with zeros. Interpreted as an IP
+    /// Prefix, this means that self will be truncated to the specified len.
     fn truncate_to_len(self, len: u8) -> Self;
 
+    /// Turn self in to a [std::net::IpAddr].
     fn into_ipaddr(self) -> std::net::IpAddr;
 
-    // temporary function, this will botch IPv6 completely.
+    /// Truncate self to a u32. For IPv4 this is a NOP. For IPv6 this
+    /// truncates to 32 bits.
     fn dangerously_truncate_to_u32(self) -> u32;
 
     // For the sake of searching for 0/0, check the the right shift, since
@@ -94,10 +103,6 @@ impl AddressFamily for IPv4 {
         IPv4::from(ip_addr.octets())
     }
 
-    fn fmt_net(net: Self) -> String {
-        std::net::Ipv4Addr::from(u32::from(net)).to_string()
-    }
-
     fn into_bit_span(net: Self, start_bit: u8, len: u8) -> BitSpan {
         BitSpan {
             bits: ((net << <U32<NetworkEndian>>::from(start_bit as u32))
@@ -107,50 +112,6 @@ impl AddressFamily for IPv4 {
         }
     }
 
-    // You can't shift with the number of bits of self, so we'll just return
-    // zero for that case.
-    //
-    // Panics if len is greater than 32 (the number of bits of self).
-    // fn truncate_to_len(self, len: u8) -> Self {
-    //     match len {
-    //         0 => U32::new(0),
-    //         1..=31 => (self >> (32 - len as u32).into()) << (32 - len).into(),
-    //         32 => self,
-    //         _ => panic!("Can't truncate to more than 32 bits"),
-    //     }
-    // }
-
-    /// Treat self as a prefix and append the given nibble to it.
-    ///
-    /// Shifts the rightmost `nibble_len` bits of `nibble` to the left to a
-    /// position `len` bits from the left, then ORs the result into self.
-    ///
-    /// For example:
-    ///
-    /// ```
-    /// # use rotonda_store::IPv4;
-    /// # use rotonda_store::AddressFamily;
-    /// let prefix = 0b10101010_00000000_00000000_00000000_u32; // 8-bit prefix
-    /// let nibble = 0b1100110_u32;                             // 7-bit nibble
-    /// let (new_prefix, new_len) = prefix.add_nibble(8, nibble, 7);
-    /// assert_eq!(new_len, 8 + 7);
-    /// assert_eq!(new_prefix, 0b10101010_11001100_00000000_00000000);
-    /// //                       ^^^^^^^^ ^^^^^^^
-    /// //                       prefix   nibble
-    /// ```
-    ///
-    /// # Panics in debug mode!
-    ///
-    /// Will panic if there is insufficient space to add the given nibble,
-    /// i.e. if `len + nibble_len >= 32`.
-    ///
-    /// ```
-    /// # use rotonda_store::IPv4;
-    /// # use rotonda_store::AddressFamily;
-    /// let prefix = 0b10101010_00000000_00000000_00000100_u32; // 30-bit prefix
-    /// let nibble = 0b1100110_u32;                             // 7-bit nibble
-    /// let (new_prefix, new_len) = prefix.add_nibble(30, nibble, 7);
-    /// ```
     fn add_bit_span(self, len: u8, bs: BitSpan) -> (U32<NetworkEndian>, u8) {
         let res = self | (bs.bits << (32 - len - bs.len) as usize);
         (res, len + bs.len)
@@ -211,10 +172,6 @@ impl AddressFamily for IPv6 {
         IPv6::from(ip_addr.octets())
     }
 
-    fn fmt_net(net: Self) -> String {
-        std::net::Ipv6Addr::from(u128::from(net)).to_string()
-    }
-
     fn into_bit_span(net: Self, start_bit: u8, len: u8) -> BitSpan {
         BitSpan {
             bits: u128::from(
@@ -225,39 +182,6 @@ impl AddressFamily for IPv6 {
         }
     }
 
-    /// Treat self as a prefix and append the given nibble to it.
-    ///
-    /// Shifts the rightmost `nibble_len` bits of `nibble` to the left to a
-    /// position `len` bits from the left, then ORs the result into self.
-    ///
-    /// For example:
-    ///
-    /// ```
-    /// # use rotonda_store::IPv6;
-    /// # use rotonda_store::AddressFamily;
-    /// let prefix = 0xF0F0F0F0_F0000000_00000000_00000000u128; // 36-bit prefix
-    /// let nibble = 0xA8A8_u32;                                // 16-bit nibble
-    /// let (new_prefix, new_len) = prefix.add_nibble(36, nibble, 16);
-    /// assert_eq!(new_len, 36 + 16);
-    /// assert_eq!(new_prefix, 0xF0F0F0F0F_A8A8000_00000000_00000000u128);
-    /// //                       ^^^^^^^^^ ^^^^
-    /// //                       prefix    nibble
-    /// ```
-    ///
-    /// # Panics only in debug mode!
-    ///
-    /// In release mode this will be UB (Undefined Behaviour)!
-    ///
-    /// Will panic if there is insufficient space to add the given nibble,
-    /// i.e. if `len + nibble_len >= 128`.
-    ///
-    /// ```
-    /// # use rotonda_store::IPv6;
-    /// # use rotonda_store::AddressFamily;
-    /// let prefix = 0xFFFFFFFF_FFFFFFFF_FFFFFFFF_FFFF0000u128; // 112-bit prefix
-    /// let nibble = 0xF00FF00F_u32;                            // 32-bit nibble
-    /// let (new_prefix, new_len) = prefix.add_nibble(112, nibble, 32);
-    /// ```
     fn add_bit_span(self, len: u8, bs: BitSpan) -> (Self, u8) {
         let res = self | ((bs.bits as u128) << (128 - len - bs.len) as usize);
         (res, len + bs.len)
diff --git a/src/types/bit_span.rs b/src/types/bit_span.rs
index 483d66b8..f0ee7829 100644
--- a/src/types/bit_span.rs
+++ b/src/types/bit_span.rs
@@ -1,5 +1,18 @@
 use crate::rib::BIT_SPAN_SIZE;
 
+// A bitspan is a bunch of bits representing the last stride in a NodeId
+// or PrefixId, as such it can have a length of 1, 2, or 3 bits, in a stride
+// length of 4 bits (which is the hard-coded value for all of the store
+// currently).
+//
+// We are storing these bits in a u32, which may seem to be wasting space
+// on first glance. However:
+// - this bitspan is never stored in the store as
+// such, it is used for intermediary calculations. The assumption is that
+// modern CPUs always throw around values aligned on 4 bytes.
+// - even if wanted to optimise for space, we have to take into account that
+// we need to shift right and left beyond the size of the final result of a
+// series of calculations.
 #[derive(Copy, Clone, Debug)]
 pub struct BitSpan {
     pub bits: u32,
diff --git a/src/types/prefix_id.rs b/src/types/prefix_id.rs
index 33bf7cfe..5f73e096 100644
--- a/src/types/prefix_id.rs
+++ b/src/types/prefix_id.rs
@@ -3,6 +3,10 @@ use zerocopy::FromBytes;
 use crate::AddressFamily;
 
 //------------ PrefixId ------------------------------------------------------
+// The type that acts both as an id for every prefix node in the prefix CHT,
+// and as the internal prefix type. It's cut to size for an AF, unlike the
+// inetnum Prefix. We use the latter on the public API.
+
 #[derive(
     Hash,
     Eq,
@@ -36,35 +40,15 @@ impl<AF: AddressFamily> PrefixId<AF> {
     }
 
     pub(crate) fn truncate_to_len(self, len: u8) -> Self {
-        // trace!("orig {:032b}", self.net);
-        // trace!(
-        //     "new  {:032b}",
-        //     self.net >> (AF::BITS - len).into() << (AF::BITS - len).into()
-        // );
-        // trace!(
-        //     "truncate to net {} len {}",
-        //     self.net >> (AF::BITS - len).into() << (AF::BITS - len).into(),
-        //     len
-        // );
         Self {
             net: self.net.truncate_to_len(len),
             len,
         }
     }
-
-    // The lsm tree, used for persistence, stores the prefix in the key with
-    // len first, so that key range lookups can be made for more-specifics in
-    // each prefix length.
-    // pub fn len_first_bytes<const PREFIX_SIZE: usize>(
-    //     &self,
-    // ) -> [u8; PREFIX_SIZE] {
-    //     let bytes = &mut [0_u8; PREFIX_SIZE];
-    //     *bytes.last_chunk_mut::<4>().unwrap() = self.net.to_be_bytes();
-    //     bytes[0] = self.len;
-    //     *bytes
-    // }
 }
 
+// There is no reasonable way for this to panic, PrefixId and inetnum's Prefix
+// represent the same data in slightly different ways.
 #[allow(clippy::unwrap_used)]
 impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
     fn from(value: inetnum::addr::Prefix) -> Self {

From 710efcee1075c332612841f7ab7b655bf70035e4 Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 15:56:52 +0100
Subject: [PATCH 133/147] harmonise NodeId - PrefixId

---
 src/prefix_cht/cht.rs                    | 54 ++++++++++-----------
 src/rib/starcast_af.rs                   |  2 +-
 src/tree_bitmap/mod.rs                   | 60 +++++++++++-------------
 src/tree_bitmap/node_cht.rs              |  7 ++-
 src/tree_bitmap/tree_bitmap_iterators.rs | 15 +++---
 src/tree_bitmap/tree_bitmap_node.rs      | 40 +++++++++-------
 src/types/bit_span.rs                    |  2 +
 src/types/prefix_id.rs                   | 26 ++++++----
 src/types/tests.rs                       | 11 ++---
 9 files changed, 110 insertions(+), 107 deletions(-)

diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index 11b93227..e875767a 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -10,6 +10,7 @@ use log::{debug, log_enabled, trace};
 use roaring::RoaringBitmap;
 
 use crate::cht::{nodeset_size, prev_node_size};
+use crate::errors::{FatalError, FatalResult};
 use crate::prefix_record::Meta;
 use crate::stats::UpsertReport;
 use crate::types::RouteStatus;
@@ -38,10 +39,8 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     #[allow(clippy::type_complexity)]
     fn acquire_write_lock(
         &self,
-    ) -> Result<
-        (MutexGuard<HashMap<u32, MultiMapValue<M>>>, usize),
-        PrefixStoreError,
-    > {
+    ) -> FatalResult<(MutexGuard<HashMap<u32, MultiMapValue<M>>>, usize)>
+    {
         let mut retry_count: usize = 0;
         let backoff = Backoff::new();
 
@@ -49,9 +48,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
             // We're using lock(), which returns an Error only if another
             // thread has panicked while holding the lock. In that situtation
             // we are certainly not going to write anything.
-            if let Ok(guard) =
-                self.0.lock().map_err(|_| PrefixStoreError::ExternalError)
-            {
+            if let Ok(guard) = self.0.lock().map_err(|_| FatalError) {
                 return Ok((guard, retry_count));
             }
 
@@ -272,8 +269,7 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     pub(crate) fn upsert_record(
         &self,
         new_rec: Record<M>,
-    ) -> Result<(Option<(MultiMapValue<M>, usize)>, usize), PrefixStoreError>
-    {
+    ) -> FatalResult<(Option<(MultiMapValue<M>, usize)>, usize)> {
         let (mut record_map, retry_count) = self.acquire_write_lock()?;
         let key = new_rec.multi_uniq_id;
 
@@ -402,7 +398,7 @@ impl<AF: AddressFamily, M: Meta> StoredPrefix<AF, M> {
         // we're in.
         // let pfx_id = PrefixId::new(record.net, record.len);
         // let this_level = bits_for_len(pfx_id.get_len(), level);
-        let next_level = nodeset_size(pfx_id.get_len(), level + 1);
+        let next_level = nodeset_size(pfx_id.len(), level + 1);
 
         trace!("next level {}", next_level);
         let next_bucket: PrefixSet<AF, M> = if next_level > 0 {
@@ -410,14 +406,14 @@ impl<AF: AddressFamily, M: Meta> StoredPrefix<AF, M> {
                 "{} store: INSERT with new bucket of size {} at prefix len {}",
                 std::thread::current().name().unwrap_or("unnamed-thread"),
                 1 << next_level,
-                pfx_id.get_len()
+                pfx_id.len()
             );
             PrefixSet::init_with_p2_children(next_level as usize)
         } else {
             debug!(
                 "{} store: INSERT at LAST LEVEL with empty bucket at prefix len {}",
                 std::thread::current().name().unwrap_or("unnamed-thread"),
-                pfx_id.get_len()
+                pfx_id.len()
             );
             PrefixSet::init_with_p2_children(next_level as usize)
         };
@@ -554,7 +550,7 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
         include_withdrawn: bool,
         bmin: &RoaringBitmap,
     ) -> Option<Vec<Record<M>>> {
-        let mut prefix_set = self.0.root_for_len(prefix.get_len());
+        let mut prefix_set = self.0.root_for_len(prefix.len());
         let mut level: u8 = 0;
         let backoff = Backoff::new();
 
@@ -620,8 +616,10 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
                         );
                     }
 
-                    let (mui_count, retry_count) =
-                        stored_prefix.record_map.upsert_record(record)?;
+                    let (mui_count, retry_count) = stored_prefix
+                        .record_map
+                        .upsert_record(record)
+                        .map_err(|_| PrefixStoreError::FatalError)?;
 
                     // See if someone beat us to creating the record.
                     if mui_count.is_some() {
@@ -640,8 +638,8 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
                         std::thread::current()
                             .name()
                             .unwrap_or("unnamed-thread"),
-                        prefix.get_net(),
-                        prefix.get_len()
+                        prefix.bits(),
+                        prefix.len()
                     );
                     }
                     prefix_is_new = false;
@@ -650,8 +648,10 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
                     // caller's record.
                     stored_prefix.set_ps_outdated(guard)?;
 
-                    let (mui_count, retry_count) =
-                        stored_prefix.record_map.upsert_record(record)?;
+                    let (mui_count, retry_count) = stored_prefix
+                        .record_map
+                        .upsert_record(record)
+                        .map_err(|_| PrefixStoreError::FatalError)?;
                     mui_is_new = mui_count.is_none();
 
                     if let Some(tbi) = update_path_selections {
@@ -688,7 +688,7 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
         search_prefix_id: PrefixId<AF>,
     ) -> (&StoredPrefix<AF, M>, bool) {
         trace!("non_recursive_retrieve_prefix_mut_with_guard");
-        let mut prefix_set = self.0.root_for_len(search_prefix_id.get_len());
+        let mut prefix_set = self.0.root_for_len(search_prefix_id.len());
         let mut level: u8 = 0;
 
         trace!("root prefix_set {:?}", prefix_set);
@@ -720,8 +720,8 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
                             .get_or_init(index, || {
                                 StoredPrefix::new(
                                     PrefixId::new(
-                                        search_prefix_id.get_net(),
-                                        search_prefix_id.get_len(),
+                                        search_prefix_id.bits(),
+                                        search_prefix_id.len(),
                                     ),
                                     level,
                                 )
@@ -773,7 +773,7 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
             usize,
         )>,
     ) {
-        let mut prefix_set = self.0.root_for_len(id.get_len());
+        let mut prefix_set = self.0.root_for_len(id.len());
         let mut parents = [None; 32];
         let mut level: u8 = 0;
         let backoff = Backoff::new();
@@ -822,7 +822,7 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
         // } else {
         //     0
         // };
-        let last_level = prev_node_size(id.get_len(), level);
+        let last_level = prev_node_size(id.len(), level);
         // trace!(
         //     "bits division {}; no of bits {}",
         //     this_level,
@@ -832,11 +832,11 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
         //     "calculated index ({} << {}) >> {}",
         //     id.get_net(),
         //     last_level,
-        //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
+        //   net>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
         // );
         // HASHING FUNCTION
-        let size = nodeset_size(id.get_len(), level);
-        ((id.get_net() << AF::from_u32(last_level as u32))
+        let size = nodeset_size(id.len(), level);
+        ((id.bits() << AF::from_u32(last_level as u32))
             >> AF::from_u8((<AF>::BITS - size) % <AF>::BITS))
         .dangerously_truncate_to_u32() as usize
     }
diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index 1aaa7af6..dd1b7126 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -122,7 +122,7 @@ impl<
                     }
                     report.cas_count += retry_count as usize;
                     if !exists {
-                        self.counters.inc_prefixes_count(prefix.get_len());
+                        self.counters.inc_prefixes_count(prefix.len());
                         report.prefix_new = true;
                     }
                     report
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index eabda825..b9f670f5 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -6,7 +6,7 @@ mod tree_bitmap_node;
 mod tree_bitmap_query;
 
 pub(crate) use tree_bitmap_node::{
-    NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter, StrideNodeId,
+    NodeId, NodeMoreSpecificChildIter, NodeMoreSpecificsPrefixIter,
     TreeBitMapNode,
 };
 use zerocopy::FromZeros;
@@ -239,7 +239,7 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
 
         let _retry_count = tree_bitmap
             .store_node(
-                StrideNodeId::dangerously_new_with_id_as_is(
+                NodeId::dangerously_new_with_id_as_is(
                     <AF as FromZeros>::new_zeroed(),
                     0,
                 ),
@@ -266,7 +266,7 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
         pfx: PrefixId<AF>,
         mui: u32,
     ) -> Result<(u32, bool), PrefixStoreError> {
-        if pfx.get_len() == 0 {
+        if pfx.len() == 0 {
             let prefix_new =
                 !self.default_route_exists.swap(true, Ordering::Acquire);
             return self
@@ -282,18 +282,17 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
 
         let retry_and_exists = loop {
             stride_end += STRIDE_SIZE;
-            let nibble_len = if pfx.get_len() < stride_end {
-                STRIDE_SIZE + pfx.get_len() - stride_end
+            let nibble_len = if pfx.len() < stride_end {
+                STRIDE_SIZE + pfx.len() - stride_end
             } else {
                 STRIDE_SIZE
             };
-
             let bit_span = AF::into_bit_span(
-                pfx.get_net(),
+                pfx.bits(),
                 stride_end - STRIDE_SIZE,
                 nibble_len,
             );
-            let is_last_stride = pfx.get_len() <= stride_end;
+            let is_last_stride = pfx.len() <= stride_end;
             let stride_start = stride_end - STRIDE_SIZE;
 
             // this counts the number of retry_count for this loop only,
@@ -310,8 +309,8 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
                         // All the bits of the search prefix, but with
                         // a length set to the start of the current
                         // stride.
-                        StrideNodeId::dangerously_new_with_id_as_is(
-                            pfx.get_net(),
+                        NodeId::dangerously_new_with_id_as_is(
+                            pfx.bits(),
                             stride_start,
                         ),
                         // the length of THIS stride
@@ -325,8 +324,8 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
 
                             // get a new identifier for the node we're
                             // going to create.
-                            let new_id = StrideNodeId::new_with_cleaned_id(
-                                pfx.get_net(),
+                            let new_id = NodeId::new_with_cleaned_id(
+                                pfx.bits(),
                                 stride_start + bit_span.len,
                             );
 
@@ -556,10 +555,10 @@ Giving up this node. This shouldn't happen!",
 
     fn store_node(
         &self,
-        id: StrideNodeId<AF>,
+        id: NodeId<AF>,
         multi_uniq_id: u32,
         new_node: TreeBitMapNode<AF>,
-    ) -> Result<(StrideNodeId<AF>, u32), FatalError> {
+    ) -> Result<(NodeId<AF>, u32), FatalError> {
         if log_enabled!(log::Level::Trace) {
             debug!(
                 "{} store: Store node {}: {:?} mui {}",
@@ -715,7 +714,7 @@ Giving up this node. This shouldn't happen!",
 
     pub fn retrieve_node_mut(
         &self,
-        id: StrideNodeId<AF>,
+        id: NodeId<AF>,
         mui: u32,
     ) -> Option<&TreeBitMapNode<AF>> {
         // HASHING FUNCTION
@@ -797,7 +796,7 @@ Giving up this node. This shouldn't happen!",
 
     pub fn retrieve_node(
         &self,
-        id: StrideNodeId<AF>,
+        id: NodeId<AF>,
     ) -> Option<&TreeBitMapNode<AF>> {
         // HASHING FUNCTION
         let mut level = 0;
@@ -839,7 +838,7 @@ Giving up this node. This shouldn't happen!",
 
     pub(crate) fn retrieve_node_for_mui(
         &self,
-        id: StrideNodeId<AF>,
+        id: NodeId<AF>,
         mui: u32,
     ) -> Option<&TreeBitMapNode<AF>> {
         // HASHING FUNCTION
@@ -900,8 +899,8 @@ Giving up this node. This shouldn't happen!",
         }
     }
 
-    pub(crate) fn get_root_node_id(&self) -> StrideNodeId<AF> {
-        StrideNodeId::dangerously_new_with_id_as_is(
+    pub(crate) fn get_root_node_id(&self) -> NodeId<AF> {
+        NodeId::dangerously_new_with_id_as_is(
             <AF as FromZeros>::new_zeroed(),
             0,
         )
@@ -929,7 +928,6 @@ Giving up this node. This shouldn't happen!",
     }
 
     // Stride related methods
-
     // pub fn get_stride_sizes(&self) -> &[u8] {
     //     self.node_buckets.get_stride_sizes()
     // }
@@ -939,36 +937,32 @@ Giving up this node. This shouldn't happen!",
     pub(crate) fn get_node_id_for_prefix(
         &self,
         prefix: &PrefixId<AF>,
-    ) -> (StrideNodeId<AF>, BitSpan) {
+    ) -> (NodeId<AF>, BitSpan) {
         trace!(
             "prefix id bits: {:032b} len: {}",
-            prefix.get_net(),
-            prefix.get_len()
+            prefix.bits(),
+            prefix.len()
         );
         let mut acc = 0;
         // for i in self.get_stride_sizes() {
         loop {
             acc += STRIDE_SIZE;
-            if acc >= prefix.get_len() {
+            if acc >= prefix.len() {
                 let node_len = acc - STRIDE_SIZE;
                 return (
-                    StrideNodeId::new_with_cleaned_id(
-                        prefix.get_net(),
-                        node_len,
-                    ),
+                    NodeId::new_with_cleaned_id(prefix.bits(), node_len),
                     // NOT THE HASHING FUNCTION!
                     // Do the right shift in a checked manner, for the sake
                     // of 0/0. A search for 0/0 will perform a 0 << MAX_LEN,
                     // which will panic in debug mode (undefined behaviour
                     // in prod).
                     BitSpan::new(
-                        ((prefix.get_net() << AF::from_u8(node_len))
+                        ((prefix.bits() << AF::from_u8(node_len))
                             .checked_shr_or_zero(
-                                (AF::BITS - (prefix.get_len() - node_len))
-                                    .into(),
+                                (AF::BITS - (prefix.len() - node_len)).into(),
                             ))
                         .dangerously_truncate_to_u32(),
-                        prefix.get_len() - node_len,
+                        prefix.len() - node_len,
                     ),
                 );
             }
@@ -1025,7 +1019,7 @@ Giving up this node. This shouldn't happen!",
     // element, where each element in the list has an array of its own that
     // uses the hash function with the level incremented.
 
-    pub(crate) fn hash_node_id(id: StrideNodeId<AF>, level: u8) -> usize {
+    pub(crate) fn hash_node_id(id: NodeId<AF>, level: u8) -> usize {
         // And, this is all of our hashing function.
         // let last_level = if level > 0 {
         //     bits_for_len(id.len(), level - 1)
diff --git a/src/tree_bitmap/node_cht.rs b/src/tree_bitmap/node_cht.rs
index 8b9f3280..f66add41 100644
--- a/src/tree_bitmap/node_cht.rs
+++ b/src/tree_bitmap/node_cht.rs
@@ -4,12 +4,11 @@ use log::{debug, log_enabled};
 
 use roaring::RoaringBitmap;
 
+use super::tree_bitmap_node::{NodeId, TreeBitMapNode};
 use crate::cht::{Cht, OnceBoxSlice, Value};
 use crate::errors::{FatalError, FatalResult};
-use crate::types::AddressFamily;
-
-use super::tree_bitmap_node::{StrideNodeId, TreeBitMapNode};
 use crate::types::errors::PrefixStoreError;
+use crate::types::AddressFamily;
 
 pub(crate) type NodeCht<AF, const ROOT_SIZE: usize> =
     Cht<NodeSet<AF>, ROOT_SIZE, 4>;
@@ -20,7 +19,7 @@ where
     Self: Sized,
     AF: AddressFamily,
 {
-    pub(crate) node_id: StrideNodeId<AF>,
+    pub(crate) node_id: NodeId<AF>,
     // The ptrbitarr and pfxbitarr for this node
     pub(crate) node: TreeBitMapNode<AF>,
     // Child nodes linked from this node
diff --git a/src/tree_bitmap/tree_bitmap_iterators.rs b/src/tree_bitmap/tree_bitmap_iterators.rs
index 7c204539..b70fa529 100644
--- a/src/tree_bitmap/tree_bitmap_iterators.rs
+++ b/src/tree_bitmap/tree_bitmap_iterators.rs
@@ -187,7 +187,7 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> Iterator
         trace!("search lm prefix for {:?}", self.prefix);
 
         loop {
-            if self.prefix.get_len() == 0 {
+            if self.prefix.len() == 0 {
                 return None;
             }
 
@@ -195,8 +195,7 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> Iterator
                 return Some(self.prefix);
             }
 
-            self.prefix =
-                self.prefix.truncate_to_len(self.prefix.get_len() - 1);
+            self.prefix = self.prefix.truncate_to_len(self.prefix.len() - 1);
         }
     }
 }
@@ -263,7 +262,7 @@ impl<'a, AF: AddressFamily, const ROOT_SIZE: usize>
         trace!("more specifics for {:?}", start_prefix_id);
 
         // A v4 /32 or a v6 /128 doesn't have more specific prefixes 🤓.
-        if start_prefix_id.get_len() >= AF::BITS {
+        if start_prefix_id.len() >= AF::BITS {
             None
         } else {
             // calculate the node start_prefix_id lives in.
@@ -272,8 +271,8 @@ impl<'a, AF: AddressFamily, const ROOT_SIZE: usize>
             trace!("start node {}", start_node_id);
             trace!(
                 "start prefix id {:032b} (len {})",
-                start_prefix_id.get_net(),
-                start_prefix_id.get_len()
+                start_prefix_id.bits(),
+                start_prefix_id.len()
             );
             trace!(
                 "start node id   {:032b} (bits {} len {})",
@@ -331,13 +330,13 @@ impl<'a, AF: AddressFamily, const ROOT_SIZE: usize>
     ) -> impl Iterator<Item = PrefixId<AF>> + 'a {
         if log_enabled!(log::Level::Trace) {
             trace!("less specifics for {}", Prefix::from(start_prefix_id));
-            trace!("level {}, len {}", 0, start_prefix_id.get_len());
+            trace!("level {}, len {}", 0, start_prefix_id.len());
         }
 
         LessSpecificPrefixIter {
             tree: self,
             prefix: start_prefix_id,
-            cur_level: start_prefix_id.get_len(),
+            cur_level: start_prefix_id.len(),
         }
     }
 
diff --git a/src/tree_bitmap/tree_bitmap_node.rs b/src/tree_bitmap/tree_bitmap_node.rs
index 24a7edab..46856422 100644
--- a/src/tree_bitmap/tree_bitmap_node.rs
+++ b/src/tree_bitmap/tree_bitmap_node.rs
@@ -81,7 +81,7 @@ where
     // Iterate over the more specific prefixes ids contained in this node
     pub(crate) fn more_specific_pfx_iter(
         &self,
-        base_prefix: StrideNodeId<AF>,
+        base_prefix: NodeId<AF>,
         start_bs: BitSpan,
     ) -> NodeMoreSpecificsPrefixIter<AF> {
         debug_assert!(start_bs.check());
@@ -95,7 +95,7 @@ where
     // base_prefix and corresponding bit_span.
     pub(crate) fn more_specific_ptr_iter(
         &self,
-        base_prefix: StrideNodeId<AF>,
+        base_prefix: NodeId<AF>,
         start_bs: BitSpan,
     ) -> NodeMoreSpecificChildIter<AF> {
         debug_assert!(start_bs.check());
@@ -128,7 +128,7 @@ where
         bit_span: BitSpan,
         // all the bits of the search prefix, but with the length set to
         // the length of this stride. So bits are set beyond its length.
-        base_prefix: StrideNodeId<AF>,
+        base_prefix: NodeId<AF>,
         // stride_len: u8,
         is_last_stride: bool,
     ) -> (NewNodeOrIndex<AF>, u32) {
@@ -332,7 +332,7 @@ type PtrBitArr = u16;
 
 #[derive(Debug, Copy, Clone)]
 pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily> {
-    base_prefix: StrideNodeId<AF>,
+    base_prefix: NodeId<AF>,
     bitrange: PtrBitArr,
     start_bs: BitSpan,
     start_cursor: u8,
@@ -341,7 +341,7 @@ pub(crate) struct NodeMoreSpecificChildIter<AF: AddressFamily> {
 impl<AF: AddressFamily> std::iter::Iterator
     for NodeMoreSpecificChildIter<AF>
 {
-    type Item = StrideNodeId<AF>;
+    type Item = NodeId<AF>;
     fn next(&mut self) -> Option<Self::Item> {
         if self.bitrange == 0 {
             trace!("empty ptrbitarr. This iterator is done.");
@@ -563,7 +563,7 @@ pub(crate) fn ptr_range(ptrbitarr: u16, bs: BitSpan) -> (u16, u8) {
 // to go over a different amount of 1 << (5 - 4) = 2 iterations to reap the
 // next bit_spans of 0010 0 and 0010 1.
 pub(crate) struct NodeMoreSpecificsPrefixIter<AF: AddressFamily> {
-    base_prefix: StrideNodeId<AF>,
+    base_prefix: NodeId<AF>,
     pfxbitarr: u32,
 }
 
@@ -614,20 +614,26 @@ where
 
 pub(crate) enum NewNodeOrIndex<AF: AddressFamily> {
     NewNode(TreeBitMapNode<AF>),
-    ExistingNode(StrideNodeId<AF>),
+    ExistingNode(NodeId<AF>),
     NewPrefix,
     ExistingPrefix,
 }
 
-//--------------------- Per-Stride-Node-Id Type -----------------------------
+//--------------------- NodeId -----------------------------------------------
+
+// The type that acts as the id for a node in the treebitmap and the node CHT.
+// Its data structure is the same as [PrefixId], but its behaviour is subtly
+// different from PrefixId, i.e. a NodeId only exists at a stride boundary,
+// so it always stores multiples of 4 bits. It cannot be converted to/from
+// a Prefix.
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub struct StrideNodeId<AF: AddressFamily> {
+pub struct NodeId<AF: AddressFamily> {
     bits: AF,
     len: u8,
 }
 
-impl<AF: AddressFamily> StrideNodeId<AF> {
+impl<AF: AddressFamily> NodeId<AF> {
     pub(crate) fn dangerously_new_with_id_as_is(
         addr_bits: AF,
         len: u8,
@@ -666,7 +672,7 @@ impl<AF: AddressFamily> StrideNodeId<AF> {
 
     #[inline]
     pub(crate) fn truncate_to_len(self) -> Self {
-        StrideNodeId::new_with_cleaned_id(self.bits, self.len)
+        NodeId::new_with_cleaned_id(self.bits, self.len)
     }
 
     // clean out all bits that are set beyond the len. This function should
@@ -683,23 +689,21 @@ impl<AF: AddressFamily> StrideNodeId<AF> {
     }
 }
 
-impl<AF: AddressFamily> std::fmt::Display for StrideNodeId<AF> {
+impl<AF: AddressFamily> std::fmt::Display for NodeId<AF> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}-{}", self.bits, self.len)
     }
 }
 
-impl<AF: AddressFamily> std::convert::From<StrideNodeId<AF>>
-    for PrefixId<AF>
-{
-    fn from(id: StrideNodeId<AF>) -> Self {
+impl<AF: AddressFamily> std::convert::From<NodeId<AF>> for PrefixId<AF> {
+    fn from(id: NodeId<AF>) -> Self {
         PrefixId::new(id.bits, id.len)
     }
 }
 
-impl<AF: AddressFamily> From<(AF, u8)> for StrideNodeId<AF> {
+impl<AF: AddressFamily> From<(AF, u8)> for NodeId<AF> {
     fn from(value: (AF, u8)) -> Self {
-        StrideNodeId {
+        NodeId {
             bits: value.0,
             len: value.1,
         }
diff --git a/src/types/bit_span.rs b/src/types/bit_span.rs
index f0ee7829..e42fbd70 100644
--- a/src/types/bit_span.rs
+++ b/src/types/bit_span.rs
@@ -1,5 +1,7 @@
 use crate::rib::BIT_SPAN_SIZE;
 
+//------------ BitSpan -------------------------------------------------------
+
 // A bitspan is a bunch of bits representing the last stride in a NodeId
 // or PrefixId, as such it can have a length of 1, 2, or 3 bits, in a stride
 // length of 4 bits (which is the hard-coded value for all of the store
diff --git a/src/types/prefix_id.rs b/src/types/prefix_id.rs
index 5f73e096..819e2d0d 100644
--- a/src/types/prefix_id.rs
+++ b/src/types/prefix_id.rs
@@ -3,9 +3,11 @@ use zerocopy::FromBytes;
 use crate::AddressFamily;
 
 //------------ PrefixId ------------------------------------------------------
+
 // The type that acts both as an id for every prefix node in the prefix CHT,
 // and as the internal prefix type. It's cut to size for an AF, unlike the
-// inetnum Prefix. We use the latter on the public API.
+// inetnum Prefix, as not to waste memory. We use the latter on the public
+// API.
 
 #[derive(
     Hash,
@@ -22,26 +24,30 @@ use crate::AddressFamily;
 )]
 #[repr(C)]
 pub struct PrefixId<AF: AddressFamily> {
+    // DO NOT CHANGE THE ORDER OF THESE FIELDS!
+    // zerocopy uses this to concatenate the bytes in this order, and the
+    // lsm_tree needs to have `len` first, and `net` second to create keys
+    // that are correctly sorted on prefix length.
     len: u8,
-    net: AF,
+    bits: AF,
 }
 
 impl<AF: AddressFamily> PrefixId<AF> {
     pub(crate) fn new(net: AF, len: u8) -> Self {
-        PrefixId { len, net }
+        PrefixId { len, bits: net }
     }
 
-    pub(crate) fn get_net(&self) -> AF {
-        self.net
+    pub(crate) fn bits(&self) -> AF {
+        self.bits
     }
 
-    pub(crate) fn get_len(&self) -> u8 {
+    pub(crate) fn len(&self) -> u8 {
         self.len
     }
 
     pub(crate) fn truncate_to_len(self, len: u8) -> Self {
         Self {
-            net: self.net.truncate_to_len(len),
+            bits: self.bits.truncate_to_len(len),
             len,
         }
     }
@@ -53,7 +59,7 @@ impl<AF: AddressFamily> PrefixId<AF> {
 impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
     fn from(value: inetnum::addr::Prefix) -> Self {
         Self {
-            net: match value.addr() {
+            bits: match value.addr() {
                 std::net::IpAddr::V4(addr) => {
                     *AF::try_ref_from_bytes(&addr.octets()).unwrap()
                 }
@@ -71,7 +77,7 @@ impl<AF: AddressFamily> From<inetnum::addr::Prefix> for PrefixId<AF> {
 #[allow(clippy::unwrap_used)]
 impl<AF: AddressFamily> From<PrefixId<AF>> for inetnum::addr::Prefix {
     fn from(value: PrefixId<AF>) -> Self {
-        Self::new(value.get_net().into_ipaddr(), value.get_len()).unwrap()
+        Self::new(value.bits().into_ipaddr(), value.len()).unwrap()
     }
 }
 
@@ -82,7 +88,7 @@ impl<AF: AddressFamily, const PREFIX_SIZE: usize> From<[u8; PREFIX_SIZE]>
     fn from(value: [u8; PREFIX_SIZE]) -> Self {
         Self {
             // This cannot panic for values of PREFIX_SIZE greater than 1
-            net: *AF::ref_from_bytes(&value.as_slice()[1..]).unwrap(),
+            bits: *AF::ref_from_bytes(&value.as_slice()[1..]).unwrap(),
             len: value[0],
         }
     }
diff --git a/src/types/tests.rs b/src/types/tests.rs
index a070b4f7..20230d68 100644
--- a/src/types/tests.rs
+++ b/src/types/tests.rs
@@ -5,19 +5,18 @@ use std::error::Error;
 
 #[test]
 fn test_af_1() -> Result<(), Box<dyn Error>> {
-    use crate::tree_bitmap::StrideNodeId;
+    use crate::tree_bitmap::NodeId;
     use crate::types::BitSpan;
     use crate::AddressFamily;
     use crate::IPv4;
 
     let bit_addr: IPv4 = 0b1111_1111_1111_1111_1111_1111_1111_1111.into();
-    let base_prefix =
-        StrideNodeId::dangerously_new_with_id_as_is(bit_addr, 32);
+    let base_prefix = NodeId::dangerously_new_with_id_as_is(bit_addr, 32);
 
     assert_eq!(base_prefix.bits(), bit_addr);
     assert_eq!(base_prefix.truncate_to_len().bits(), base_prefix.bits());
     assert_eq!(
-        StrideNodeId::dangerously_new_with_id_as_is(
+        NodeId::dangerously_new_with_id_as_is(
             base_prefix.bits().truncate_to_len(28),
             28
         )
@@ -35,10 +34,10 @@ fn test_af_1() -> Result<(), Box<dyn Error>> {
 #[test]
 fn test_af_2() -> Result<(), Box<dyn Error>> {
     use crate::IPv4;
-    use crate::{tree_bitmap::StrideNodeId, types::BitSpan};
+    use crate::{tree_bitmap::NodeId, types::BitSpan};
 
     let bit_addr: IPv4 = 0b1111_1111_1111_1111_1111_1111_1111_1111.into();
-    let nu_prefix = StrideNodeId::dangerously_new_with_id_as_is(bit_addr, 8);
+    let nu_prefix = NodeId::dangerously_new_with_id_as_is(bit_addr, 8);
 
     assert_eq!(nu_prefix.bits(), bit_addr);
     assert_eq!(

From 305d3cf37929588a2426d17cf0d6ee557198e104 Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 18:51:57 +0100
Subject: [PATCH 134/147] update comments

---
 src/lib.rs                          | 52 ++++++++++++++++++++++++
 src/prefix_cht/cht.rs               | 55 ++++++--------------------
 src/rib/starcast_af.rs              | 19 +++++++--
 src/tree_bitmap/mod.rs              | 61 ++++++++++++++++++++++-------
 src/tree_bitmap/node_cht.rs         |  2 +
 src/tree_bitmap/tree_bitmap_node.rs |  2 +-
 6 files changed, 128 insertions(+), 63 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 32e79aa7..be442288 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -25,10 +25,62 @@
 //!
 //! [^1]: <https://www.cs.cornell.edu/courses/cs419/2005sp/tree-bitmap.pdf>
 //! [^2]: <https://blog.nlnetlabs.nl/donkeys-mules-horses/>
+
+//                                ┌───────────────────┐
+//                                │    StarCastRib    │
+//                                └────────┬┬─────────┘
+//                        ┌────────────────┘└─────────────────┐
+//                     ┌──▼──┐                             ┌──▼──┐
+//                     │ v4  │                             │ v4  │
+//                     └──┬──┘                             └──┬──┘
+//            ┌───────────┼──────────┐            ┌───────────┼──────────┐
+//      ┌─────▼────┐┌─────▼────┐┌───▼─────┐┌─────▼────┐┌─────▼────┐┌────▼───┐
+//      │treebitmap││prefix_cht││lsm_tree ││treebitmap││prefix_cht││lsm_tree│
+//      └─────┬────┘└──────────┘└─────────┘└─────┬────┘└──────────┘└────────┘
+//     ┌──────┴─────┐                      ┌──────┴─────┐
+// ┌───▼────┐┌──────▼────┐             ┌───▼────┐┌──────▼────┐
+// │node_cht││muis_bitmap│             │node_cht││muis_bitmap│
+// └────────┘└───────────┘             └────────┘└───────────┘
+
+// Rotonda-store is a fairly layered repo, it uses three different
+// types of trees, that are all hidden behind one public interface.
+
+// `rib::starcast::StarCastRib`, holds that public API. This is the RIB
+// that stores (route-like) data for IPv4/IPv6 unicast and multicast (hence
+// *cast). This is a good starting point to dive into this repo.
+
+// `rib::starcast_af::StarCastAfRib` holds the three trees for a store, per
+// Address Family. From there `tree_bitmap` (the mod.rs file), holds the tree
+// bitmap, `tree_bit_map::node_cht` holds the CHT that stores the nodes for
+// the tree bitmap. Next to the tree, it also holds a bitmap that indexes all
+// muis that are withdrawn for the whole tree. The tree bitmap is used for
+// all strategies.
+
+// `prefix_cht::cht` holds the CHT that stores all the route-like data for the
+// in-memory strategies. This CHT is the same data-structure that is used for
+// the nodes, but it stores `MultiMap` typed values in its nodes (described in
+// the same file).
+
+// `lsm_tree` (again, in the mod.rs file) holds the log-structured merge tree
+// used for persistent storage on disk.
+
+//------------ Modules -------------------------------------------------------
+
+// the Chained Hash Table that is used by the treebitmap, and the CHT for the
+// prefixes (for in-menory storage of prefixes).
 mod cht;
+
+// The log-structured merge tree, used as persistent storage (on disk).
 mod lsm_tree;
+
+// The Chained Hash Table that stores the records for the prefixers in memory
 mod prefix_cht;
+
+// The Treebitmap, that stores the existence of all prefixes, and that is used
+//for all strategies.
 mod tree_bitmap;
+
+// Types, both public and private, that are used throughout the store.
 mod types;
 
 #[macro_use]
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index e875767a..55982ea3 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -22,9 +22,11 @@ use crate::{
     },
 };
 
-// ----------- MultiMap ------------------------------------------------------
-// This is the record that holds the aggregates at the top-level for a given
-// prefix.
+//------------ MultiMap ------------------------------------------------------
+//
+// This is the collection of records or a given prefix, keyed on the multi
+// unique identifier ("mui"). Note that the record contains more than just
+// the // meta-data typed value ("M").
 
 #[derive(Debug)]
 pub struct MultiMap<M: Meta>(
@@ -72,8 +74,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     }
 
     pub fn _len(&self) -> usize {
-        // let c_map = Arc::clone(&self.0);
-        // let record_map = c_map.lock().unwrap();
         let record_map = self.acquire_read_guard();
         record_map.len()
     }
@@ -83,8 +83,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         mui: u32,
         include_withdrawn: bool,
     ) -> Option<Record<M>> {
-        // let c_map = Arc::clone(&self.0);
-        // let record_map = c_map.lock().unwrap();
         let record_map = self.acquire_read_guard();
 
         record_map.get(&mui).and_then(|r| -> Option<Record<M>> {
@@ -97,8 +95,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     }
 
     pub fn best_backup(&self, tbi: M::TBI) -> (Option<u32>, Option<u32>) {
-        // let c_map = Arc::clone(&self.0);
-        // let record_map = c_map.lock().unwrap();
         let record_map = self.acquire_read_guard();
         let ord_routes = record_map
             .iter()
@@ -114,9 +110,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         bmin: &RoaringBitmap,
         rewrite_status: RouteStatus,
     ) -> Option<Record<M>> {
-        // let c_map = Arc::clone(&self.0);
-        // let record_map = c_map.lock().unwrap();
-
         let record_map = self.acquire_read_guard();
         record_map.get(&mui).map(|r| {
             // We'll return a cloned record: the record in the store remains
@@ -190,8 +183,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         bmin: &RoaringBitmap,
         rewrite_status: RouteStatus,
     ) -> Vec<Record<M>> {
-        // let c_map = Arc::clone(&self.0);
-        // let record_map = c_map.lock().unwrap();
         let record_map = self.acquire_read_guard();
         record_map
             .iter()
@@ -206,8 +197,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
     }
 
     pub fn _as_records(&self) -> Vec<Record<M>> {
-        // let c_map = Arc::clone(&self.0);
-        // let record_map = c_map.lock().unwrap();
         let record_map = self.acquire_read_guard();
         record_map
             .iter()
@@ -222,8 +211,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
         &self,
         bmin: &RoaringBitmap,
     ) -> Vec<Record<M>> {
-        // let c_map = Arc::clone(&self.0);
-        // let record_map = c_map.lock().unwrap();
         let record_map = self.acquire_read_guard();
         record_map
             .iter()
@@ -241,8 +228,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
 
     // Change the local status of the record for this mui to Withdrawn.
     pub fn mark_as_withdrawn_for_mui(&self, mui: u32, ltime: u64) {
-        // let c_map = Arc::clone(&self.0);
-        // let mut record_map = c_map.lock().unwrap();
         let mut record_map = self.acquire_read_guard();
         if let Some(rec) = record_map.get_mut(&mui) {
             rec.set_route_status(RouteStatus::Withdrawn);
@@ -252,8 +237,6 @@ impl<M: Send + Sync + Debug + Display + Meta> MultiMap<M> {
 
     // Change the local status of the record for this mui to Active.
     pub fn mark_as_active_for_mui(&self, mui: u32, ltime: u64) {
-        // let record_map = Arc::clone(&self.0);
-        // let mut r_map = record_map.lock().unwrap();
         let mut record_map = self.acquire_read_guard();
         if let Some(rec) = record_map.get_mut(&mui) {
             rec.set_route_status(RouteStatus::Active);
@@ -380,8 +363,6 @@ impl PathSelections {
 // don't have to go into them if there's nothing there and could stop early.
 #[derive(Debug)]
 pub struct StoredPrefix<AF: AddressFamily, M: Meta> {
-    // the serial number
-    // pub serial: usize,
     // the prefix itself,
     pub prefix: PrefixId<AF>,
     // the aggregated data for this prefix
@@ -422,7 +403,6 @@ impl<AF: AddressFamily, M: Meta> StoredPrefix<AF, M> {
         let rec_map = HashMap::new();
 
         StoredPrefix {
-            // serial: 1,
             prefix: pfx_id,
             path_selections: Atomic::init(PathSelections {
                 path_selection_muis: (None, None),
@@ -529,6 +509,11 @@ impl<AF: AddressFamily, M: Meta> Value for PrefixSet<AF, M> {
     }
 }
 
+//------------ PrefixCht -----------------------------------------------------
+
+// PrefixCht is a simple wrapper around Cht. It stores the meta-data for
+// in-memeory strategies.
+
 #[derive(Debug)]
 pub(crate) struct PrefixCht<
     AF: AddressFamily,
@@ -782,8 +767,6 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
             // The index of the prefix in this array (at this len and
             // level) is calculated by performing the hash function
             // over the prefix.
-
-            // HASHING FUNCTION
             let index = Self::hash_prefix_id(id, level);
 
             if let Some(stored_prefix) = prefix_set.0.get(index) {
@@ -816,24 +799,8 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
     }
 
     pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
-        // And, this is all of our hashing function.
-        // let last_level = if level > 0 {
-        //     bits_for_len(id.get_len(), level.saturating_sub(1))
-        // } else {
-        //     0
-        // };
         let last_level = prev_node_size(id.len(), level);
-        // trace!(
-        //     "bits division {}; no of bits {}",
-        //     this_level,
-        //     this_level - last_level
-        // );
-        // trace!(
-        //     "calculated index ({} << {}) >> {}",
-        //     id.get_net(),
-        //     last_level,
-        //   net>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-        // );
+
         // HASHING FUNCTION
         let size = nodeset_size(id.len(), level);
         ((id.bits() << AF::from_u32(last_level as u32))
diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index dd1b7126..7a45d24e 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -22,17 +22,30 @@ use crate::AddressFamily;
 
 use super::config::Config;
 
-// ----------- Rib -----------------------------------------------------------
-//
+//------------ StarCastAfRib -------------------------------------------------
+
 // A Routing Information Base that consists of multiple different trees for
-// in-memory and on-disk (persisted storage).
+// in-memory and on-disk (persisted storage) for one address family. Most of
+// the methods on this struct are meant to be publicly available, however they
+// are all behind the StarCastRib interface, that abstracts over the address
+// family.
 #[derive(Debug)]
 pub(crate) struct StarCastAfRib<
     AF: AddressFamily,
+    // The type that stores the route-like data
     M: Meta,
+    // The number of root nodes for the tree bitmap (one for each 4 prefix
+    // lengths, so that's 9 for IPv4, 33 for IPv6)
     const N_ROOT_SIZE: usize,
+    // The number of root nodes for the prefix CHT (one for each prefix length
+    // that can exists, so that's 33 for IPv4, and 129 for IPv6).
     const P_ROOT_SIZE: usize,
+    // The configuration, each persistence strategy implements its own type.
     C: Config,
+    // The size of the key in the persistence store, this varies per address
+    // family. This is 18 for IPv4 (1 octet prefix length, 4 octets address
+    // part prefix, 4 octets mui, 8 octets ltime, 1 octet RouteStatus). This
+    // corresponds to the `LongKey` struct. It's 30 for IPv6.
     const KEY_SIZE: usize,
 > {
     pub config: C,
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index b9f670f5..5cad25f0 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -11,9 +11,9 @@ pub(crate) use tree_bitmap_node::{
 };
 use zerocopy::FromZeros;
 
-// ----------- THE STORE ----------------------------------------------------
+// ----------- Dev Log for the RIB -------------------------------------------
 //
-// The CustomAllocStore provides in-memory storage for the BitTreeMapNodes
+// The StarCastAfRib provides in-memory storage for the TreeBitMapNodes
 // and for prefixes and their meta-data. The storage for node is on the
 // `buckets` field, and the prefixes are stored in, well, the `prefixes`
 // field. They are both organised in the same way, as chained hash tables,
@@ -47,18 +47,14 @@ use zerocopy::FromZeros;
 // node first and then go up the tree to the requested node. The lower nodes
 // of the tree (close to the root) would be a formidable bottle-neck then.
 //
-// Currently, the meta-data is an atomically stored value, that is required to
-// implement the `Meta` and the `Clone` trait. New meta-data
-// instances are stored atomically without further ado, but updates to a
-// piece of meta-data are done by merging the previous meta-data with the new
-// meta-data, through use of the `MergeUpdate` trait.
+// Previously, the meta-data was an atomically stored value, that was required
+// to implement the `Meta` and the `Clone` trait. New meta-data instances were
+// stored atomically without further ado, but updates to a piece of meta-data
+// were done by merging the previous meta-data with the new meta-data, through
+// use of the `MergeUpdate` trait.
 //
-// The `upsert_prefix` methods retrieve only the most recent insert
-// for a prefix (for now).
-//
-// Future work could have a user-configurable retention strategy that allows
-// the meta-data to be stored as a linked-list of references, where each
-// meta-data object has a reference to its predecessor.
+// The `upsert_prefix` methods were used to retrieve only the most recent
+// insert for a prefix (for now).
 //
 // Prefix example
 //
@@ -196,6 +192,30 @@ use zerocopy::FromZeros;
 // an actual memory leak in the mt-prefix-store (and even more if they can
 // produce a fix for it).
 
+// >= 0.4
+
+// The above scheme is outdated! After done a few day of bench marking, it was
+// found that storing the meta-data in `RwLock<HashMap>` structures actually
+// performs better in both time, and space. Also the overall performance
+// is  way more predictable and somewhat linearly related to the busyness of
+// the whole system. Furthermore it was found that using RwLock around the
+// HashMaps, instead of mutexes (from std) was around 2% slower at insert
+// time, while we believe (we haven't tested this), that read performance will
+// be superior to mutex. In terms of usability `RwLock<HasMap>` do not require
+// the user to implement the RCU-style `MergeUpdate` trait (it is removed
+// now).
+
+// Adding the possibilty of storing more than one piece of meta-data for a
+// prefix (through the use the MUI), made the RCU style storing very awkward:
+// all the previous pieces of meta-data (let's call them records), collected
+// in a HashMap, needed to copied out of the store, modified, and copied back
+// in, while being able to fail, and retried. Locking these HashMaps is way
+// more efficient, both in time (copying costs time), and memory (copying,
+// costs, well, memory). So what we have now, is a hybrid tree, where the
+// "core" consists of RCU-style, lock-free nodes (that can't be deleted!), and
+// locked structures at the "edges" (not leaves, because all nodes can carry
+// meta-data).
+
 use crate::cht::{nodeset_size, prev_node_size, Cht, Value};
 use crate::errors::{FatalError, FatalResult};
 use crate::rib::STRIDE_SIZE;
@@ -220,11 +240,19 @@ use ansi_term::Colour;
 
 //--------------------- TreeBitMap ------------------------------------------
 
+// The tree that holds the existence information for all prefixes for all
+//strategies. This tree is also used to find all less- and more-specifics and
+//iterate over them. It also holds a bitmap that contains RIB-wide withdrawn
+//muis (peers in most cases).
 #[derive(Debug)]
-pub struct TreeBitMap<AF: AddressFamily, const ROOT_SIZE: usize> {
+pub(crate) struct TreeBitMap<AF: AddressFamily, const ROOT_SIZE: usize> {
+    // the chained hash table that backs the treebitmap
     node_cht: NodeCht<AF, ROOT_SIZE>,
+    // the bitmap that holds RIB-wide withdrawn muis (e.g. peers)
     withdrawn_muis_bmin: Atomic<RoaringBitmap>,
+    // number of prefixes in the store, etc.
     counters: Counters,
+    // see the rant on update_default_route_prefix_meta
     default_route_exists: AtomicBool,
 }
 
@@ -553,12 +581,14 @@ Giving up this node. This shouldn't happen!",
         }
     }
 
+    // Store a new node in the tree, or merge the existing node with this
+    // node. This might fail disastrously, e.g. in case of failed I/O.
     fn store_node(
         &self,
         id: NodeId<AF>,
         multi_uniq_id: u32,
         new_node: TreeBitMapNode<AF>,
-    ) -> Result<(NodeId<AF>, u32), FatalError> {
+    ) -> FatalResult<(NodeId<AF>, u32)> {
         if log_enabled!(log::Level::Trace) {
             debug!(
                 "{} store: Store node {}: {:?} mui {}",
@@ -625,6 +655,7 @@ Giving up this node. This shouldn't happen!",
                             .node_set
                             .update_rbm_index(multi_uniq_id)?;
 
+                        // merge_with herre contains the critical section!
                         if !its_us && ptrbitarr != 0 {
                             retry_count += 1;
                             stored_node.node.ptrbitarr.merge_with(ptrbitarr);
diff --git a/src/tree_bitmap/node_cht.rs b/src/tree_bitmap/node_cht.rs
index f66add41..ca4507f9 100644
--- a/src/tree_bitmap/node_cht.rs
+++ b/src/tree_bitmap/node_cht.rs
@@ -19,6 +19,8 @@ where
     Self: Sized,
     AF: AddressFamily,
 {
+    // the id of this node. since we're using linked lists to store nodes in
+    // first-come-first-served order, we need to store the actual node id.
     pub(crate) node_id: NodeId<AF>,
     // The ptrbitarr and pfxbitarr for this node
     pub(crate) node: TreeBitMapNode<AF>,
diff --git a/src/tree_bitmap/tree_bitmap_node.rs b/src/tree_bitmap/tree_bitmap_node.rs
index 46856422..1cdb70ff 100644
--- a/src/tree_bitmap/tree_bitmap_node.rs
+++ b/src/tree_bitmap/tree_bitmap_node.rs
@@ -628,7 +628,7 @@ pub(crate) enum NewNodeOrIndex<AF: AddressFamily> {
 // a Prefix.
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub struct NodeId<AF: AddressFamily> {
+pub(crate) struct NodeId<AF: AddressFamily> {
     bits: AF,
     len: u8,
 }

From 4ac3375cd7ae974fc477c64c01199ac48d576a7b Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 19:53:03 +0100
Subject: [PATCH 135/147] update comments

---
 src/cht/oncebox.rs                  |  10 +
 src/lsm_tree/mod.rs                 | 324 +++-------------------------
 src/tree_bitmap/tree_bitmap_node.rs |   5 -
 src/types/af.rs                     |   1 +
 4 files changed, 37 insertions(+), 303 deletions(-)

diff --git a/src/cht/oncebox.rs b/src/cht/oncebox.rs
index 96d058fa..d05973d8 100644
--- a/src/cht/oncebox.rs
+++ b/src/cht/oncebox.rs
@@ -2,6 +2,12 @@ use std::ptr::null_mut;
 use std::slice;
 use std::sync::atomic::{AtomicPtr, Ordering};
 
+//------------ OnceBox -------------------------------------------------------
+//
+// Create an atomic pointer once, never to be modified. The pointee can be
+// changed, if enough considerations around atomically updating values are
+// taken into account. Used by the Chained Hash Table (Cht) in `cht`.
+
 #[derive(Debug, Default)]
 pub struct OnceBox<T> {
     ptr: AtomicPtr<T>,
@@ -61,6 +67,10 @@ impl<T> Drop for OnceBox<T> {
     }
 }
 
+//------------ OnceBoxSlice --------------------------------------------------
+//
+// A slice of OnceBoxes, subject to the same constraints. Used in Cht.
+
 #[derive(Debug, Default)]
 pub(crate) struct OnceBoxSlice<T> {
     ptr: AtomicPtr<OnceBox<T>>,
diff --git a/src/lsm_tree/mod.rs b/src/lsm_tree/mod.rs
index ee76088d..203dc181 100644
--- a/src/lsm_tree/mod.rs
+++ b/src/lsm_tree/mod.rs
@@ -1,5 +1,3 @@
-//------------ PersistTree ---------------------------------------------------
-
 use std::marker::PhantomData;
 use std::path::Path;
 
@@ -19,21 +17,16 @@ use crate::types::prefix_record::{ValueHeader, ZeroCopyRecord};
 use crate::types::{AddressFamily, Record};
 use crate::types::{PrefixId, RouteStatus};
 
-pub(crate) trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
-    TryFromBytes + KnownLayout + IntoBytes + Unaligned + Immutable
-{
-    // fn mut_from_bytes(
-    //     bytes: &mut [u8],
-    // ) -> std::result::Result<&mut Self, ZeroCopyMutError<'_, Self>> {
-    //     Self::try_mut_from_bytes(bytes.as_mut_bytes())
-    // }
+//------------ Key -----------------------------------------------------------
 
-    // fn from_bytes(
-    //     bytes: &[u8],
-    // ) -> std::result::Result<&Self, ZeroCopyError<'_, Self>> {
-    //     Self::try_ref_from_bytes(bytes.as_bytes())
-    // }
+// The type of key used to create entries in the LsmTree. Can be short or
+// long. Short keys overwrite existing values for existing (prefix, mui)
+// pairs, whereas long keys append values with existing (prefix, mui), thus
+// creating persisted historical records.
 
+pub(crate) trait Key<AF: AddressFamily, const KEY_SIZE: usize>:
+    TryFromBytes + KnownLayout + IntoBytes + Unaligned + Immutable
+{
     // Try to extract a header from the bytes for reading only. If this
     // somehow fails, we don't know what to do anymore. Data may be corrupted,
     // so it probably should not be retried.
@@ -50,13 +43,6 @@ pub(crate) trait KeySize<AF: AddressFamily, const KEY_SIZE: usize>:
         LongKey::try_mut_from_bytes(bytes.as_mut_bytes())
             .map_err(|_| FatalError)
     }
-
-    // fn _short_key(bytes: &[u8]) -> &ShortKey<AF> {
-    //     trace!("short key from bytes {:?}", bytes);
-    //     let s_b = &bytes[..(AF::BITS as usize / 8) + 6];
-    //     trace!("short key {:?}", s_b);
-    //     ShortKey::try_ref_from_prefix(bytes).unwrap().0
-    // }
 }
 
 #[derive(Debug, KnownLayout, Immutable, FromBytes, Unaligned, IntoBytes)]
@@ -84,7 +70,7 @@ pub struct LongKey<AF: AddressFamily> {
     status: RouteStatus,      // 1
 } // 18 or 30
 
-impl<AF: AddressFamily, const KEY_SIZE: usize> KeySize<AF, KEY_SIZE>
+impl<AF: AddressFamily, const KEY_SIZE: usize> Key<AF, KEY_SIZE>
     for ShortKey<AF>
 {
 }
@@ -98,7 +84,7 @@ impl<AF: AddressFamily> From<(PrefixId<AF>, u32)> for ShortKey<AF> {
     }
 }
 
-impl<AF: AddressFamily, const KEY_SIZE: usize> KeySize<AF, KEY_SIZE>
+impl<AF: AddressFamily, const KEY_SIZE: usize> Key<AF, KEY_SIZE>
     for LongKey<AF>
 {
 }
@@ -116,13 +102,18 @@ impl<AF: AddressFamily> From<(PrefixId<AF>, u32, u64, RouteStatus)>
     }
 }
 
+//------------ LsmTree -------------------------------------------------------
+
+// The log-structured merge tree that backs the persistent store (on disk).
+
 pub struct LsmTree<
+    // The address family that this tree stores. IPv4 or IPv6.
     AF: AddressFamily,
-    K: KeySize<AF, KEY_SIZE>,
-    // The size in bytes of the prefix in the persisted storage (disk), this
-    // amounnts to the bytes for the addres (4 for IPv4, 16 for IPv6) and 1
-    // bytefor the prefix length.
-    // const PREFIX_SIZE: usize,
+    // The Key type for this tree. This can basically be a long key, if the
+    // store needs to store historical records, or a short key, if it should
+    // overwrite records for (prefix, mui) pairs, effectively only keeping the
+    // current state.
+    K: Key<AF, KEY_SIZE>,
     // The size in bytes of the complete key in the persisted storage, this
     // is PREFIX_SIZE bytes (4; 16) + mui size (4) + ltime (8)
     const KEY_SIZE: usize,
@@ -133,7 +124,7 @@ pub struct LsmTree<
     _k: PhantomData<K>,
 }
 
-impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
+impl<AF: AddressFamily, K: Key<AF, KEY_SIZE>, const KEY_SIZE: usize>
     LsmTree<AF, K, KEY_SIZE>
 {
     pub fn new(persist_path: &Path) -> FatalResult<LsmTree<AF, K, KEY_SIZE>> {
@@ -163,7 +154,7 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
     }
 
     // Based on the properties of the lsm_tree we can assume that the key and
-    // value concatenated in this method always has a lenght of greater than
+    // value concatenated in this method always has a length of greater than
     // KEYS_SIZE, a global constant for the store per AF.
     #[allow(clippy::indexing_slicing)]
     pub fn get_records_for_prefix(
@@ -250,7 +241,6 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
             // All muis, exclude withdrawn routes
             (None, false) => {
                 // get all records for this prefix
-                // let prefix_b = &prefix.to_len_first_bytes::<PREFIX_SIZE>();
                 self.tree
                     .prefix(prefix.as_bytes(), None, None)
                     .filter_map(|r| {
@@ -403,218 +393,6 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
             .collect::<Vec<_>>()
     }
 
-    // fn get_records_for_more_specific_prefix_in_len(
-    //     &self,
-    //     prefix: PrefixId<AF>,
-    //     len: u8,
-    // ) -> Box<
-    //     dyn DoubleEndedIterator<
-    //         Item = Result<
-    //             (lsm_tree::Slice, lsm_tree::Slice),
-    //             lsm_tree::Error,
-    //         >,
-    //     >,
-    // > {
-    //     let start = PrefixId::new(prefix.get_net(), len);
-    //     let end: [u8; PREFIX_SIZE] = start.inc_len().to_len_first_bytes();
-
-    //     self.tree.range(start.to_len_first_bytes()..end, None, None)
-    // }
-
-    // fn enrich_prefix<M: Meta>(
-    //     &self,
-    //     prefix: PrefixId<AF>,
-    //     mui: Option<u32>,
-    //     include_withdrawn: bool,
-    //     bmin: &RoaringBitmap,
-    // ) -> Vec<PublicRecord<M>> {
-    //     self.get_records_for_prefix(prefix, mui, include_withdrawn, bmin)
-    // .into_iter()
-    // .filter_map(|mut r| {
-    //     if !include_withdrawn && r.status == RouteStatus::Withdrawn {
-    //         return None;
-    //     }
-    //     if bmin.contains(r.multi_uniq_id) {
-    //         if !include_withdrawn {
-    //             return None;
-    //         }
-    //         r.status = RouteStatus::Withdrawn;
-    //     }
-    //     Some(r)
-    // })
-    // .collect()
-    // }
-
-    // fn enrich_prefixes<M: Meta>(
-    //     &self,
-    //     prefixes: Option<Vec<PrefixId<AF>>>,
-    //     mui: Option<u32>,
-    //     include_withdrawn: bool,
-    //     bmin: &RoaringBitmap,
-    // ) -> Option<FamilyRecord<AF, M>> {
-    //     prefixes.map(|recs| {
-    //         recs.iter()
-    //             .flat_map(move |pfx| {
-    //                 Some((
-    //                     *pfx,
-    //                     self.get_records_for_prefix(
-    //                         *pfx,
-    //                         mui,
-    //                         include_withdrawn,
-    //                         bmin,
-    //                     )
-    //                     .into_iter()
-    //                     .filter_map(|mut r| {
-    //                         if bmin.contains(r.multi_uniq_id) {
-    //                             if !include_withdrawn {
-    //                                 return None;
-    //                             }
-    //                             r.status = RouteStatus::Withdrawn;
-    //                         }
-    //                         Some(r)
-    //                     })
-    //                     .collect(),
-    //                 ))
-    //             })
-    //             .collect()
-    //     })
-    // }
-
-    // fn sparse_record_set<M: Meta>(
-    //     &self,
-    //     prefixes: Option<Vec<PrefixId<AF>>>,
-    // ) -> Option<FamilyRecord<AF, M>> {
-    //     prefixes.map(|recs| {
-    //         recs.iter().flat_map(|pfx| Some((*pfx, vec![]))).collect()
-    //     })
-    // }
-
-    // pub(crate) fn match_prefix<M: Meta>(
-    //     &self,
-    //     search_pfxs: TreeQueryResult<AF>,
-    //     options: &MatchOptions,
-    //     bmin: &RoaringBitmap,
-    // ) -> FamilyQueryResult<AF, M> {
-    //     let (prefix, prefix_meta) = if let Some(prefix) = search_pfxs.prefix {
-    //         (
-    //             prefix,
-    //             self.get_records_for_prefix(
-    //                 prefix,
-    //                 options.mui,
-    //                 options.include_withdrawn,
-    //                 bmin,
-    //             ),
-    //         )
-    //     } else {
-    //         return FamilyQueryResult {
-    //             match_type: MatchType::EmptyMatch,
-    //             prefix: None,
-    //             prefix_meta: vec![],
-    //             less_specifics: if options.include_less_specifics {
-    //                 search_pfxs.less_specifics.map(|v| {
-    //                     v.into_iter().map(|p| (p, vec![])).collect::<Vec<_>>()
-    //                 })
-    //             } else {
-    //                 None
-    //             },
-    //             more_specifics: if options.include_more_specifics {
-    //                 search_pfxs.more_specifics.map(|v| {
-    //                     v.into_iter().map(|p| (p, vec![])).collect::<Vec<_>>()
-    //                 })
-    //             } else {
-    //                 None
-    //             },
-    //         };
-    //     };
-
-    //     let mut res = match options.include_history {
-    //         // All the records for all the prefixes
-    //         IncludeHistory::All => FamilyQueryResult {
-    //             prefix: Some(prefix),
-    //             prefix_meta,
-    //             match_type: search_pfxs.match_type,
-    //             less_specifics: self.enrich_prefixes(
-    //                 search_pfxs.less_specifics,
-    //                 options.mui,
-    //                 options.include_withdrawn,
-    //                 bmin,
-    //             ),
-    //             more_specifics: search_pfxs.more_specifics.map(|ms| {
-    //                 self.more_specific_prefix_iter_from(
-    //                     prefix,
-    //                     ms.iter().map(|p| p.get_len()).collect::<Vec<_>>(),
-    //                     options.mui,
-    //                     bmin,
-    //                     options.include_withdrawn,
-    //                 )
-    //                 .collect::<Vec<_>>()
-    //             }),
-    //         },
-    //         // Only the search prefix itself has historical records attached
-    //         // to it, other prefixes (less|more specifics), have no records
-    //         // attached. Not useful with the MemoryOnly strategy (historical
-    //         // records are neve kept in memory).
-    //         IncludeHistory::SearchPrefix => FamilyQueryResult {
-    //             prefix: Some(prefix),
-    //             prefix_meta,
-    //             match_type: search_pfxs.match_type,
-    //             less_specifics: self
-    //                 .sparse_record_set(search_pfxs.less_specifics),
-    //             more_specifics: self
-    //                 .sparse_record_set(search_pfxs.more_specifics),
-    //         },
-    //         // Only the most recent record of the search prefix is returned
-    //         // with the prefixes. This is used for the PersistOnly strategy.
-    //         IncludeHistory::None => {
-    //             println!("Include history: None");
-    //             FamilyQueryResult {
-    //                 prefix: Some(prefix),
-    //                 prefix_meta,
-    //                 match_type: search_pfxs.match_type,
-    //                 less_specifics: search_pfxs.less_specifics.map(|ls| {
-    //                     self.less_specific_prefix_iter_from(
-    //                         ls,
-    //                         options.mui,
-    //                         bmin,
-    //                         options.include_withdrawn,
-    //                     )
-    //                     .collect::<Vec<_>>()
-    //                 }),
-    //                 more_specifics: search_pfxs.more_specifics.map(|ms| {
-    //                     self.more_specific_prefix_iter_from(
-    //                         prefix,
-    //                         ms.iter()
-    //                             .map(|p| p.get_len())
-    //                             .collect::<Vec<_>>(),
-    //                         options.mui,
-    //                         bmin,
-    //                         options.include_withdrawn,
-    //                     )
-    //                     .collect::<Vec<_>>()
-    //                 }),
-    //             }
-    //         }
-    //     };
-
-    //     res.match_type = match (options.match_type, &res) {
-    //         (_, res) if !res.prefix_meta.is_empty() => MatchType::ExactMatch,
-    //         (MatchType::LongestMatch | MatchType::EmptyMatch, _) => {
-    //             if res
-    //                 .less_specifics
-    //                 .as_ref()
-    //                 .is_some_and(|lp| !lp.is_empty())
-    //             {
-    //                 MatchType::LongestMatch
-    //             } else {
-    //                 MatchType::EmptyMatch
-    //             }
-    //         }
-    //         (MatchType::ExactMatch, _) => MatchType::EmptyMatch,
-    //     };
-
-    //     res
-    // }
-
     pub fn flush_to_disk(&self) -> Result<(), lsm_tree::Error> {
         let segment = self.tree.flush_active_memtable(0);
 
@@ -641,7 +419,6 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
         self.counters.get_prefixes_count().iter().sum()
     }
 
-    //
     #[allow(clippy::indexing_slicing)]
     pub fn get_prefixes_count_for_len(
         &self,
@@ -740,60 +517,11 @@ impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
             _k: PhantomData,
         }
     }
-
-    // pub(crate) fn more_specific_prefix_iter_from<'a, M: Meta + 'a>(
-    //     &'a self,
-    //     search_prefix: PrefixId<AF>,
-    //     mut search_lengths: Vec<u8>,
-    //     mui: Option<u32>,
-    //     global_withdrawn_bmin: &'a RoaringBitmap,
-    //     include_withdrawn: bool,
-    // ) -> impl Iterator<Item = (PrefixId<AF>, Vec<Vec<u8>>)> + 'a {
-    //     trace!("search more specifics in the persist store.");
-    //     if search_lengths.is_empty() {
-    //         for l in search_prefix.get_len() + 1..=AF::BITS {
-    //             search_lengths.push(l);
-    //         }
-    //     }
-    //     println!("more specific prefix lengths {:?}", search_lengths);
-
-    //     let len = search_lengths.pop().unwrap();
-    //     let cur_range = self
-    //         .get_records_for_more_specific_prefix_in_len(search_prefix, len);
-
-    //     MoreSpecificPrefixIter {
-    //         store: self,
-    //         search_prefix,
-    //         search_lengths,
-    //         mui,
-    //         global_withdrawn_bmin,
-    //         include_withdrawn,
-    //         cur_range,
-    //         next_rec: None,
-    //     }
-    // }
-
-    // pub(crate) fn less_specific_prefix_iter_from<'a, M: Meta + 'a>(
-    //     &'a self,
-    //     search_lengths: Vec<PrefixId<AF>>,
-    //     mui: Option<u32>,
-    //     global_withdrawn_bmin: &'a RoaringBitmap,
-    //     include_withdrawn: bool,
-    // ) -> impl Iterator<Item = (PrefixId<AF>, Vec<PublicRecord<M>>)> + 'a {
-    //     LessSpecificPrefixIter {
-    //         store: self,
-    //         search_lengths,
-    //         mui,
-    //         global_withdrawn_bmin,
-    //         include_withdrawn,
-    //         _m: PhantomData,
-    //     }
-    // }
 }
 
 impl<
         AF: AddressFamily,
-        K: KeySize<AF, KEY_SIZE>,
+        K: Key<AF, KEY_SIZE>,
         // const PREFIX_SIZE: usize,
         const KEY_SIZE: usize,
     > std::fmt::Debug for LsmTree<AF, K, KEY_SIZE>
@@ -808,7 +536,7 @@ impl<
 // specified offset.
 pub(crate) struct PersistedPrefixIter<
     AF: AddressFamily,
-    K: KeySize<AF, KEY_SIZE>,
+    K: Key<AF, KEY_SIZE>,
     const KEY_SIZE: usize,
 > {
     cur_rec: Option<Vec<FatalResult<Vec<u8>>>>,
@@ -818,8 +546,8 @@ pub(crate) struct PersistedPrefixIter<
     _k: PhantomData<K>,
 }
 
-impl<AF: AddressFamily, K: KeySize<AF, KEY_SIZE>, const KEY_SIZE: usize>
-    Iterator for PersistedPrefixIter<AF, K, KEY_SIZE>
+impl<AF: AddressFamily, K: Key<AF, KEY_SIZE>, const KEY_SIZE: usize> Iterator
+    for PersistedPrefixIter<AF, K, KEY_SIZE>
 {
     type Item = Vec<FatalResult<Vec<u8>>>;
     fn next(&mut self) -> Option<Self::Item> {
diff --git a/src/tree_bitmap/tree_bitmap_node.rs b/src/tree_bitmap/tree_bitmap_node.rs
index 1cdb70ff..07257473 100644
--- a/src/tree_bitmap/tree_bitmap_node.rs
+++ b/src/tree_bitmap/tree_bitmap_node.rs
@@ -660,11 +660,6 @@ impl<AF: AddressFamily> NodeId<AF> {
         self.bits
     }
 
-    pub fn set_len(mut self, len: u8) -> Self {
-        self.len = len;
-        self
-    }
-
     pub(crate) fn add_to_len(mut self, len: u8) -> Self {
         self.len += len;
         self
diff --git a/src/types/af.rs b/src/types/af.rs
index ebcd7e10..05bd523b 100644
--- a/src/types/af.rs
+++ b/src/types/af.rs
@@ -4,6 +4,7 @@ use zerocopy::{NetworkEndian, U128, U32};
 use crate::types::BitSpan;
 
 //------------ AddressFamily (trait) ----------------------------------------
+//
 /// The address family of an IP address as a Trait.
 ///
 /// The idea of this trait is that each family will have a separate type to be

From 658f2089ec21b1ef04fdb1189124c85faae49f33 Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 19:55:26 +0100
Subject: [PATCH 136/147] allow multiple crate versions lint

---
 src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib.rs b/src/lib.rs
index be442288..a964a51f 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -4,7 +4,7 @@
     clippy::panic,
     clippy::indexing_slicing
 )]
-
+#![allow(clippy::multiple_crate_versions)]
 //! A library that provides abstractions for a BGP Routing Information Base
 //! (RIB) for different AFI/SAFI types, as a database.
 //!

From 8d80f2492ca50446834d8145227144c5098d9c4f Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 20:12:46 +0100
Subject: [PATCH 137/147] fix error type

---
 src/types/errors.rs | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/src/types/errors.rs b/src/types/errors.rs
index 9f540a71..252e4f78 100644
--- a/src/types/errors.rs
+++ b/src/types/errors.rs
@@ -1,6 +1,8 @@
 use std::fmt;
 
-/// Possible errors returned by methods on a RIB
+/// Possible errors returned by methods on a RIB. Most of these errors are
+// recoverable, there is one variant [PrefixStoreError::FatalError] that is
+// unrecoverable, like the stand-alone type.
 #[derive(Debug, PartialEq, Eq)]
 pub enum PrefixStoreError {
     /// There is too much contention while creating a node: the store has
@@ -34,9 +36,6 @@ pub enum PrefixStoreError {
     /// A record was specifically requested from the in-memory data structure,
     /// but the record is not in memory. It may be persisted to disk.
     RecordNotInMemory,
-    /// An error external to to our execution happened, most notably another
-    ///thread panicking while trying to acquire a lock.
-    ExternalError,
     /// The method returning this error was trying to persist records to disk
     /// but failed. Retrying is safe, but may be yield the same result.
     PersistFailed,
@@ -92,13 +91,6 @@ impl fmt::Display for PrefixStoreError {
                     persisted."
                 )
             }
-            PrefixStoreError::ExternalError => {
-                write!(
-                f,
-                "Error: An action could not be completed, due to another \
-                thread panicking."
-            )
-            }
             PrefixStoreError::StatusUnknown => {
                 write!(
                     f,
@@ -118,6 +110,9 @@ impl fmt::Display for PrefixStoreError {
     }
 }
 
+/// An unrecoverable error, that can occur during disk I/O or writing memory.
+/// All data in the store should be considered corrupy and the application
+/// receiving this error should probably terminate.
 #[derive(Debug, Copy, Clone)]
 pub struct FatalError;
 

From b8eb0ec74356704b6b0c5ca9826819811e43321d Mon Sep 17 00:00:00 2001
From: Jasper den Hertog <jasper@plainspace.com>
Date: Wed, 26 Mar 2025 20:32:46 +0100
Subject: [PATCH 138/147] prefix_meta -> records

---
 examples/multi_thread_multi_prefix.rs  |  2 +-
 examples/multi_thread_single_prefix.rs |  2 +-
 src/lib.rs                             |  2 +-
 src/rib/starcast_af_query.rs           | 10 +++++-----
 src/types/match_options.rs             | 16 +++++++++++++---
 tests/best-path.rs                     |  2 +-
 tests/concurrency.rs                   |  4 ++--
 tests/treebitmap.rs                    | 16 ++++++++--------
 8 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/examples/multi_thread_multi_prefix.rs b/examples/multi_thread_multi_prefix.rs
index e63098fa..b9d31110 100644
--- a/examples/multi_thread_multi_prefix.rs
+++ b/examples/multi_thread_multi_prefix.rs
@@ -88,7 +88,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                     guard,
                                 )
                                 .unwrap()
-                                .prefix_meta;
+                                .records;
                             x += 1;
                         }
 
diff --git a/examples/multi_thread_single_prefix.rs b/examples/multi_thread_single_prefix.rs
index 562e25dd..801c8aa5 100644
--- a/examples/multi_thread_single_prefix.rs
+++ b/examples/multi_thread_single_prefix.rs
@@ -74,7 +74,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                             include_history: IncludeHistory::None,
                         },
                         guard,
-                    ).unwrap().prefix_meta;
+                    ).unwrap().records;
                             x += 1;
                         }
 
diff --git a/src/lib.rs b/src/lib.rs
index a964a51f..5951926b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -107,7 +107,7 @@ pub use types::prefix_record;
 /// Error types returned by a RIB
 pub use types::errors;
 
-/// Trait that defines the AFIs 1 (IPv4) and 2 (IPv6)
+/// Trait that defines the AFIs 1 (IPv4) and 2 (IPv6).
 pub use types::af::AddressFamily;
 
 /// The underlying value (u32) and trait impl for AFI 1.
diff --git a/src/rib/starcast_af_query.rs b/src/rib/starcast_af_query.rs
index 989a135c..9be2bf17 100644
--- a/src/rib/starcast_af_query.rs
+++ b/src/rib/starcast_af_query.rs
@@ -130,7 +130,7 @@ impl<
 
         Ok(QueryResult {
             prefix,
-            prefix_meta,
+            records: prefix_meta,
             // prefix.map(|_pfx| {
             //     self.get_value(prefix_id, mui, include_withdrawn, guard)?
             //         .unwrap_or_default()
@@ -169,7 +169,7 @@ impl<
 
         Ok(QueryResult {
             prefix,
-            prefix_meta,
+            records: prefix_meta,
             match_type: MatchType::EmptyMatch,
             less_specifics,
             more_specifics: None,
@@ -268,7 +268,7 @@ impl<
                 res.and_then(|v| if v.is_empty() { None } else { Some(v) })
             })
         }) {
-            res.prefix_meta = m;
+            res.records = m;
         } else {
             res.prefix = None;
             res.match_type = MatchType::EmptyMatch;
@@ -394,7 +394,7 @@ impl<AF: AddressFamily, M: Meta> From<TreeQueryResult<AF>>
         Self {
             match_type: value.match_type,
             prefix: value.prefix.map(|p| p.into()),
-            prefix_meta: vec![],
+            records: vec![],
             less_specifics: value
                 .less_specifics
                 .map(|ls| ls.into_iter().map(|p| (p, vec![])).collect()),
@@ -436,7 +436,7 @@ impl<AF: AddressFamily, M: Meta> From<FamilyQueryResult<AF, M>>
         QueryResult {
             match_type: value.match_type,
             prefix: value.prefix.map(|p| p.into()),
-            prefix_meta: value.prefix_meta,
+            records: value.prefix_meta,
             less_specifics: value
                 .less_specifics
                 .map(|ls| ls.into_iter().collect()),
diff --git a/src/types/match_options.rs b/src/types/match_options.rs
index 2bd13e39..7c6553d8 100644
--- a/src/types/match_options.rs
+++ b/src/types/match_options.rs
@@ -4,6 +4,7 @@ use std::fmt;
 use inetnum::addr::Prefix;
 
 use super::prefix_record::Meta;
+use crate::rib::StarCastRib;
 
 //------------ MatchOptions / MatchType -------------------------------------
 
@@ -38,10 +39,19 @@ pub struct MatchOptions {
     pub include_history: IncludeHistory,
 }
 
+/// Option to set the match type for a prefix match. Type can be Exact,
+/// Longest, or Empty. The match type only applies to the `prefix` and
+/// `records` fields in the [QueryResult] that is returned by a
+/// [StarCastRib::match_prefix()] query.
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 pub enum MatchType {
+    /// Only return the requested prefix, and the associated records, if the
+    /// requested prefix exactly matches the found prefix(es) (if any).
     ExactMatch,
+    /// Return the longest matching prefix for the requested prefix (if
+    /// any). May match the prefix exactly.
     LongestMatch,
+    /// Return the longest matching prefix, or none at all.
     EmptyMatch,
 }
 
@@ -91,7 +101,7 @@ pub struct QueryResult<M: Meta> {
     /// The resulting prefix record
     pub prefix: Option<Prefix>,
     /// The meta data associated with the resulting prefix record
-    pub prefix_meta: Vec<Record<M>>,
+    pub records: Vec<Record<M>>,
     /// The less-specifics of the resulting prefix together with their meta
     /// data
     pub less_specifics: Option<RecordSet<M>>,
@@ -105,7 +115,7 @@ impl<M: Meta> QueryResult<M> {
         QueryResult {
             match_type: MatchType::EmptyMatch,
             prefix: None,
-            prefix_meta: vec![],
+            records: vec![],
             less_specifics: None,
             more_specifics: None,
         }
@@ -125,7 +135,7 @@ impl<M: Meta> fmt::Display for QueryResult<M> {
         writeln!(f, "match_type: {}", self.match_type)?;
         writeln!(f, "prefix: {}", pfx_str)?;
         write!(f, "meta: [ ")?;
-        for rec in &self.prefix_meta {
+        for rec in &self.records {
             write!(f, "{},", rec)?;
         }
         writeln!(f, " ]")?;
diff --git a/tests/best-path.rs b/tests/best-path.rs
index 562331e1..577ee463 100644
--- a/tests/best-path.rs
+++ b/tests/best-path.rs
@@ -190,7 +190,7 @@ fn test_best_path_1(// tree_bitmap: MultiThreadedStore<Ipv4Route>,
         &rotonda_store::epoch::pin(),
     );
 
-    println!("{:?}", res.as_ref().unwrap().prefix_meta);
+    println!("{:?}", res.as_ref().unwrap().records);
     let best_path = tree_bitmap.best_path(&pfx, &rotonda_store::epoch::pin());
     println!(
         "ps outdated? {}",
diff --git a/tests/concurrency.rs b/tests/concurrency.rs
index d22cc312..409896c3 100644
--- a/tests/concurrency.rs
+++ b/tests/concurrency.rs
@@ -317,7 +317,7 @@ fn test_concurrent_updates_1<C: Config + Sync + Send + 'static>(
         println!("strategy {:?}", tree_bitmap.persist_strategy());
         println!("PFX {}", res);
         assert_eq!(
-            res.prefix_meta
+            res.records
                 .iter()
                 .find(|m| m.multi_uniq_id == 2)
                 .unwrap()
@@ -510,7 +510,7 @@ fn test_concurrent_updates_2(// tree_bitmap: Arc<MultiThreadedStore<BeBytesAsn>>
         println!("RES {:#?}", res);
         assert_eq!(
             res.unwrap()
-                .prefix_meta
+                .records
                 .iter()
                 .find(|m| m.multi_uniq_id == 2)
                 .unwrap()
diff --git a/tests/treebitmap.rs b/tests/treebitmap.rs
index c9b8b2a0..1af79705 100644
--- a/tests/treebitmap.rs
+++ b/tests/treebitmap.rs
@@ -611,9 +611,9 @@ mod tests {
             guard,
         )?;
         print!(".pfx {:#?}.", all_recs_for_pfx);
-        assert_eq!(all_recs_for_pfx.prefix_meta.len(), 5);
+        assert_eq!(all_recs_for_pfx.records.len(), 5);
         let wd_rec = all_recs_for_pfx
-            .prefix_meta
+            .records
             .iter()
             .filter(|r| r.status == RouteStatus::Withdrawn)
             .collect::<Vec<_>>();
@@ -632,9 +632,9 @@ mod tests {
             },
             guard,
         )?;
-        assert_eq!(active_recs_for_pfx.prefix_meta.len(), 4);
+        assert_eq!(active_recs_for_pfx.records.len(), 4);
         assert!(!active_recs_for_pfx
-            .prefix_meta
+            .records
             .iter()
             .any(|r| r.multi_uniq_id == 1));
 
@@ -823,8 +823,8 @@ mod tests {
         println!("more_specifics match w/o withdrawn #2 {}", more_specifics);
         // We withdrew mui 1 for the requested prefix itself, since mui 2 was
         // already withdrawn above, we're left with 3 records
-        println!("PREFIX META: {:#?}", more_specifics.prefix_meta);
-        assert_eq!(more_specifics.prefix_meta.len(), 3);
+        println!("PREFIX META: {:#?}", more_specifics.records);
+        assert_eq!(more_specifics.records.len(), 3);
 
         let more_specifics = more_specifics.more_specifics.unwrap();
 
@@ -881,7 +881,7 @@ mod tests {
 
         // This prefix should not be found, since we withdrew all records
         // for it.
-        assert!(more_specifics.prefix_meta.is_empty());
+        assert!(more_specifics.records.is_empty());
 
         // ..as a result, its resulting match_type should be EmptyMatch
         assert_eq!(more_specifics.match_type, MatchType::EmptyMatch);
@@ -937,7 +937,7 @@ mod tests {
 
         trace!("{:#?}", query);
 
-        assert_eq!(query.prefix_meta.len(), 5);
+        assert_eq!(query.records.len(), 5);
 
         let less_specifics = query.less_specifics.unwrap();
 

From 5949be6b91076912a177908b244e7b62c884c88f Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 27 Mar 2025 12:04:17 +0100
Subject: [PATCH 139/147] branchless truncate_to_len

---
 src/bin/truncate_len.rs             | 46 +++++++++++++++++++++++++++++
 src/lib.rs                          | 18 +++++------
 src/tree_bitmap/tree_bitmap_node.rs |  2 +-
 src/types/af.rs                     | 38 +++++-------------------
 4 files changed, 64 insertions(+), 40 deletions(-)
 create mode 100644 src/bin/truncate_len.rs

diff --git a/src/bin/truncate_len.rs b/src/bin/truncate_len.rs
new file mode 100644
index 00000000..190f6e12
--- /dev/null
+++ b/src/bin/truncate_len.rs
@@ -0,0 +1,46 @@
+use zerocopy::{NetworkEndian, U32};
+
+// max len 128
+// fn truncate_to_len(bits: U32<NetworkEndian>, len: u8) -> u32 {
+//     match len {
+//         0 => U32::from(0),
+//         1..=31 => (bits >> (32 - len as u32)) << (32 - len as u32),
+//         32 => bits,
+//         len => panic!("Can't truncate to more than 128 bits: {}", len),
+//     }
+// }
+
+fn truncate_to_len(bits: U32<NetworkEndian>, len: u8) -> U32<NetworkEndian> {
+    match len {
+        0 => U32::new(0),
+        1..=31 => {
+            (bits >> U32::from(32 - len as u32)) << U32::from(32 - len as u32)
+        }
+        32 => bits,
+        len => panic!("Can't truncate to more than 128 bits: {}", len),
+    }
+}
+
+fn branchless_trunc(bits: U32<NetworkEndian>, len: u8) -> u32 {
+    (bits
+        & ((1_u32.rotate_right(len as u32)
+            ^ 1_u32.saturating_sub(len as u32))
+        .wrapping_sub(1)
+            ^ u32::MAX))
+        .into()
+}
+
+fn main() {
+    let ex_bits =
+        U32::<NetworkEndian>::from(0b10_1110_0100_0000_0000_0000_0000_0000);
+    for l in 0..=32 {
+        let tl1 = truncate_to_len(ex_bits, l);
+        let tl2 = branchless_trunc(ex_bits, l);
+        println!(
+            "{:032b} {:032b}",
+            branchless_trunc(ex_bits, l),
+            truncate_to_len(ex_bits, l)
+        );
+        assert_eq!(tl1, tl2);
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index 5951926b..2db27a68 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -29,18 +29,18 @@
 //                                ┌───────────────────┐
 //                                │    StarCastRib    │
 //                                └────────┬┬─────────┘
-//                        ┌────────────────┘└─────────────────┐
-//                     ┌──▼──┐                             ┌──▼──┐
-//                     │ v4  │                             │ v4  │
-//                     └──┬──┘                             └──┬──┘
-//            ┌───────────┼──────────┐            ┌───────────┼──────────┐
+//                        ┌────────────────┘└────────────────┐
+//                     ┌──▼──┐                            ┌──▼──┐
+//                     │ v4  │                            │ v4  │
+//                     └──┬──┘                            └──┬──┘
+//            ┌───────────┼─────────┐            ┌───────────┼──────────┐
 //      ┌─────▼────┐┌─────▼────┐┌───▼─────┐┌─────▼────┐┌─────▼────┐┌────▼───┐
 //      │treebitmap││prefix_cht││lsm_tree ││treebitmap││prefix_cht││lsm_tree│
 //      └─────┬────┘└──────────┘└─────────┘└─────┬────┘└──────────┘└────────┘
-//     ┌──────┴─────┐                      ┌──────┴─────┐
-// ┌───▼────┐┌──────▼────┐             ┌───▼────┐┌──────▼────┐
-// │node_cht││muis_bitmap│             │node_cht││muis_bitmap│
-// └────────┘└───────────┘             └────────┘└───────────┘
+//     ┌──────┴─────┐                     ┌──────┴─────┐
+// ┌───▼────┐┌──────▼────┐            ┌───▼────┐┌──────▼────┐
+// │node_cht││muis_bitmap│            │node_cht││muis_bitmap│
+// └────────┘└───────────┘            └────────┘└───────────┘
 
 // Rotonda-store is a fairly layered repo, it uses three different
 // types of trees, that are all hidden behind one public interface.
diff --git a/src/tree_bitmap/tree_bitmap_node.rs b/src/tree_bitmap/tree_bitmap_node.rs
index 07257473..aa707a2f 100644
--- a/src/tree_bitmap/tree_bitmap_node.rs
+++ b/src/tree_bitmap/tree_bitmap_node.rs
@@ -671,7 +671,7 @@ impl<AF: AddressFamily> NodeId<AF> {
     }
 
     // clean out all bits that are set beyond the len. This function should
-    // be used before doing any ORing to add a nibble.
+    // be used before doing any ORing to add a bitspan.
     #[inline]
     pub(crate) fn with_cleaned_id(&self) -> (AF, u8) {
         (self.bits.truncate_to_len(self.len), self.len)
diff --git a/src/types/af.rs b/src/types/af.rs
index 05bd523b..b1a8b991 100644
--- a/src/types/af.rs
+++ b/src/types/af.rs
@@ -127,22 +127,11 @@ impl AddressFamily for IPv4 {
         self.into()
     }
 
-    // We are totally allowing panic here: the panicking arm holds an
-    // invariant that's a super basic assumption of this whole store. If this
-    // panics than this whole library should not be used, and be checked for
-    // logic errors everywhere. For performance reasons we are leaving out the
-    // FatalResult wrapper.
-    #[allow(clippy::panic)]
     fn truncate_to_len(self, len: u8) -> Self {
-        match len {
-            0 => U32::new(0),
-            1..=31 => {
-                (self >> U32::from(32 - len as u32))
-                    << U32::from(32 - len as u32)
-            }
-            32 => self,
-            len => panic!("Can't truncate to more than 128 bits: {}", len),
-        }
+        self & ((1_u32.rotate_right(len as u32)
+            ^ 1_u32.saturating_sub(len as u32))
+        .wrapping_sub(1)
+            ^ u32::MAX)
     }
 
     fn checked_shr_or_zero(self, rhs: u32) -> Self {
@@ -188,22 +177,11 @@ impl AddressFamily for IPv6 {
         (res, len + bs.len)
     }
 
-    // We are totally allowing panic here: the panicking arm holds an
-    // invariant that's a super basic assumption of this whole store. If this
-    // panics than this whole library should not be used, and be checked for
-    // logic errors everywhere. For performance reasons we are leaving out the
-    // FatalResult wrapper.
-    #[allow(clippy::panic)]
     fn truncate_to_len(self, len: u8) -> Self {
-        match len {
-            0 => U128::new(0),
-            1..=127 => {
-                (self >> U128::from(128 - len as u128))
-                    << U128::from(128 - len as u128)
-            }
-            128 => self,
-            len => panic!("Can't truncate to more than 128 bits: {}", len),
-        }
+        self & ((1_u128.rotate_right(len as u32)
+            ^ 1_u128.saturating_sub(len as u128))
+        .wrapping_sub(1)
+            ^ u128::MAX)
     }
 
     fn into_ipaddr(self) -> std::net::IpAddr {

From b255c0c8759bea67def6e9bc03061b1eb80d7c44 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 27 Mar 2025 12:14:32 +0100
Subject: [PATCH 140/147] add truncate_len test bin

---
 src/bin/truncate_len.rs | 31 +++++++++++--------------------
 1 file changed, 11 insertions(+), 20 deletions(-)

diff --git a/src/bin/truncate_len.rs b/src/bin/truncate_len.rs
index 190f6e12..7512a69f 100644
--- a/src/bin/truncate_len.rs
+++ b/src/bin/truncate_len.rs
@@ -1,14 +1,6 @@
 use zerocopy::{NetworkEndian, U32};
 
-// max len 128
-// fn truncate_to_len(bits: U32<NetworkEndian>, len: u8) -> u32 {
-//     match len {
-//         0 => U32::from(0),
-//         1..=31 => (bits >> (32 - len as u32)) << (32 - len as u32),
-//         32 => bits,
-//         len => panic!("Can't truncate to more than 128 bits: {}", len),
-//     }
-// }
+// max len 128!
 
 fn truncate_to_len(bits: U32<NetworkEndian>, len: u8) -> U32<NetworkEndian> {
     match len {
@@ -31,16 +23,15 @@ fn branchless_trunc(bits: U32<NetworkEndian>, len: u8) -> u32 {
 }
 
 fn main() {
-    let ex_bits =
-        U32::<NetworkEndian>::from(0b10_1110_0100_0000_0000_0000_0000_0000);
-    for l in 0..=32 {
-        let tl1 = truncate_to_len(ex_bits, l);
-        let tl2 = branchless_trunc(ex_bits, l);
-        println!(
-            "{:032b} {:032b}",
-            branchless_trunc(ex_bits, l),
-            truncate_to_len(ex_bits, l)
-        );
-        assert_eq!(tl1, tl2);
+    for b in 0..=u32::MAX {
+        if b % (1024 * 256) == 0 {
+            print!(".");
+        }
+        for l in 0..=32 {
+            let tl1 = truncate_to_len(U32::<NetworkEndian>::new(b), l);
+            let tl2 = branchless_trunc(U32::<NetworkEndian>::new(b), l);
+            // println!("{:032b} {:032b}", tl1, tl2);
+            assert_eq!(tl1, tl2);
+        }
     }
 }

From 6f26f576066c68864dcc735acdec37b2697f4940 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 27 Mar 2025 14:10:26 +0100
Subject: [PATCH 141/147] remove unused import

---
 src/types/match_options.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/types/match_options.rs b/src/types/match_options.rs
index 7c6553d8..49c9caea 100644
--- a/src/types/match_options.rs
+++ b/src/types/match_options.rs
@@ -4,7 +4,6 @@ use std::fmt;
 use inetnum::addr::Prefix;
 
 use super::prefix_record::Meta;
-use crate::rib::StarCastRib;
 
 //------------ MatchOptions / MatchType -------------------------------------
 

From 58d58f8339a13369bb5bc41de1c1ca1959d52222 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Thu, 27 Mar 2025 18:08:59 +0100
Subject: [PATCH 142/147] counters reform

---
 src/lsm_tree/mod.rs                      | 18 +++++---
 src/prefix_cht/cht.rs                    | 42 ++++++++++++++----
 src/rib/starcast.rs                      | 32 +++++++++-----
 src/rib/starcast_af.rs                   | 42 +++++++++++-------
 src/rib/starcast_af_query.rs             | 52 ++---------------------
 src/tree_bitmap/mod.rs                   | 54 ++++++------------------
 src/tree_bitmap/tree_bitmap_iterators.rs |  2 +-
 src/types/stats.rs                       | 31 +++++++++-----
 8 files changed, 131 insertions(+), 142 deletions(-)

diff --git a/src/lsm_tree/mod.rs b/src/lsm_tree/mod.rs
index 203dc181..ebc6530b 100644
--- a/src/lsm_tree/mod.rs
+++ b/src/lsm_tree/mod.rs
@@ -157,7 +157,7 @@ impl<AF: AddressFamily, K: Key<AF, KEY_SIZE>, const KEY_SIZE: usize>
     // value concatenated in this method always has a length of greater than
     // KEYS_SIZE, a global constant for the store per AF.
     #[allow(clippy::indexing_slicing)]
-    pub fn get_records_for_prefix(
+    pub fn records_for_prefix(
         &self,
         prefix: PrefixId<AF>,
         mui: Option<u32>,
@@ -341,7 +341,7 @@ impl<AF: AddressFamily, K: Key<AF, KEY_SIZE>, const KEY_SIZE: usize>
         }
     }
 
-    pub fn get_most_recent_record_for_prefix_mui(
+    pub fn most_recent_record_for_prefix_mui(
         &self,
         prefix: PrefixId<AF>,
         mui: u32,
@@ -374,7 +374,7 @@ impl<AF: AddressFamily, K: Key<AF, KEY_SIZE>, const KEY_SIZE: usize>
         res.map(|r| Some(r.to_vec()))
     }
 
-    pub(crate) fn get_records_with_keys_for_prefix_mui(
+    pub(crate) fn records_with_keys_for_prefix_mui(
         &self,
         prefix: PrefixId<AF>,
         mui: u32,
@@ -415,17 +415,21 @@ impl<AF: AddressFamily, K: Key<AF, KEY_SIZE>, const KEY_SIZE: usize>
         self.tree.disk_space()
     }
 
-    pub fn get_prefixes_count(&self) -> usize {
-        self.counters.get_prefixes_count().iter().sum()
+    pub fn prefixes_count(&self) -> usize {
+        self.counters.prefixes_count().iter().sum()
+    }
+
+    pub fn routes_count(&self) -> usize {
+        self.counters.routes_count()
     }
 
     #[allow(clippy::indexing_slicing)]
-    pub fn get_prefixes_count_for_len(
+    pub fn prefixes_count_for_len(
         &self,
         len: u8,
     ) -> Result<usize, PrefixStoreError> {
         if len <= AF::BITS {
-            Ok(self.counters.get_prefixes_count()[len as usize])
+            Ok(self.counters.prefixes_count()[len as usize])
         } else {
             Err(PrefixStoreError::StoreNotReadyError)
         }
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index 55982ea3..58d855bc 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -12,7 +12,7 @@ use roaring::RoaringBitmap;
 use crate::cht::{nodeset_size, prev_node_size};
 use crate::errors::{FatalError, FatalResult};
 use crate::prefix_record::Meta;
-use crate::stats::UpsertReport;
+use crate::stats::{Counters, UpsertReport};
 use crate::types::RouteStatus;
 use crate::{
     cht::{Cht, OnceBoxSlice, Value},
@@ -519,13 +519,19 @@ pub(crate) struct PrefixCht<
     AF: AddressFamily,
     M: Meta,
     const ROOT_SIZE: usize,
->(Cht<PrefixSet<AF, M>, ROOT_SIZE, 1>);
+> {
+    bush: Cht<PrefixSet<AF, M>, ROOT_SIZE, 1>,
+    counters: Counters,
+}
 
 impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
     PrefixCht<AF, M, ROOT_SIZE>
 {
     pub(crate) fn init() -> Self {
-        Self(<Cht<PrefixSet<AF, M>, ROOT_SIZE, 1>>::init())
+        Self {
+            bush: <Cht<PrefixSet<AF, M>, ROOT_SIZE, 1>>::init(),
+            counters: Counters::default(),
+        }
     }
 
     pub(crate) fn get_records_for_prefix(
@@ -535,7 +541,7 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
         include_withdrawn: bool,
         bmin: &RoaringBitmap,
     ) -> Option<Vec<Record<M>>> {
-        let mut prefix_set = self.0.root_for_len(prefix.len());
+        let mut prefix_set = self.bush.root_for_len(prefix.len());
         let mut level: u8 = 0;
         let backoff = Backoff::new();
 
@@ -610,8 +616,14 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
                     if mui_count.is_some() {
                         mui_is_new = false;
                         prefix_is_new = false;
+                    } else {
+                        self.counters.inc_routes_count();
                     }
 
+                    if prefix_is_new {
+                        self.counters
+                            .inc_prefixes_count(stored_prefix.prefix.len());
+                    }
                     (mui_count, retry_count)
                 }
                 // There already is a StoredPrefix with a record at this
@@ -637,7 +649,13 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
                         .record_map
                         .upsert_record(record)
                         .map_err(|_| PrefixStoreError::FatalError)?;
-                    mui_is_new = mui_count.is_none();
+
+                    // if the mui is new, we didn't overwrite an existing
+                    // route, so that's a new one!
+                    if mui_count.is_none() {
+                        mui_is_new = true;
+                        self.counters.inc_routes_count();
+                    };
 
                     if let Some(tbi) = update_path_selections {
                         stored_prefix
@@ -673,7 +691,7 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
         search_prefix_id: PrefixId<AF>,
     ) -> (&StoredPrefix<AF, M>, bool) {
         trace!("non_recursive_retrieve_prefix_mut_with_guard");
-        let mut prefix_set = self.0.root_for_len(search_prefix_id.len());
+        let mut prefix_set = self.bush.root_for_len(search_prefix_id.len());
         let mut level: u8 = 0;
 
         trace!("root prefix_set {:?}", prefix_set);
@@ -758,7 +776,7 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
             usize,
         )>,
     ) {
-        let mut prefix_set = self.0.root_for_len(id.len());
+        let mut prefix_set = self.bush.root_for_len(id.len());
         let mut parents = [None; 32];
         let mut level: u8 = 0;
         let backoff = Backoff::new();
@@ -798,7 +816,15 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
         }
     }
 
-    pub(crate) fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
+    pub(crate) fn prefixes_count(&self) -> usize {
+        self.counters.prefixes_count().iter().sum()
+    }
+
+    pub(crate) fn routes_count(&self) -> usize {
+        self.counters.nodes_count()
+    }
+
+    fn hash_prefix_id(id: PrefixId<AF>, level: u8) -> usize {
         let last_level = prev_node_size(id.len(), level);
 
         // HASHING FUNCTION
diff --git a/src/rib/starcast.rs b/src/rib/starcast.rs
index 79269b52..c843b7ff 100644
--- a/src/rib/starcast.rs
+++ b/src/rib/starcast.rs
@@ -800,11 +800,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     }
 
     /// Request the number of all prefixes in the store.
-    ///
-    /// Note that this method will actually traverse the complete
-    /// tree.
     pub fn prefixes_count(&self) -> UpsertCounters {
-        self.v4.get_prefixes_count() + self.v6.get_prefixes_count()
+        self.v4.prefixes_count() + self.v6.prefixes_count()
     }
 
     /// Request the number of all IPv4 prefixes in the store.
@@ -813,7 +810,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// number in the store, due to contention at the time of
     /// reading the value.
     pub fn prefixes_v4_count(&self) -> UpsertCounters {
-        self.v4.get_prefixes_count()
+        self.v4.prefixes_count()
     }
 
     /// Request the number of all IPv4 prefixes with the
@@ -826,7 +823,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         &self,
         len: u8,
     ) -> Result<UpsertCounters, PrefixStoreError> {
-        self.v4.get_prefixes_count_for_len(len)
+        self.v4.prefixes_count_for_len(len)
     }
 
     /// Request the number of all IPv6 prefixes in the store.
@@ -835,7 +832,7 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     /// number in the store, due to contention at the time of
     /// reading the value.
     pub fn prefixes_v6_count(&self) -> UpsertCounters {
-        self.v6.get_prefixes_count()
+        self.v6.prefixes_count()
     }
 
     /// Returns the number of all IPv6 prefixes with the
@@ -848,7 +845,22 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
         &self,
         len: u8,
     ) -> Result<UpsertCounters, PrefixStoreError> {
-        self.v6.get_prefixes_count_for_len(len)
+        self.v6.prefixes_count_for_len(len)
+    }
+
+    /// Request the number of all routes in the store.
+    pub fn routes_count(&self) -> UpsertCounters {
+        self.v4.routes_count() + self.v6.routes_count()
+    }
+
+    /// Request the number of all IPv4 routes in the store.
+    pub fn routes_count_v4(&self) -> UpsertCounters {
+        self.v4.routes_count()
+    }
+
+    /// Request the number of all IPv6 routes in the store.
+    pub fn routes_count_v6(&self) -> UpsertCounters {
+        self.v6.routes_count()
     }
 
     /// Request the number of nodes in the store.
@@ -890,8 +902,8 @@ impl<'a, M: Meta, C: Config> StarCastRib<M, C> {
     // The Store statistics.
     pub fn stats(&self) -> StoreStats {
         StoreStats {
-            v4: self.v4.counters.get_prefix_stats(),
-            v6: self.v6.counters.get_prefix_stats(),
+            v4: self.v4.counters.prefix_stats(),
+            v6: self.v6.counters.prefix_stats(),
         }
     }
 
diff --git a/src/rib/starcast_af.rs b/src/rib/starcast_af.rs
index 7a45d24e..84f826e2 100644
--- a/src/rib/starcast_af.rs
+++ b/src/rib/starcast_af.rs
@@ -214,7 +214,7 @@ impl<
     }
 
     pub fn get_nodes_count(&self) -> usize {
-        self.tree_bitmap.get_nodes_count()
+        self.tree_bitmap.nodes_count()
     }
 
     // Change the status of the record for the specified (prefix, mui)
@@ -243,8 +243,8 @@ impl<
                     prefix, mui
                 );
                 if let Some(p_tree) = self.persist_tree.as_ref() {
-                    let stored_prefixes = p_tree
-                        .get_records_with_keys_for_prefix_mui(prefix, mui);
+                    let stored_prefixes =
+                        p_tree.records_with_keys_for_prefix_mui(prefix, mui);
 
                     for rkv in stored_prefixes {
                         if let Ok(r) = rkv {
@@ -316,8 +316,8 @@ impl<
             }
             PersistStrategy::PersistOnly => {
                 if let Some(p_tree) = self.persist_tree.as_ref() {
-                    if let Ok(Some(record_b)) = p_tree
-                        .get_most_recent_record_for_prefix_mui(prefix, mui)
+                    if let Ok(Some(record_b)) =
+                        p_tree.most_recent_record_for_prefix_mui(prefix, mui)
                     {
                         let header = ValueHeader {
                             ltime,
@@ -399,20 +399,31 @@ impl<
         !self.tree_bitmap.withdrawn_muis_bmin(guard).contains(mui)
     }
 
-    pub(crate) fn get_prefixes_count(&self) -> UpsertCounters {
+    pub(crate) fn prefixes_count(&self) -> UpsertCounters {
         UpsertCounters {
-            in_memory_count: self.tree_bitmap.get_prefixes_count(),
+            in_memory_count: self.prefix_cht.prefixes_count(),
             persisted_count: self
                 .persist_tree
                 .as_ref()
-                .map_or(0, |p| p.get_prefixes_count()),
-            total_count: self.counters.get_prefixes_count().iter().sum(),
+                .map_or(0, |p| p.prefixes_count()),
+            total_count: self.counters.prefixes_count().iter().sum(),
+        }
+    }
+
+    pub(crate) fn routes_count(&self) -> UpsertCounters {
+        UpsertCounters {
+            in_memory_count: self.prefix_cht.routes_count(),
+            persisted_count: self
+                .persist_tree
+                .as_ref()
+                .map_or(0, |p| p.routes_count()),
+            total_count: self.counters.routes_count(),
         }
     }
 
     // the len check does it all.
     #[allow(clippy::indexing_slicing, clippy::unwrap_used)]
-    pub fn get_prefixes_count_for_len(
+    pub fn prefixes_count_for_len(
         &self,
         len: u8,
     ) -> Result<UpsertCounters, PrefixStoreError> {
@@ -420,11 +431,12 @@ impl<
             Ok(UpsertCounters {
                 in_memory_count: self
                     .tree_bitmap
-                    .get_prefixes_count_for_len(len)?,
-                persisted_count: self.persist_tree.as_ref().map_or(0, |p| {
-                    p.get_prefixes_count_for_len(len).unwrap()
-                }),
-                total_count: self.counters.get_prefixes_count()[len as usize],
+                    .prefixes_count_for_len(len)?,
+                persisted_count: self
+                    .persist_tree
+                    .as_ref()
+                    .map_or(0, |p| p.prefixes_count_for_len(len).unwrap()),
+                total_count: self.counters.prefixes_count()[len as usize],
             })
         } else {
             Err(PrefixStoreError::PrefixLengthInvalid)
diff --git a/src/rib/starcast_af_query.rs b/src/rib/starcast_af_query.rs
index 9be2bf17..4db2a1cb 100644
--- a/src/rib/starcast_af_query.rs
+++ b/src/rib/starcast_af_query.rs
@@ -42,7 +42,7 @@ impl<
                 self.persist_tree
                     .as_ref()
                     .and_then(|tree| {
-                        tree.get_records_for_prefix(
+                        tree.records_for_prefix(
                             prefix_id,
                             mui,
                             include_withdrawn,
@@ -70,25 +70,6 @@ impl<
                                     }
                                 })
                                 .collect::<FatalResult<Vec<_>>>()
-                            // let record: FatalResult<&ZeroCopyRecord<AF>> =
-                            //     ZeroCopyRecord::try_ref_from_bytes(
-                            //         bytes
-                            //             .as_ref()
-                            //             .map_err(|_| FatalError)
-                            //             .unwrap(),
-                            //     )
-                            //     .map_err(|_| FatalError);
-                            // Record::<M> {
-                            //     multi_uniq_id: record.multi_uniq_id,
-                            //     ltime: record.ltime,
-                            //     status: record.status,
-                            //     meta: <Vec<u8>>::from(
-                            //         record.meta.as_ref(),
-                            //     )
-                            //     .into(),
-                            // }
-                            // })
-                            // .collect::<Vec<_>>()
                         })
                     })
                     .transpose()
@@ -115,7 +96,7 @@ impl<
             None
         };
 
-        let prefix_meta = self
+        let records = self
             .get_value(prefix_id, mui, include_withdrawn, guard)?
             .unwrap_or_default();
 
@@ -130,12 +111,7 @@ impl<
 
         Ok(QueryResult {
             prefix,
-            records: prefix_meta,
-            // prefix.map(|_pfx| {
-            //     self.get_value(prefix_id, mui, include_withdrawn, guard)?
-            //         .unwrap_or_default()
-            // })
-            // .unwrap_or(vec![]),
+            records,
             match_type: MatchType::EmptyMatch,
             less_specifics: None,
             more_specifics,
@@ -204,28 +180,6 @@ impl<
         })
         .into_iter()
         .flatten()
-        // .chain(
-        //     (if mui.is_some_and(|m| {
-        //         self.config.persist_strategy == PersistStrategy::WriteAhead
-        //             || (!include_withdrawn && self.mui_is_withdrawn(m, guard))
-        //     }) {
-        //         None
-        //     } else {
-        //         let global_withdrawn_bmin =
-        //             self.in_memory_tree.withdrawn_muis_bmin(guard);
-        //         self.persist_tree.as_ref().map(|persist_tree| {
-        //             persist_tree.more_specific_prefix_iter_from(
-        //                 prefix_id,
-        //                 vec![],
-        //                 mui,
-        //                 global_withdrawn_bmin,
-        //                 include_withdrawn,
-        //             )
-        //         })
-        //     })
-        //     .into_iter()
-        //     .flatten(),
-        // )
     }
 
     pub(crate) fn less_specifics_iter_from(
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 5cad25f0..5cfa6931 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -323,9 +323,6 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
             let is_last_stride = pfx.len() <= stride_end;
             let stride_start = stride_end - STRIDE_SIZE;
 
-            // this counts the number of retry_count for this loop only,
-            // but ultimately we will return the accumulated count of all
-            // retry_count from this macro.
             let node_result = {
                 let local_retry_count = 0;
                 // retrieve_node_mut updates the bitmap index if
@@ -341,8 +338,6 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> TreeBitMap<AF, ROOT_SIZE> {
                             pfx.bits(),
                             stride_start,
                         ),
-                        // the length of THIS stride
-                        // stride,
                         is_last_stride,
                     ) {
                         (NewNodeOrIndex::NewNode(n), retry_count) => {
@@ -448,7 +443,7 @@ Giving up this node. This shouldn't happen!",
 
     pub fn prefix_exists(&self, prefix_id: PrefixId<AF>) -> bool {
         trace!("pe exists {:?}?", prefix_id);
-        let (node_id, bs) = self.get_node_id_for_prefix(&prefix_id);
+        let (node_id, bs) = self.node_id_for_prefix(&prefix_id);
 
         match self.retrieve_node(node_id) {
             Some(n) => {
@@ -465,7 +460,7 @@ Giving up this node. This shouldn't happen!",
         mui: u32,
     ) -> bool {
         trace!("pe exists {:?}?", prefix_id);
-        let (node_id, bs) = self.get_node_id_for_prefix(&prefix_id);
+        let (node_id, bs) = self.node_id_for_prefix(&prefix_id);
 
         match self.retrieve_node_for_mui(node_id, mui) {
             Some(n) => {
@@ -937,35 +932,30 @@ Giving up this node. This shouldn't happen!",
         )
     }
 
-    pub fn get_nodes_count(&self) -> usize {
-        self.counters.get_nodes_count()
+    pub fn nodes_count(&self) -> usize {
+        self.counters.nodes_count()
     }
 
-    pub fn get_prefixes_count(&self) -> usize {
-        self.counters.get_prefixes_count().iter().sum()
+    pub fn prefixes_count(&self) -> usize {
+        self.counters.prefixes_count().iter().sum()
     }
 
     // len checking does it all
     #[allow(clippy::indexing_slicing)]
-    pub fn get_prefixes_count_for_len(
+    pub fn prefixes_count_for_len(
         &self,
         len: u8,
     ) -> Result<usize, PrefixStoreError> {
         if len <= AF::BITS {
-            Ok(self.counters.get_prefixes_count()[len as usize])
+            Ok(self.counters.prefixes_count()[len as usize])
         } else {
             Err(PrefixStoreError::PrefixLengthInvalid)
         }
     }
 
-    // Stride related methods
-    // pub fn get_stride_sizes(&self) -> &[u8] {
-    //     self.node_buckets.get_stride_sizes()
-    // }
-
     // Calculates the id of the node that COULD host a prefix in its
     // ptrbitarr.
-    pub(crate) fn get_node_id_for_prefix(
+    pub(crate) fn node_id_for_prefix(
         &self,
         prefix: &PrefixId<AF>,
     ) -> (NodeId<AF>, BitSpan) {
@@ -975,7 +965,6 @@ Giving up this node. This shouldn't happen!",
             prefix.len()
         );
         let mut acc = 0;
-        // for i in self.get_stride_sizes() {
         loop {
             acc += STRIDE_SIZE;
             if acc >= prefix.len() {
@@ -1110,19 +1099,10 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> std::fmt::Display
     for TreeBitMap<AF, ROOT_SIZE>
 {
     fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(_f, "{} prefixes created", self.get_prefixes_count())?;
-        writeln!(_f, "{} nodes created", self.get_nodes_count())?;
+        writeln!(_f, "{} prefixes created", self.prefixes_count())?;
+        writeln!(_f, "{} nodes created", self.nodes_count())?;
         writeln!(_f)?;
 
-        // writeln!(
-        //     _f,
-        //     "stride division {:?}",
-        //     self.get_stride_sizes()
-        //         .iter()
-        //         .map_while(|s| if s > &0 { Some(*s) } else { None })
-        //         .collect::<Vec<_>>()
-        // )?;
-
         writeln!(
             _f,
             "level\t[{}] prefixes-occupied/max-prefixes percentage_occupied",
@@ -1131,20 +1111,10 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> std::fmt::Display
 
         let bars = ["▏", "▎", "▍", "▌", "▋", "▊", "▉"];
         const SCALE: u32 = 5500;
-
-        // trace!(
-        //     "stride_sizes {:?}",
-        //     self.get_stride_sizes()
-        //         .iter()
-        //         .map_while(|s| if s > &0 { Some(*s) } else { None })
-        //         .enumerate()
-        //         .collect::<Vec<(usize, u8)>>()
-        // );
-
         for crate::stats::CreatedNodes {
             depth_level: len,
             count: prefix_count,
-        } in self.counters.get_prefix_stats()
+        } in self.counters.prefix_stats()
         {
             let max_pfx = u128::overflowing_pow(2, len as u32);
             let n = (prefix_count as u32 / SCALE) as usize;
diff --git a/src/tree_bitmap/tree_bitmap_iterators.rs b/src/tree_bitmap/tree_bitmap_iterators.rs
index b70fa529..56af8eaa 100644
--- a/src/tree_bitmap/tree_bitmap_iterators.rs
+++ b/src/tree_bitmap/tree_bitmap_iterators.rs
@@ -267,7 +267,7 @@ impl<'a, AF: AddressFamily, const ROOT_SIZE: usize>
         } else {
             // calculate the node start_prefix_id lives in.
             let (start_node_id, start_bs) =
-                self.get_node_id_for_prefix(&start_prefix_id);
+                self.node_id_for_prefix(&start_prefix_id);
             trace!("start node {}", start_node_id);
             trace!(
                 "start prefix id {:032b} (len {})",
diff --git a/src/types/stats.rs b/src/types/stats.rs
index e0e924c1..07a5304b 100644
--- a/src/types/stats.rs
+++ b/src/types/stats.rs
@@ -110,19 +110,21 @@ impl Debug for CreatedNodes {
 }
 
 //------------ Counters -----------------------------------------------------
+//
+// This is the struct that's part of the data structure of each tree type.
 
 #[derive(Debug)]
 pub(crate) struct Counters {
-    // number of created nodes in the in-mem tree
+    // number of created nodes in the TreeBitMap. Set to 0 for other trees.
     nodes: AtomicUsize,
-    // number of unique prefixes in the store
+    // number of unique prefixes in the tree
     prefixes: [AtomicUsize; 129],
-    // number of unique (prefix, mui) values inserted in the in-mem tree
+    // number of unique (prefix, mui) values inserted in the tree.
     routes: AtomicUsize,
 }
 
 impl Counters {
-    pub fn get_nodes_count(&self) -> usize {
+    pub fn nodes_count(&self) -> usize {
         self.nodes.load(Ordering::Relaxed)
     }
 
@@ -130,7 +132,7 @@ impl Counters {
         self.nodes.fetch_add(1, Ordering::Relaxed);
     }
 
-    pub fn get_prefixes_count(&self) -> Vec<usize> {
+    pub fn prefixes_count(&self) -> Vec<usize> {
         self.prefixes
             .iter()
             .map(|pc| pc.load(Ordering::Relaxed))
@@ -149,7 +151,7 @@ impl Counters {
         }
     }
 
-    pub fn get_prefix_stats(&self) -> Vec<CreatedNodes> {
+    pub fn prefix_stats(&self) -> Vec<CreatedNodes> {
         self.prefixes
             .iter()
             .enumerate()
@@ -167,6 +169,10 @@ impl Counters {
             .collect()
     }
 
+    pub fn routes_count(&self) -> usize {
+        self.routes.load(Ordering::Relaxed)
+    }
+
     pub fn inc_routes_count(&self) {
         self.routes.fetch_add(1, Ordering::Relaxed);
     }
@@ -189,6 +195,11 @@ impl Default for Counters {
     }
 }
 
+//------------ UpsertCounters ------------------------------------------------
+//
+// The Counters struct holds atomic values, so this struct exists to return a
+// set of counters from the RIB to users.
+
 #[derive(Debug)]
 pub struct UpsertCounters {
     // number of unique inserted prefixes|routes in the in-mem tree
@@ -215,9 +226,9 @@ impl UpsertCounters {
 
 impl std::fmt::Display for UpsertCounters {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "Unique Items in-memory:\t{}", self.in_memory_count)?;
-        write!(f, "Unique persisted Items:\t{}", self.persisted_count)?;
-        write!(f, "Total inserted Items:\t{}", self.total_count)
+        writeln!(f, "Unique Items in-memory:\t{}", self.in_memory_count)?;
+        writeln!(f, "Unique persisted Items:\t{}", self.persisted_count)?;
+        writeln!(f, "Total inserted Items:\t{}", self.total_count)
     }
 }
 
@@ -241,7 +252,7 @@ impl std::ops::Add for UpsertCounters {
     }
 }
 
-//------------ StoreStats ----------------------------------------------
+//------------ StoreStats ----------------------------------------------------
 
 #[derive(Debug)]
 pub struct StoreStats {

From dba5c6c722154c0039217f6ce1724dbc798f4274 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 16 Apr 2025 16:00:24 +0200
Subject: [PATCH 143/147] less-specific iter bugfix (skipping over prefix
 length)

---
 src/tree_bitmap/tree_bitmap_iterators.rs |   1 -
 tests/less-specifics.rs                  | 133 +++++++++++++++++++++++
 2 files changed, 133 insertions(+), 1 deletion(-)
 create mode 100644 tests/less-specifics.rs

diff --git a/src/tree_bitmap/tree_bitmap_iterators.rs b/src/tree_bitmap/tree_bitmap_iterators.rs
index 56af8eaa..89607992 100644
--- a/src/tree_bitmap/tree_bitmap_iterators.rs
+++ b/src/tree_bitmap/tree_bitmap_iterators.rs
@@ -236,7 +236,6 @@ impl<AF: AddressFamily, const ROOT_SIZE: usize> Iterator
 
             let lvl_pfx = self.prefix.truncate_to_len(self.cur_level);
             if self.tree.prefix_exists(lvl_pfx) {
-                self.cur_level = self.cur_level.saturating_sub(1);
                 return Some(lvl_pfx);
             }
 
diff --git a/tests/less-specifics.rs b/tests/less-specifics.rs
new file mode 100644
index 00000000..b783baa9
--- /dev/null
+++ b/tests/less-specifics.rs
@@ -0,0 +1,133 @@
+// type Prefix4<'a> = Prefix<u32, PrefixAs>;
+use inetnum::addr::Prefix;
+use rotonda_store::{
+    epoch,
+    prefix_record::{Record, RouteStatus},
+    rib::{config::Config, StarCastRib},
+    test_types::PrefixAs,
+};
+
+use std::error::Error;
+
+mod common {
+    use std::io::Write;
+
+    pub fn init() {
+        let _ = env_logger::builder()
+            .format(|buf, record| writeln!(buf, "{}", record.args()))
+            .is_test(true)
+            .try_init();
+    }
+}
+
+rotonda_store::all_strategies![
+    test_ms_1;
+    test_less_specifics;
+    PrefixAs
+];
+
+fn test_less_specifics<C: Config>(
+    tree_bitmap: StarCastRib<PrefixAs, C>,
+) -> Result<(), Box<dyn Error>> {
+    crate::common::init();
+
+    let pfxs = [
+        Prefix::new(std::net::Ipv4Addr::new(57, 86, 0, 0).into(), 16)?,
+        Prefix::new(std::net::Ipv4Addr::new(57, 86, 0, 0).into(), 15)?,
+        Prefix::new(std::net::Ipv4Addr::new(57, 84, 0, 0).into(), 14)?,
+    ];
+    for pfx in pfxs.iter() {
+        tree_bitmap.insert(
+            pfx,
+            Record::new(
+                0,
+                0,
+                RouteStatus::Active,
+                PrefixAs::new_from_u32(666),
+            ),
+            None,
+        )?;
+    }
+    println!("------ end of inserts\n");
+
+    let guard = &epoch::pin();
+    for (i, spfx) in &[
+        (
+            0,
+            (
+                &Prefix::new(
+                    std::net::Ipv4Addr::new(57, 86, 0, 0).into(),
+                    17,
+                ),
+                None,
+                // These are the indexes to pfxs.2 vec.
+                // These are all supposed to show up in the result.
+                vec![0, 1, 2],
+            ),
+        ),
+        (
+            0,
+            (
+                &Prefix::new(
+                    std::net::Ipv4Addr::new(57, 86, 0, 0).into(),
+                    16,
+                ),
+                None,
+                vec![1, 2],
+            ),
+        ),
+        (
+            0,
+            (
+                &Prefix::new(
+                    std::net::Ipv4Addr::new(57, 86, 0, 0).into(),
+                    15,
+                ),
+                None,
+                vec![2],
+            ),
+        ),
+        (
+            0,
+            (
+                &Prefix::new(
+                    std::net::Ipv4Addr::new(57, 84, 0, 0).into(),
+                    14,
+                ),
+                None,
+                vec![],
+            ),
+        ),
+    ] {
+        println!("round {}", i);
+        println!("search for: {}", (*spfx.0)?);
+        println!("search prefix: {}", spfx.0.unwrap());
+
+        let less_iter = tree_bitmap.less_specifics_iter_from(
+            &spfx.0.unwrap(),
+            spfx.1,
+            true,
+            guard,
+        );
+
+        for (i, p) in less_iter.enumerate() {
+            let p = p.unwrap();
+            println!("less_iter {} i {}", p, i);
+            assert_eq!(p.prefix, pfxs[spfx.2[i]])
+        }
+
+        println!("--");
+        println!("all prefixes");
+
+        for (i, p) in tree_bitmap
+            .prefixes_iter_v4(guard)
+            .enumerate()
+            .map(|(i, p)| (i, p.as_ref().unwrap().prefix))
+        {
+            println!("ls {}: {}", i, p);
+        }
+
+        println!("-----------");
+    }
+    Ok(())
+}

From 58d6afd196f760236f7680e18d78d3bcb864554a Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 16 Apr 2025 16:01:13 +0200
Subject: [PATCH 144/147] small formatiing changes

---
 examples/full_table_multiple_trees_json.rs |  2 +-
 examples/multi_thread_4.rs                 |  2 +-
 examples/multi_thread_multi_prefix.rs      |  2 +-
 src/bin/cli.rs                             | 10 +++++-----
 src/types/bit_span.rs                      |  1 -
 src/types/stats.rs                         | 14 +++++++-------
 6 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/examples/full_table_multiple_trees_json.rs b/examples/full_table_multiple_trees_json.rs
index 6660e6db..3c52ff0c 100644
--- a/examples/full_table_multiple_trees_json.rs
+++ b/examples/full_table_multiple_trees_json.rs
@@ -134,7 +134,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             println!("\"inserts_num\": {},", inserts_num);
             println!("\"insert_duration_nanos\": {},", dur_insert_nanos);
             println!(
-                "\"global_prefix_vec_size\": {},",
+                "\"global_prefix_vec_size\": {:?},",
                 tree_bitmap.prefixes_count()
             );
             println!(
diff --git a/examples/multi_thread_4.rs b/examples/multi_thread_4.rs
index 10e62cd2..459cc3ed 100644
--- a/examples/multi_thread_4.rs
+++ b/examples/multi_thread_4.rs
@@ -103,7 +103,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
                         if x % 1_000_000 == 0 {
                             println!(
-                                "{:?} {} (prefixes count: {},
+                                "{:?} {} (prefixes count: {:?},
                                 nodes count: {}",
                                 std::thread::current().name(),
                                 x,
diff --git a/examples/multi_thread_multi_prefix.rs b/examples/multi_thread_multi_prefix.rs
index b9d31110..b38be9f6 100644
--- a/examples/multi_thread_multi_prefix.rs
+++ b/examples/multi_thread_multi_prefix.rs
@@ -122,7 +122,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             "increased pfx to {}",
             pfx_arc.clone().load(std::sync::atomic::Ordering::Relaxed)
         );
-        println!("prefix count: {}", tree_bitmap.prefixes_count());
+        println!("prefix count: {:?}", tree_bitmap.prefixes_count());
 
         threads.clone().for_each(|t| {
             t.thread().unpark();
diff --git a/src/bin/cli.rs b/src/bin/cli.rs
index a376826e..57f7faf2 100644
--- a/src/bin/cli.rs
+++ b/src/bin/cli.rs
@@ -118,7 +118,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                             );
                                         });
                                     println!(
-                                        "ipv4 prefixes :\t{}",
+                                        "ipv4 prefixes :\t{:?}",
                                         tree_bitmap.prefixes_v4_count()
                                     );
                                 }
@@ -133,17 +133,17 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                             );
                                         });
                                     println!(
-                                        "ipv6 prefixes :\t{}",
+                                        "ipv6 prefixes :\t{:?}",
                                         tree_bitmap.prefixes_v6_count()
                                     );
                                 }
                                 _ => {
                                     println!(
-                                        "ipv4 prefixes :\t{}",
+                                        "ipv4 prefixes :\t{:?}",
                                         tree_bitmap.prefixes_v4_count()
                                     );
                                     println!(
-                                        "ipv6 prefixes :\t{}",
+                                        "ipv6 prefixes :\t{:?}",
                                         tree_bitmap.prefixes_v6_count()
                                     );
                                     tree_bitmap
@@ -156,7 +156,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                                             );
                                         });
                                     println!(
-                                        "total prefixes :\t{}",
+                                        "total prefixes :\t{:?}",
                                         tree_bitmap.prefixes_count()
                                     );
                                 }
diff --git a/src/types/bit_span.rs b/src/types/bit_span.rs
index e42fbd70..aa8433ba 100644
--- a/src/types/bit_span.rs
+++ b/src/types/bit_span.rs
@@ -43,7 +43,6 @@ impl BitSpan {
     }
 
     pub(crate) fn check(&self) -> bool {
-        println!("check bit span: {:?}", self);
         if self.len == 0 && self.bits == 0 {
             return true;
         };
diff --git a/src/types/stats.rs b/src/types/stats.rs
index 07a5304b..7b1b6291 100644
--- a/src/types/stats.rs
+++ b/src/types/stats.rs
@@ -224,13 +224,13 @@ impl UpsertCounters {
     }
 }
 
-impl std::fmt::Display for UpsertCounters {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(f, "Unique Items in-memory:\t{}", self.in_memory_count)?;
-        writeln!(f, "Unique persisted Items:\t{}", self.persisted_count)?;
-        writeln!(f, "Total inserted Items:\t{}", self.total_count)
-    }
-}
+// impl std::fmt::Display for UpsertCounters {
+//     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+//         writeln!(f, "Unique Items in-memory:\t{}", self.in_memory_count)?;
+//         writeln!(f, "Unique persisted Items:\t{}", self.persisted_count)?;
+//         writeln!(f, "Total inserted Items:\t{}", self.total_count)
+//     }
+// }
 
 impl std::ops::AddAssign for UpsertCounters {
     fn add_assign(&mut self, rhs: Self) {

From e7e67dfaeca02f27be2b1e9f631a4d35508d5ff8 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 23 Apr 2025 10:08:38 +0200
Subject: [PATCH 145/147] saturating_sub in prev_node_size; debug_asserts for
 other unchecked operations

---
 src/cht/mod.rs         | 20 ++++++++++++++++++--
 src/tree_bitmap/mod.rs | 12 ++++++++++++
 src/types/af.rs        | 37 +++++++++++++++++++++++++++++++++++++
 3 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/cht/mod.rs b/src/cht/mod.rs
index ea32b46f..22293dae 100644
--- a/src/cht/mod.rs
+++ b/src/cht/mod.rs
@@ -93,12 +93,20 @@ impl<V: Value, const ROOT_SIZE: usize, const STRIDES_PER_BUCKET: usize>
 // 10   00   04 4
 // ...
 pub fn nodeset_size(len: u8, lvl: u8) -> u8 {
+    // The multiplication here will only ever overflow if the (len, lvl) input
+    // is out of bounds for IPv4 or IPv6 prefixes. Therefore we are including
+    // a debug_assert here to panic in debug mode if this happens.  In release
+    // compiles this may NOT be noticable, because the undefined behaviour
+    // is most probably the desired behaviour (saturating). But it's UB for a
+    // reason, so we should not rely on it, and verify that we are not hitting
+    // that behaviour.
+    debug_assert!(4_u8.checked_mul(lvl + 1).is_some());
     4_u8.saturating_sub((4 * (lvl + 1)).saturating_sub(len))
 }
 
 // The value of the set of the parent of this one. used to calculate the shift
 // offset in the hash for the CHT, so this is basically the `nodeset_size`
-// shifted on (len, lvl) combination downwards.
+// shifted one (len, lvl) combination downwards.
 //
 // len lvl prev
 // 00  00  00
@@ -127,5 +135,13 @@ pub fn nodeset_size(len: u8, lvl: u8) -> u8 {
 // 09  02  08
 // 09  03  09
 pub fn prev_node_size(len: u8, lvl: u8) -> u8 {
-    (lvl * 4) - lvl.saturating_sub(len >> 2) * ((lvl * 4) - len)
+    // The multiplication here will only ever overflow if the (len, lvl) input
+    // is out of bounds for IPv4 or IPv6 prefixes. Therefore we are including
+    // a debug_assert here to panic in debug mode if this happens.  In release
+    // compiles this may NOT be noticable, because the undefined behaviour
+    // is most probably the desired behaviour (saturating). But it's UB for a
+    // reason, so we should not rely on it, and verify that we are not hitting
+    // that behaviour.
+    debug_assert!(4_u8.checked_mul(lvl).is_some());
+    (lvl * 4) - lvl.saturating_sub(len >> 2) * ((lvl * 4).saturating_sub(len))
 }
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 5cfa6931..0f35d26d 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -1056,6 +1056,18 @@ Giving up this node. This shouldn't happen!",
         // );
         // HASHING FUNCTION
         let size = nodeset_size(id.len(), level);
+
+        // shifting left and right here should never overflow for inputs
+        // (NodeId, level) that are valid for IPv4 and IPv6. In release
+        // compiles this may NOT be noticable, because the undefined behaviour
+        // is most probably the desired behaviour (saturating). But it's UB
+        // for a reason, so we should not rely on it, and verify that we are
+        // not hitting that behaviour.
+        debug_assert!(id.bits().checked_shl(last_level as u32).is_some());
+        debug_assert!((id.bits() << AF::from_u8(last_level))
+            .checked_shr(u32::from((<AF>::BITS - size) % <AF>::BITS))
+            .is_some());
+
         ((id.bits() << AF::from_u8(last_level))
             >> AF::from_u8((<AF>::BITS - size) % <AF>::BITS))
         .dangerously_truncate_to_u32() as usize
diff --git a/src/types/af.rs b/src/types/af.rs
index b1a8b991..0bdd90bc 100644
--- a/src/types/af.rs
+++ b/src/types/af.rs
@@ -76,6 +76,11 @@ pub trait AddressFamily:
     // in debug mode. A failed check will simply retutrn zero. Used in
     // finding node_ids (always zero for 0/0).
     fn checked_shr_or_zero(self, rhs: u32) -> Self;
+    fn checked_shl_or_zero(self, rhs: u32) -> Self;
+
+    // These checked shifts are for use in debug asserts only.
+    fn checked_shr(self, rhs: u32) -> Option<Self::Inner>;
+    fn checked_shl(self, rhs: u32) -> Option<Self::Inner>;
 }
 
 //-------------- Ipv4 Type --------------------------------------------------
@@ -141,6 +146,22 @@ impl AddressFamily for IPv4 {
         }
         self >> U32::<NetworkEndian>::from(rhs)
     }
+
+    fn checked_shl_or_zero(self, rhs: u32) -> Self {
+        trace!("CHECKED_SHL_OR_ZERO {} >> {}", u32::from(self), rhs);
+        if rhs == 0 || rhs >= 32 {
+            return 0.into();
+        }
+        self << U32::<NetworkEndian>::from(rhs)
+    }
+
+    fn checked_shr(self, rhs: u32) -> Option<u32> {
+        u32::from(self).checked_shr(rhs)
+    }
+
+    fn checked_shl(self, rhs: u32) -> Option<u32> {
+        u32::from(self).checked_shl(rhs)
+    }
 }
 
 //-------------- Ipv6 Type --------------------------------------------------
@@ -201,6 +222,22 @@ impl AddressFamily for IPv6 {
         self >> U128::from(rhs as u128)
     }
 
+    fn checked_shl_or_zero(self, rhs: u32) -> Self {
+        if rhs >= 128 {
+            return U128::from(0);
+        };
+
+        self << U128::from(rhs as u128)
+    }
+
+    fn checked_shr(self, rhs: u32) -> Option<u128> {
+        u128::from(self).checked_shr(rhs)
+    }
+
+    fn checked_shl(self, rhs: u32) -> Option<u128> {
+        u128::from(self).checked_shl(rhs)
+    }
+
     fn from_u8(value: u8) -> Self {
         IPv6::from([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, value])
     }

From 22893518f26034ff3004a1872a288f622aa419de Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 23 Apr 2025 14:11:31 +0200
Subject: [PATCH 146/147] add tests for valid ranges in hash functions

---
 src/cht/mod.rs         | 33 +++++++++++++++++++++++++++++++++
 src/prefix_cht/cht.rs  | 36 ++++++++++++++++++++++++++++++++++++
 src/tree_bitmap/mod.rs | 30 ++++++++++++++++--------------
 3 files changed, 85 insertions(+), 14 deletions(-)

diff --git a/src/cht/mod.rs b/src/cht/mod.rs
index 22293dae..69e523ac 100644
--- a/src/cht/mod.rs
+++ b/src/cht/mod.rs
@@ -104,6 +104,28 @@ pub fn nodeset_size(len: u8, lvl: u8) -> u8 {
     4_u8.saturating_sub((4 * (lvl + 1)).saturating_sub(len))
 }
 
+// This test tests both that the outocome of our optimized nodeset_size
+// function is the same as our 'naive' approach, and that we do not rely on
+// undefined behaviour because of overflowing multiplication or addition.
+#[test]
+fn test_nodeset_size_valid_range() {
+    for len in 0..128 {
+        for lvl in 0..(len / 4) {
+            let res = 4 * (lvl + 1);
+            let nss = if res < len {
+                4
+            } else if res >= len + 4 {
+                0
+            } else if len % 4 == 0 {
+                4
+            } else {
+                len % 4
+            };
+            debug_assert_eq!(nss, nodeset_size(len, lvl));
+        }
+    }
+}
+
 // The value of the set of the parent of this one. used to calculate the shift
 // offset in the hash for the CHT, so this is basically the `nodeset_size`
 // shifted one (len, lvl) combination downwards.
@@ -145,3 +167,14 @@ pub fn prev_node_size(len: u8, lvl: u8) -> u8 {
     debug_assert!(4_u8.checked_mul(lvl).is_some());
     (lvl * 4) - lvl.saturating_sub(len >> 2) * ((lvl * 4).saturating_sub(len))
 }
+
+// In this test we're only testing to no rely on undefined behaviour for all
+// inputs in the valid range
+#[test]
+fn test_prev_node_size_valid_range() {
+    for len in 0..128 {
+        for lvl in 0..(len / 4) {
+            prev_node_size(len, lvl);
+        }
+    }
+}
diff --git a/src/prefix_cht/cht.rs b/src/prefix_cht/cht.rs
index 58d855bc..13e34080 100644
--- a/src/prefix_cht/cht.rs
+++ b/src/prefix_cht/cht.rs
@@ -13,7 +13,11 @@ use crate::cht::{nodeset_size, prev_node_size};
 use crate::errors::{FatalError, FatalResult};
 use crate::prefix_record::Meta;
 use crate::stats::{Counters, UpsertReport};
+#[cfg(test)]
+use crate::test_types::NoMeta;
 use crate::types::RouteStatus;
+#[cfg(test)]
+use crate::IPv6;
 use crate::{
     cht::{Cht, OnceBoxSlice, Value},
     types::{
@@ -829,8 +833,40 @@ impl<AF: AddressFamily, M: Meta, const ROOT_SIZE: usize>
 
         // HASHING FUNCTION
         let size = nodeset_size(id.len(), level);
+
+        // shifting left and right here should never overflow for inputs
+        // (NodeId, level) that are valid for IPv4 and IPv6. In release
+        // compiles this may NOT be noticable, because the undefined behaviour
+        // is most probably the desired behaviour (saturating). But it's UB
+        // for a reason, so we should not rely on it, and verify that we are
+        // not hitting that behaviour.
+        debug_assert!(id.bits().checked_shl(last_level as u32).is_some());
+        debug_assert!((id.bits() << AF::from_u32(last_level as u32))
+            .checked_shr(u32::from((<AF>::BITS - size) % <AF>::BITS))
+            .is_some());
+
         ((id.bits() << AF::from_u32(last_level as u32))
             >> AF::from_u8((<AF>::BITS - size) % <AF>::BITS))
         .dangerously_truncate_to_u32() as usize
     }
+
+    #[allow(clippy::unwrap_used)]
+    #[cfg(test)]
+    fn test_valid_range() {
+        let ip_addr = std::net::IpAddr::V6(
+            "0::".parse::<std::net::Ipv6Addr>().unwrap(),
+        );
+        for len in 0..128 {
+            for lvl in 0..(len / 4) {
+                let p_id =
+                    PrefixId::<AF>::from(Prefix::new(ip_addr, len).unwrap());
+                Self::hash_prefix_id(p_id, lvl);
+            }
+        }
+    }
+}
+
+#[test]
+fn test_hashing_prefix_id_valid_range() {
+    PrefixCht::<IPv6, NoMeta, 129>::test_valid_range()
 }
diff --git a/src/tree_bitmap/mod.rs b/src/tree_bitmap/mod.rs
index 0f35d26d..7dea5c95 100644
--- a/src/tree_bitmap/mod.rs
+++ b/src/tree_bitmap/mod.rs
@@ -221,6 +221,8 @@ use crate::errors::{FatalError, FatalResult};
 use crate::rib::STRIDE_SIZE;
 use crate::stats::Counters;
 use crate::types::{BitSpan, PrefixId};
+#[cfg(test)]
+use crate::IPv6;
 use crossbeam_epoch::{Atomic, Guard, Owned, Shared};
 use log::{debug, error, log_enabled, trace};
 use node_cht::{NodeCht, NodeSet, StoredNode};
@@ -1040,21 +1042,7 @@ Giving up this node. This shouldn't happen!",
     // uses the hash function with the level incremented.
 
     pub(crate) fn hash_node_id(id: NodeId<AF>, level: u8) -> usize {
-        // And, this is all of our hashing function.
-        // let last_level = if level > 0 {
-        //     bits_for_len(id.len(), level - 1)
-        // } else {
-        //     0
-        // };
         let last_level = prev_node_size(id.len(), level);
-        // trace!("bits division {}", this_level);
-        // trace!(
-        //     "calculated index ({} << {}) >> {}",
-        //     id.get_id().0,
-        //     last_level,
-        //     ((<AF>::BITS - (this_level - last_level)) % <AF>::BITS) as usize
-        // );
-        // HASHING FUNCTION
         let size = nodeset_size(id.len(), level);
 
         // shifting left and right here should never overflow for inputs
@@ -1072,8 +1060,22 @@ Giving up this node. This shouldn't happen!",
             >> AF::from_u8((<AF>::BITS - size) % <AF>::BITS))
         .dangerously_truncate_to_u32() as usize
     }
+
+    #[cfg(test)]
+    fn test_valid_range() {
+        for len in 0..128 {
+            for lvl in 0..(len / 4) {
+                let n_id = NodeId::<AF>::from((AF::from_u32(0), len));
+                Self::hash_node_id(n_id, lvl);
+            }
+        }
+    }
 }
 
+#[test]
+fn test_hashing_node_id_valid_range() {
+    TreeBitMap::<IPv6, 129>::test_valid_range()
+}
 // Partition for stride 4
 //
 // ptr bits never happen in the first half of the bitmap for the stride-size. Consequently the ptrbitarr can be an integer type

From 54cd24194c47f9bbe084ed1267afe3fb00d7c256 Mon Sep 17 00:00:00 2001
From: "Density 21.5" <jasper@plainspace.com>
Date: Wed, 23 Apr 2025 16:46:11 +0200
Subject: [PATCH 147/147] use released routecore

---
 Cargo.toml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 7afb0a0b..e3080082 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,10 +27,10 @@ parking_lot_core = "0.9.10"
 inetnum   = "0.1"
 log       = "^0.4"
 roaring 	= "0.10.3"
-routecore       = { git = "https://github.com/nlnetlabs/routecore", branch = "dev", version = "0.5.2-dev", features = ["bgp", "bmp", "fsm", "serde", "mrt"] }
+routecore       = { version = "0.5.2", features = ["bgp", "bmp", "fsm", "serde", "mrt"] }
 ansi_term   = { version = "0.12", optional = true }
 csv         = { version = "1", optional = true }
-rustyline   = { version = "13", optional = true }
+rustyline   = { version = "15", optional = true }
 clap        = { version = "4.4", optional = true, features = ["derive"] }
 rayon = { version = "1.10", optional = true }
 memmap2 = { version = "0.9", optional = true }
@@ -44,7 +44,7 @@ zerocopy = { version = "0.8.17", features = ["derive"] }
 
 [dev-dependencies]
 csv         = { version = "1" }
-env_logger  = { version = "0.10" }
+env_logger  = { version = "0.11" }
 
 [features]
 cli = ["ansi_term", "rustyline", "csv"]