Repair: remove calls to new_from_data (#7173)

* remove most calls to new_from_data from repair: test_verify_shred_response test_get_best_repair_shreds
* add helpers in shredder.rs

Author: alexpyattaev

core/src/repair/outstanding_requests.rs

@@ -85,10 +85,8 @@ pub struct RequestStatus<T> {
 #[cfg(test)]
 pub(crate) mod tests {
     use {
-        super::*,
-        crate::repair::serve_repair::ShredRepairType,
-        solana_ledger::shred::{Shred, ShredFlags},
-        solana_time_utils::timestamp,
+        super::*, crate::repair::serve_repair::ShredRepairType, solana_keypair::Keypair,
+        solana_ledger::shred::Shredder, solana_time_utils::timestamp,
     };
 
     #[test]
@@ -109,7 +107,8 @@ pub(crate) mod tests {
         let repair_type = ShredRepairType::Orphan(9);
         let mut outstanding_requests = OutstandingRequests::default();
         let nonce = outstanding_requests.add_request(repair_type, timestamp());
-        let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0);
+        let keypair = Keypair::new();
+        let shred = Shredder::single_shred_for_tests(0, &keypair);
 
         let expire_timestamp = outstanding_requests
             .requests
@@ -128,8 +127,8 @@ pub(crate) mod tests {
         let repair_type = ShredRepairType::Orphan(9);
         let mut outstanding_requests = OutstandingRequests::default();
         let nonce = outstanding_requests.add_request(repair_type, timestamp());
-
-        let shred = Shred::new_from_data(0, 0, 0, &[], ShredFlags::empty(), 0, 0, 0);
+        let keypair = Keypair::new();
+        let shred = Shredder::single_shred_for_tests(0, &keypair);
         let mut expire_timestamp = outstanding_requests
             .requests
             .get(&nonce)

core/src/repair/repair_response.rs

@@ -48,7 +48,7 @@ mod test {
         super::*,
         solana_keypair::Keypair,
         solana_ledger::{
-            shred::{Shred, ShredFlags},
+            shred::Shredder,
             sigverify_shreds::{verify_shred_cpu, LruCache},
         },
         solana_packet::PacketFlags,
@@ -63,19 +63,9 @@ mod test {
     fn run_test_sigverify_shred_cpu_repair(slot: Slot) {
         solana_logger::setup();
         let cache = RwLock::new(LruCache::new(/*capacity:*/ 128));
-        let mut shred = Shred::new_from_data(
-            slot,
-            0xc0de,
-            0xdead,
-            &[1, 2, 3, 4],
-            ShredFlags::LAST_SHRED_IN_SLOT,
-            0,
-            0,
-            0xc0de,
-        );
-        assert_eq!(shred.slot(), slot);
         let keypair = Keypair::new();
-        shred.sign(&keypair);
+        let shred = Shredder::single_shred_for_tests(slot, &keypair);
+
         trace!("signature {}", shred.signature());
         let nonce = 9;
         let mut packet = repair_response_packet_from_bytes(

core/src/repair/repair_weighted_traversal.rs

@@ -142,9 +142,10 @@ pub mod test {
         super::*,
         crate::repair::repair_service::sleep_shred_deferment_period,
         solana_hash::Hash,
+        solana_keypair::Keypair,
         solana_ledger::{
             get_tmp_ledger_path,
-            shred::{Shred, ShredFlags},
+            shred::{ProcessShredsStats, ReedSolomonCache, Shred, Shredder},
         },
         solana_runtime::bank_utils,
         trees::tr,
@@ -281,22 +282,24 @@ pub mod test {
         repairs = vec![];
         outstanding_repairs = HashMap::new();
         slot_meta_cache = HashMap::default();
+        let keypair = Keypair::new();
+        let reed_solomon_cache = ReedSolomonCache::default();
+
         let completed_shreds: Vec<Shred> = [0, 2, 4, 6]
             .iter()
-            .map(|slot| {
-                let parent_offset = u16::from(*slot != 0);
-                let shred = Shred::new_from_data(
-                    *slot,
-                    last_shred as u32, // index
-                    parent_offset,
-                    &[0u8; 8], // data
-                    ShredFlags::LAST_SHRED_IN_SLOT,
-                    8,                 // reference_tick
-                    0,                 // version
-                    last_shred as u32, // fec_set_index
+            .flat_map(|slot| {
+                let shredder = Shredder::new(*slot, slot.saturating_sub(1), 0, 42).unwrap();
+                let (shreds, _) = shredder.entries_to_merkle_shreds_for_tests(
+                    &keypair,
+                    &[],
+                    true,
+                    Some(Hash::default()),
+                    last_shred as u32,
+                    last_shred as u32,
+                    &reed_solomon_cache,
+                    &mut ProcessShredsStats::default(),
                 );
-                assert!(shred.sanitize().is_ok());
-                shred
+                shreds
             })
             .collect();
         blockstore

core/src/repair/serve_repair.rs

@@ -1346,7 +1346,9 @@ mod tests {
             blockstore_processor::fill_blockstore_slot_with_ticks,
             genesis_utils::{create_genesis_config, GenesisConfigInfo},
             get_tmp_ledger_path_auto_delete,
-            shred::{max_ticks_per_n_shreds, Shred, ShredFlags},
+            shred::{
+                max_ticks_per_n_shreds, ProcessShredsStats, ReedSolomonCache, Shred, Shredder,
+            },
         },
         solana_perf::packet::{deserialize_from_with_limit, Packet, PacketFlags, PacketRef},
         solana_pubkey::Pubkey,
@@ -1368,19 +1370,8 @@ mod tests {
 
     #[test]
     fn test_deserialize_shred_as_ping() {
-        let data_buf = vec![7u8, 44]; // REPAIR_RESPONSE_SERIALIZED_PING_BYTES - SIZE_OF_DATA_SHRED_HEADERS
         let keypair = Keypair::new();
-        let mut shred = Shred::new_from_data(
-            123, // slot
-            456, // index
-            111, // parent_offset
-            &data_buf,
-            ShredFlags::empty(),
-            222, // reference_tick
-            333, // version
-            444, // fec_set_index
-        );
-        shred.sign(&keypair);
+        let shred = Shredder::single_shred_for_tests(123, &keypair);
         let mut pkt = Packet::default();
         shred.copy_to_packet(&mut pkt);
         pkt.meta_mut().size = REPAIR_RESPONSE_SERIALIZED_PING_BYTES;
@@ -1822,26 +1813,37 @@ mod tests {
     }
 
     #[test]
-    fn test_run_window_request() {
-        run_window_request(2, 9);
-    }
-
     /// test window requests respond with the right shred, and do not overrun
-    pub fn run_window_request(slot: Slot, nonce: Nonce) {
+    fn test_run_window_request() {
+        let slot = 2;
+        let nonce = 9;
         let recycler = PacketBatchRecycler::default();
         solana_logger::setup();
         let ledger_path = get_tmp_ledger_path_auto_delete!();
         let blockstore = Arc::new(Blockstore::open(ledger_path.path()).unwrap());
         let handler = StandardRepairHandler::new(blockstore.clone());
         let rv = handler.run_window_request(&recycler, &socketaddr_any!(), slot, 0, nonce);
         assert!(rv.is_none());
-        let shred = Shred::new_from_data(slot, 1, 1, &[], ShredFlags::empty(), 0, 2, 0);
+        let shredder = Shredder::new(slot, slot - 1, 0, 2).unwrap();
+        let keypair = Keypair::new();
+        let reed_solomon_cache = ReedSolomonCache::default();
+        let index = 1;
+        let (mut shreds, _) = shredder.entries_to_merkle_shreds_for_tests(
+            &keypair,
+            &[],
+            true,
+            Some(Hash::default()),
+            index as u32,
+            index as u32,
+            &reed_solomon_cache,
+            &mut ProcessShredsStats::default(),
+        );
+        shreds.truncate(1);
 
         blockstore
-            .insert_shreds(vec![shred], None, false)
+            .insert_shreds(shreds, None, false)
             .expect("Expect successful ledger write");
 
-        let index = 1;
         let mut rv = handler
             .run_window_request(&recycler, &socketaddr_any!(), slot, index, nonce)
             .expect("packets");
@@ -2262,7 +2264,20 @@ mod tests {
     #[test]
     fn test_verify_shred_response() {
         fn new_test_data_shred(slot: Slot, index: u32) -> Shred {
-            Shred::new_from_data(slot, index, 1, &[], ShredFlags::empty(), 0, 0, 0)
+            let shredder = Shredder::new(slot, slot.saturating_sub(1), 0, 0).unwrap();
+            let keypair = Keypair::new();
+            let reed_solomon_cache = ReedSolomonCache::default();
+            let (mut shreds, _) = shredder.entries_to_merkle_shreds_for_tests(
+                &keypair,
+                &[],
+                true,
+                Some(Hash::default()),
+                0,
+                0,
+                &reed_solomon_cache,
+                &mut ProcessShredsStats::default(),
+            );
+            shreds.remove(index as usize)
         }
         let repair = ShredRepairType::Orphan(9);
         // Ensure new options are added to this test

ledger/src/shredder.rs

@@ -194,6 +194,24 @@ impl Shredder {
             Ok(data)
         }
     }
+    /// Produce a single shred with no payload
+    /// for use in tests and such
+    #[cfg(feature = "dev-context-only-utils")]
+    pub fn single_shred_for_tests(slot: Slot, keypair: &Keypair) -> Shred {
+        let shredder = Shredder::new(slot, slot.saturating_sub(1), 0, 42).unwrap();
+        let reed_solomon_cache = ReedSolomonCache::default();
+        let (mut shreds, _) = shredder.entries_to_merkle_shreds_for_tests(
+            keypair,
+            &[],
+            true,
+            Some(Hash::default()),
+            0,
+            0,
+            &reed_solomon_cache,
+            &mut ProcessShredsStats::default(),
+        );
+        shreds.pop().unwrap()
+    }
 }
 
 impl ReedSolomonCache {