Skip to content

Commit e98209d

Browse files
jimmygchenjimmygchen
authored
Implement PeerDAS subnet decoupling (aka custody groups) (#6736)
* Implement PeerDAS subnet decoupling (aka custody groups). * Merge branch 'unstable' into decouple-subnets * Refactor feature testing for spec tests (#6737) Squashed commit of the following: commit 898d05e Merge: ffbd25e 7e0cdde Author: Jimmy Chen <[email protected]> Date: Tue Dec 24 14:41:19 2024 +1100 Merge branch 'unstable' into refactor-ef-tests-features commit ffbd25e Author: Jimmy Chen <[email protected]> Date: Tue Dec 24 14:40:38 2024 +1100 Fix `SszStatic` tests for PeerDAS: exclude eip7594 test vectors when testing Electra types. commit aa593cf Author: Jimmy Chen <[email protected]> Date: Fri Dec 20 12:08:54 2024 +1100 Refactor spec testing for features and simplify usage. * Fix build. * Add input validation and improve arithmetic handling when calculating custody groups. * Address review comments re code style consistency. * Merge branch 'unstable' into decouple-subnets # Conflicts: # beacon_node/beacon_chain/src/kzg_utils.rs # beacon_node/beacon_chain/src/observed_data_sidecars.rs # beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs # common/eth2_network_config/built_in_network_configs/chiado/config.yaml # common/eth2_network_config/built_in_network_configs/gnosis/config.yaml # common/eth2_network_config/built_in_network_configs/holesky/config.yaml # common/eth2_network_config/built_in_network_configs/mainnet/config.yaml # common/eth2_network_config/built_in_network_configs/sepolia/config.yaml # consensus/types/src/chain_spec.rs * Update consensus/types/src/chain_spec.rs Co-authored-by: Lion - dapplion <[email protected]> * Merge remote-tracking branch 'origin/unstable' into decouple-subnets * Update error handling. * Address review comment. * Merge remote-tracking branch 'origin/unstable' into decouple-subnets # Conflicts: # consensus/types/src/chain_spec.rs * Update PeerDAS spec tests to `1.5.0-beta.0` and fix failing unit tests. * Merge remote-tracking branch 'origin/unstable' into decouple-subnets # Conflicts: # beacon_node/lighthouse_network/src/peer_manager/mod.rs
1 parent dd7591f commit e98209d

39 files changed

+551
-429
lines changed

‎beacon_node/beacon_chain/src/block_verification_types.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ impl<E: EthSpec> RpcBlock<E> {
165165
let inner = if !custody_columns.is_empty() {
166166
RpcBlockInner::BlockAndCustodyColumns(
167167
block,
168-
RuntimeVariableList::new(custody_columns, spec.number_of_columns)?,
168+
RuntimeVariableList::new(custody_columns, spec.number_of_columns as usize)?,
169169
)
170170
} else {
171171
RpcBlockInner::Block(block)

‎beacon_node/beacon_chain/src/data_availability_checker.rs

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -117,21 +117,16 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
117117
spec: Arc<ChainSpec>,
118118
log: Logger,
119119
) -> Result<Self, AvailabilityCheckError> {
120-
let custody_subnet_count = if import_all_data_columns {
121-
spec.data_column_sidecar_subnet_count as usize
122-
} else {
123-
spec.custody_requirement as usize
124-
};
125-
126-
let subnet_sampling_size =
127-
std::cmp::max(custody_subnet_count, spec.samples_per_slot as usize);
128-
let sampling_column_count =
129-
subnet_sampling_size.saturating_mul(spec.data_columns_per_subnet());
120+
let custody_group_count = spec.custody_group_count(import_all_data_columns);
121+
// This should only panic if the chain spec contains invalid values.
122+
let sampling_size = spec
123+
.sampling_size(custody_group_count)
124+
.expect("should compute node sampling size from valid chain spec");
130125

131126
let inner = DataAvailabilityCheckerInner::new(
132127
OVERFLOW_LRU_CAPACITY,
133128
store,
134-
sampling_column_count,
129+
sampling_size as usize,
135130
spec.clone(),
136131
)?;
137132
Ok(Self {
@@ -148,7 +143,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
148143
}
149144

150145
pub(crate) fn is_supernode(&self) -> bool {
151-
self.get_sampling_column_count() == self.spec.number_of_columns
146+
self.get_sampling_column_count() == self.spec.number_of_columns as usize
152147
}
153148

154149
/// Checks if the block root is currenlty in the availability cache awaiting import because
@@ -433,7 +428,7 @@ impl<T: BeaconChainTypes> DataAvailabilityChecker<T> {
433428
.map(CustodyDataColumn::into_inner)
434429
.collect::<Vec<_>>();
435430
let all_data_columns =
436-
RuntimeVariableList::from_vec(all_data_columns, self.spec.number_of_columns);
431+
RuntimeVariableList::from_vec(all_data_columns, self.spec.number_of_columns as usize);
437432

438433
// verify kzg for all data columns at once
439434
if !all_data_columns.is_empty() {

‎beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -598,7 +598,7 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
598598

599599
// If we're sampling all columns, it means we must be custodying all columns.
600600
let custody_column_count = self.sampling_column_count();
601-
let total_column_count = self.spec.number_of_columns;
601+
let total_column_count = self.spec.number_of_columns as usize;
602602
let received_column_count = pending_components.verified_data_columns.len();
603603

604604
if pending_components.reconstruction_started {
@@ -607,7 +607,7 @@ impl<T: BeaconChainTypes> DataAvailabilityCheckerInner<T> {
607607
if custody_column_count != total_column_count {
608608
return ReconstructColumnsDecision::No("not required for full node");
609609
}
610-
if received_column_count == self.spec.number_of_columns {
610+
if received_column_count >= total_column_count {
611611
return ReconstructColumnsDecision::No("all columns received");
612612
}
613613
if received_column_count < total_column_count / 2 {

‎beacon_node/beacon_chain/src/data_column_verification.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ fn verify_data_column_sidecar<E: EthSpec>(
423423
data_column: &DataColumnSidecar<E>,
424424
spec: &ChainSpec,
425425
) -> Result<(), GossipDataColumnError> {
426-
if data_column.index >= spec.number_of_columns as u64 {
426+
if data_column.index >= spec.number_of_columns {
427427
return Err(GossipDataColumnError::InvalidColumnIndex(data_column.index));
428428
}
429429
if data_column.kzg_commitments.is_empty() {
@@ -611,7 +611,7 @@ fn verify_index_matches_subnet<E: EthSpec>(
611611
spec: &ChainSpec,
612612
) -> Result<(), GossipDataColumnError> {
613613
let expected_subnet: u64 =
614-
DataColumnSubnetId::from_column_index::<E>(data_column.index as usize, spec).into();
614+
DataColumnSubnetId::from_column_index(data_column.index, spec).into();
615615
if expected_subnet != subnet {
616616
return Err(GossipDataColumnError::InvalidSubnetId {
617617
received: subnet,

‎beacon_node/beacon_chain/src/kzg_utils.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ fn build_data_column_sidecars<E: EthSpec>(
193193
blob_cells_and_proofs_vec: Vec<CellsAndKzgProofs>,
194194
spec: &ChainSpec,
195195
) -> Result<DataColumnSidecarList<E>, String> {
196-
let number_of_columns = spec.number_of_columns;
196+
let number_of_columns = spec.number_of_columns as usize;
197197
let max_blobs_per_block = spec
198198
.max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch()))
199199
as usize;
@@ -428,7 +428,7 @@ mod test {
428428
.kzg_commitments_merkle_proof()
429429
.unwrap();
430430

431-
assert_eq!(column_sidecars.len(), spec.number_of_columns);
431+
assert_eq!(column_sidecars.len(), spec.number_of_columns as usize);
432432
for (idx, col_sidecar) in column_sidecars.iter().enumerate() {
433433
assert_eq!(col_sidecar.index, idx as u64);
434434

@@ -461,7 +461,7 @@ mod test {
461461
)
462462
.unwrap();
463463

464-
for i in 0..spec.number_of_columns {
464+
for i in 0..spec.number_of_columns as usize {
465465
assert_eq!(reconstructed_columns.get(i), column_sidecars.get(i), "{i}");
466466
}
467467
}

‎beacon_node/beacon_chain/src/observed_data_sidecars.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ impl<E: EthSpec> ObservableDataSidecar for DataColumnSidecar<E> {
5959
}
6060

6161
fn max_num_of_items(spec: &ChainSpec, _slot: Slot) -> usize {
62-
spec.number_of_columns
62+
spec.number_of_columns as usize
6363
}
6464
}
6565

‎beacon_node/http_api/src/block_id.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ impl BlockId {
347347

348348
let num_found_column_keys = column_indices.len();
349349
let num_required_columns = chain.spec.number_of_columns / 2;
350-
let is_blob_available = num_found_column_keys >= num_required_columns;
350+
let is_blob_available = num_found_column_keys >= num_required_columns as usize;
351351

352352
if is_blob_available {
353353
let data_columns = column_indices

‎beacon_node/http_api/src/publish_blocks.rs

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -395,9 +395,8 @@ fn build_gossip_verified_data_columns<T: BeaconChainTypes>(
395395
let gossip_verified_data_columns = data_column_sidecars
396396
.into_iter()
397397
.map(|data_column_sidecar| {
398-
let column_index = data_column_sidecar.index as usize;
399-
let subnet =
400-
DataColumnSubnetId::from_column_index::<T::EthSpec>(column_index, &chain.spec);
398+
let column_index = data_column_sidecar.index;
399+
let subnet = DataColumnSubnetId::from_column_index(column_index, &chain.spec);
401400
let gossip_verified_column =
402401
GossipVerifiedDataColumn::new(data_column_sidecar, subnet.into(), chain);
403402

@@ -520,10 +519,7 @@ fn publish_column_sidecars<T: BeaconChainTypes>(
520519
let pubsub_messages = data_column_sidecars
521520
.into_iter()
522521
.map(|data_col| {
523-
let subnet = DataColumnSubnetId::from_column_index::<T::EthSpec>(
524-
data_col.index as usize,
525-
&chain.spec,
526-
);
522+
let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec);
527523
PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col)))
528524
})
529525
.collect::<Vec<_>>();

‎beacon_node/lighthouse_network/src/discovery/enr.rs

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ pub const ETH2_ENR_KEY: &str = "eth2";
2525
pub const ATTESTATION_BITFIELD_ENR_KEY: &str = "attnets";
2626
/// The ENR field specifying the sync committee subnet bitfield.
2727
pub const SYNC_COMMITTEE_BITFIELD_ENR_KEY: &str = "syncnets";
28-
/// The ENR field specifying the peerdas custody subnet count.
29-
pub const PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY: &str = "csc";
28+
/// The ENR field specifying the peerdas custody group count.
29+
pub const PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY: &str = "cgc";
3030

3131
/// Extension trait for ENR's within Eth2.
3232
pub trait Eth2Enr {
@@ -38,8 +38,8 @@ pub trait Eth2Enr {
3838
&self,
3939
) -> Result<EnrSyncCommitteeBitfield<E>, &'static str>;
4040

41-
/// The peerdas custody subnet count associated with the ENR.
42-
fn custody_subnet_count<E: EthSpec>(&self, spec: &ChainSpec) -> Result<u64, &'static str>;
41+
/// The peerdas custody group count associated with the ENR.
42+
fn custody_group_count<E: EthSpec>(&self, spec: &ChainSpec) -> Result<u64, &'static str>;
4343

4444
fn eth2(&self) -> Result<EnrForkId, &'static str>;
4545
}
@@ -67,16 +67,16 @@ impl Eth2Enr for Enr {
6767
.map_err(|_| "Could not decode the ENR syncnets bitfield")
6868
}
6969

70-
fn custody_subnet_count<E: EthSpec>(&self, spec: &ChainSpec) -> Result<u64, &'static str> {
71-
let csc = self
72-
.get_decodable::<u64>(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY)
73-
.ok_or("ENR custody subnet count non-existent")?
74-
.map_err(|_| "Could not decode the ENR custody subnet count")?;
70+
fn custody_group_count<E: EthSpec>(&self, spec: &ChainSpec) -> Result<u64, &'static str> {
71+
let cgc = self
72+
.get_decodable::<u64>(PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY)
73+
.ok_or("ENR custody group count non-existent")?
74+
.map_err(|_| "Could not decode the ENR custody group count")?;
7575

76-
if csc >= spec.custody_requirement && csc <= spec.data_column_sidecar_subnet_count {
77-
Ok(csc)
76+
if (spec.custody_requirement..=spec.number_of_custody_groups).contains(&cgc) {
77+
Ok(cgc)
7878
} else {
79-
Err("Invalid custody subnet count in ENR")
79+
Err("Invalid custody group count in ENR")
8080
}
8181
}
8282

@@ -253,14 +253,14 @@ pub fn build_enr<E: EthSpec>(
253253
&bitfield.as_ssz_bytes().into(),
254254
);
255255

256-
// only set `csc` if PeerDAS fork epoch has been scheduled
256+
// only set `cgc` if PeerDAS fork epoch has been scheduled
257257
if spec.is_peer_das_scheduled() {
258-
let custody_subnet_count = if config.subscribe_all_data_column_subnets {
259-
spec.data_column_sidecar_subnet_count
258+
let custody_group_count = if config.subscribe_all_data_column_subnets {
259+
spec.number_of_custody_groups
260260
} else {
261261
spec.custody_requirement
262262
};
263-
builder.add_value(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY, &custody_subnet_count);
263+
builder.add_value(PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY, &custody_group_count);
264264
}
265265

266266
builder
@@ -287,11 +287,11 @@ fn compare_enr(local_enr: &Enr, disk_enr: &Enr) -> bool {
287287
&& (local_enr.udp4().is_none() || local_enr.udp4() == disk_enr.udp4())
288288
&& (local_enr.udp6().is_none() || local_enr.udp6() == disk_enr.udp6())
289289
// we need the ATTESTATION_BITFIELD_ENR_KEY and SYNC_COMMITTEE_BITFIELD_ENR_KEY and
290-
// PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY key to match, otherwise we use a new ENR. This will
290+
// PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY key to match, otherwise we use a new ENR. This will
291291
// likely only be true for non-validating nodes.
292292
&& local_enr.get_decodable::<Bytes>(ATTESTATION_BITFIELD_ENR_KEY) == disk_enr.get_decodable(ATTESTATION_BITFIELD_ENR_KEY)
293293
&& local_enr.get_decodable::<Bytes>(SYNC_COMMITTEE_BITFIELD_ENR_KEY) == disk_enr.get_decodable(SYNC_COMMITTEE_BITFIELD_ENR_KEY)
294-
&& local_enr.get_decodable::<Bytes>(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY) == disk_enr.get_decodable(PEERDAS_CUSTODY_SUBNET_COUNT_ENR_KEY)
294+
&& local_enr.get_decodable::<Bytes>(PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY) == disk_enr.get_decodable(PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY)
295295
}
296296

297297
/// Loads enr from the given directory
@@ -348,7 +348,7 @@ mod test {
348348
}
349349

350350
#[test]
351-
fn custody_subnet_count_default() {
351+
fn custody_group_count_default() {
352352
let config = NetworkConfig {
353353
subscribe_all_data_column_subnets: false,
354354
..NetworkConfig::default()
@@ -358,13 +358,13 @@ mod test {
358358
let enr = build_enr_with_config(config, &spec).0;
359359

360360
assert_eq!(
361-
enr.custody_subnet_count::<E>(&spec).unwrap(),
361+
enr.custody_group_count::<E>(&spec).unwrap(),
362362
spec.custody_requirement,
363363
);
364364
}
365365

366366
#[test]
367-
fn custody_subnet_count_all() {
367+
fn custody_group_count_all() {
368368
let config = NetworkConfig {
369369
subscribe_all_data_column_subnets: true,
370370
..NetworkConfig::default()
@@ -373,8 +373,8 @@ mod test {
373373
let enr = build_enr_with_config(config, &spec).0;
374374

375375
assert_eq!(
376-
enr.custody_subnet_count::<E>(&spec).unwrap(),
377-
spec.data_column_sidecar_subnet_count,
376+
enr.custody_group_count::<E>(&spec).unwrap(),
377+
spec.number_of_custody_groups,
378378
);
379379
}
380380

‎beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
//! The subnet predicate used for searching for a particular subnet.
22
use super::*;
33
use crate::types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield};
4-
use itertools::Itertools;
54
use slog::trace;
65
use std::ops::Deref;
7-
use types::{ChainSpec, DataColumnSubnetId};
6+
use types::data_column_custody_group::compute_subnets_for_node;
7+
use types::ChainSpec;
88

99
/// Returns the predicate for a given subnet.
1010
pub fn subnet_predicate<E>(
@@ -37,13 +37,9 @@ where
3737
.as_ref()
3838
.is_ok_and(|b| b.get(*s.deref() as usize).unwrap_or(false)),
3939
Subnet::DataColumn(s) => {
40-
if let Ok(custody_subnet_count) = enr.custody_subnet_count::<E>(&spec) {
41-
DataColumnSubnetId::compute_custody_subnets::<E>(
42-
enr.node_id().raw(),
43-
custody_subnet_count,
44-
&spec,
45-
)
46-
.is_ok_and(|mut subnets| subnets.contains(s))
40+
if let Ok(custody_group_count) = enr.custody_group_count::<E>(&spec) {
41+
compute_subnets_for_node(enr.node_id().raw(), custody_group_count, &spec)
42+
.is_ok_and(|subnets| subnets.contains(s))
4743
} else {
4844
false
4945
}

0 commit comments

Comments
 (0)