diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ad9d875214b..61e961690fe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -143,13 +143,13 @@ jobs: id: cache-graph uses: actions/cache@v2 with: - path: lightning/net_graph-2021-02-12.bin - key: ldk-net_graph-05f0c5a0d772-2020-02-12.bin + path: lightning/net_graph-2021-05-27.bin + key: ldk-net_graph-45d86ead641d-2021-05-27.bin - name: Fetch routing graph snapshot if: steps.cache-graph.outputs.cache-hit != 'true' run: | - wget -O lightning/net_graph-2021-02-12.bin https://bitcoin.ninja/ldk-net_graph-05f0c5a0d772-2020-02-12.bin - if [ "$(sha256sum lightning/net_graph-2021-02-12.bin | awk '{ print $1 }')" != "7116fca78551fedc714a604cec0ad1ca66caa77bb4d0051290258e7a10e0c6e7" ]; then + wget -O lightning/net_graph-2021-05-27.bin https://bitcoin.ninja/ldk-net_graph-45d86ead641d-2021-05-27.bin + if [ "$(sha256sum lightning/net_graph-2021-05-27.bin | awk '{ print $1 }')" != "3d6261187cfa583255d978efb908b51c2f4dc4ad9a7160cd2c5263c9a4830121" ]; then echo "Bad hash" exit 1 fi diff --git a/lightning/src/chain/package.rs b/lightning/src/chain/package.rs index 2e1e50371dc..2c590833e94 100644 --- a/lightning/src/chain/package.rs +++ b/lightning/src/chain/package.rs @@ -21,7 +21,7 @@ use bitcoin::hash_types::Txid; use bitcoin::secp256k1::key::{SecretKey,PublicKey}; use ln::PaymentPreimage; -use ln::chan_utils::{TxCreationKeys, HTLCOutputInCommitment, HTLC_OUTPUT_IN_COMMITMENT_SIZE}; +use ln::chan_utils::{TxCreationKeys, HTLCOutputInCommitment}; use ln::chan_utils; use ln::msgs::DecodeError; use chain::chaininterface::{FeeEstimator, ConfirmationTarget, MIN_RELAY_FEE_SAT_PER_1000_WEIGHT}; @@ -86,15 +86,15 @@ impl RevokedOutput { } } -impl_writeable!(RevokedOutput, 33*3 + 32 + 8 + 8 + 2, { - per_commitment_point, - counterparty_delayed_payment_base_key, - counterparty_htlc_base_key, - per_commitment_key, - weight, - amount, - on_counterparty_tx_csv -}); +impl_writeable_tlv_based!(RevokedOutput, { + (0, per_commitment_point), + (2, counterparty_delayed_payment_base_key), + (4, counterparty_htlc_base_key), + (6, per_commitment_key), + (8, weight), + (10, amount), + (12, on_counterparty_tx_csv), +}, {}, {}); /// A struct to describe a revoked offered output and corresponding information to generate a /// solving witness. @@ -130,15 +130,15 @@ impl RevokedHTLCOutput { } } -impl_writeable!(RevokedHTLCOutput, 33*3 + 32 + 8 + 8 + HTLC_OUTPUT_IN_COMMITMENT_SIZE, { - per_commitment_point, - counterparty_delayed_payment_base_key, - counterparty_htlc_base_key, - per_commitment_key, - weight, - amount, - htlc -}); +impl_writeable_tlv_based!(RevokedHTLCOutput, { + (0, per_commitment_point), + (2, counterparty_delayed_payment_base_key), + (4, counterparty_htlc_base_key), + (6, per_commitment_key), + (8, weight), + (10, amount), + (12, htlc), +}, {}, {}); /// A struct to describe a HTLC output on a counterparty commitment transaction. /// @@ -167,13 +167,13 @@ impl CounterpartyOfferedHTLCOutput { } } -impl_writeable!(CounterpartyOfferedHTLCOutput, 33*3 + 32 + HTLC_OUTPUT_IN_COMMITMENT_SIZE, { - per_commitment_point, - counterparty_delayed_payment_base_key, - counterparty_htlc_base_key, - preimage, - htlc -}); +impl_writeable_tlv_based!(CounterpartyOfferedHTLCOutput, { + (0, per_commitment_point), + (2, counterparty_delayed_payment_base_key), + (4, counterparty_htlc_base_key), + (6, preimage), + (8, htlc), +}, {}, {}); /// A struct to describe a HTLC output on a counterparty commitment transaction. /// @@ -198,12 +198,12 @@ impl CounterpartyReceivedHTLCOutput { } } -impl_writeable!(CounterpartyReceivedHTLCOutput, 33*3 + HTLC_OUTPUT_IN_COMMITMENT_SIZE, { - per_commitment_point, - counterparty_delayed_payment_base_key, - counterparty_htlc_base_key, - htlc -}); +impl_writeable_tlv_based!(CounterpartyReceivedHTLCOutput, { + (0, per_commitment_point), + (2, counterparty_delayed_payment_base_key), + (4, counterparty_htlc_base_key), + (6, htlc), +}, {}, {}); /// A struct to describe a HTLC output on holder commitment transaction. /// @@ -224,10 +224,11 @@ impl HolderHTLCOutput { } } -impl_writeable!(HolderHTLCOutput, 0, { - preimage, - amount -}); +impl_writeable_tlv_based!(HolderHTLCOutput, { + (0, amount), +}, { + (2, preimage), +}, {}); /// A struct to describe the channel output on the funding transaction. /// @@ -245,9 +246,9 @@ impl HolderFundingOutput { } } -impl_writeable!(HolderFundingOutput, 0, { - funding_redeemscript -}); +impl_writeable_tlv_based!(HolderFundingOutput, { + (0, funding_redeemscript), +}, {}, {}); /// A wrapper encapsulating all in-protocol differing outputs types. /// @@ -703,10 +704,11 @@ impl Writeable for PackageTemplate { outpoint.write(writer)?; rev_outp.write(writer)?; } - self.soonest_conf_deadline.write(writer)?; - self.feerate_previous.write(writer)?; - self.height_timer.write(writer)?; - self.height_original.write(writer)?; + write_tlv_fields!(writer, { + (0, self.soonest_conf_deadline), + (2, self.feerate_previous), + (4, self.height_original), + }, { (6, self.height_timer) }); Ok(()) } } @@ -730,10 +732,15 @@ impl Readable for PackageTemplate { PackageSolvingData::HolderFundingOutput(..) => { (PackageMalleability::Untractable, false) }, } } else { return Err(DecodeError::InvalidValue); }; - let soonest_conf_deadline = Readable::read(reader)?; - let feerate_previous = Readable::read(reader)?; - let height_timer = Readable::read(reader)?; - let height_original = Readable::read(reader)?; + let mut soonest_conf_deadline = 0; + let mut feerate_previous = 0; + let mut height_timer = None; + let mut height_original = 0; + read_tlv_fields!(reader, { + (0, soonest_conf_deadline), + (2, feerate_previous), + (4, height_original) + }, { (6, height_timer) }); Ok(PackageTemplate { inputs, malleability, diff --git a/lightning/src/chain/transaction.rs b/lightning/src/chain/transaction.rs index 502eb895b26..4db5fbed6a8 100644 --- a/lightning/src/chain/transaction.rs +++ b/lightning/src/chain/transaction.rs @@ -73,6 +73,14 @@ impl OutPoint { vout: self.index as u32, } } + + /// Creates a dummy BitcoinOutPoint, useful for deserializing into. + pub(crate) fn null() -> Self { + Self { + txid: Default::default(), + index: 0 + } + } } impl_writeable!(OutPoint, 0, { txid, index }); diff --git a/lightning/src/ln/chan_utils.rs b/lightning/src/ln/chan_utils.rs index 28b5a9f8bc9..48b5046cced 100644 --- a/lightning/src/ln/chan_utils.rs +++ b/lightning/src/ln/chan_utils.rs @@ -178,6 +178,7 @@ impl Writeable for CounterpartyCommitmentSecrets { writer.write_all(secret)?; writer.write_all(&byte_utils::be64_to_array(*idx))?; } + write_tlv_fields!(writer, {}, {}); Ok(()) } } @@ -188,7 +189,7 @@ impl Readable for CounterpartyCommitmentSecrets { *secret = Readable::read(reader)?; *idx = Readable::read(reader)?; } - + read_tlv_fields!(reader, {}, {}); Ok(Self { old_secrets }) } } @@ -322,8 +323,13 @@ pub struct TxCreationKeys { pub broadcaster_delayed_payment_key: PublicKey, } -impl_writeable!(TxCreationKeys, 33*5, - { per_commitment_point, revocation_key, broadcaster_htlc_key, countersignatory_htlc_key, broadcaster_delayed_payment_key }); +impl_writeable_tlv_based!(TxCreationKeys, { + (0, per_commitment_point), + (2, revocation_key), + (4, broadcaster_htlc_key), + (6, countersignatory_htlc_key), + (8, broadcaster_delayed_payment_key), +}, {}, {}); /// One counterparty's public keys which do not change over the life of a channel. #[derive(Clone, PartialEq)] @@ -349,14 +355,13 @@ pub struct ChannelPublicKeys { pub htlc_basepoint: PublicKey, } -impl_writeable!(ChannelPublicKeys, 33*5, { - funding_pubkey, - revocation_basepoint, - payment_point, - delayed_payment_basepoint, - htlc_basepoint -}); - +impl_writeable_tlv_based!(ChannelPublicKeys, { + (0, funding_pubkey), + (2, revocation_basepoint), + (4, payment_point), + (6, delayed_payment_basepoint), + (8, htlc_basepoint), +}, {}, {}); impl TxCreationKeys { /// Create per-state keys from channel base points and the per-commitment point. @@ -429,16 +434,14 @@ pub struct HTLCOutputInCommitment { pub transaction_output_index: Option, } -impl_writeable_len_match!(HTLCOutputInCommitment, { - { HTLCOutputInCommitment { transaction_output_index: None, .. }, HTLC_OUTPUT_IN_COMMITMENT_SIZE - 4 }, - { _, HTLC_OUTPUT_IN_COMMITMENT_SIZE } - }, { - offered, - amount_msat, - cltv_expiry, - payment_hash, - transaction_output_index -}); +impl_writeable_tlv_based!(HTLCOutputInCommitment, { + (0, offered), + (2, amount_msat), + (4, cltv_expiry), + (6, payment_hash), +}, { + (8, transaction_output_index) +}, {}); #[inline] pub(crate) fn get_htlc_redeemscript_with_explicit_keys(htlc: &HTLCOutputInCommitment, broadcaster_htlc_key: &PublicKey, countersignatory_htlc_key: &PublicKey, revocation_key: &PublicKey) -> Script { @@ -622,18 +625,19 @@ impl ChannelTransactionParameters { } } -impl_writeable!(CounterpartyChannelTransactionParameters, 0, { - pubkeys, - selected_contest_delay -}); +impl_writeable_tlv_based!(CounterpartyChannelTransactionParameters, { + (0, pubkeys), + (2, selected_contest_delay), +}, {}, {}); -impl_writeable!(ChannelTransactionParameters, 0, { - holder_pubkeys, - holder_selected_contest_delay, - is_outbound_from_holder, - counterparty_parameters, - funding_outpoint -}); +impl_writeable_tlv_based!(ChannelTransactionParameters, { + (0, holder_pubkeys), + (2, holder_selected_contest_delay), + (4, is_outbound_from_holder), +}, { + (6, counterparty_parameters), + (8, funding_outpoint), +}, {}); /// Static channel fields used to build transactions given per-commitment fields, organized by /// broadcaster/countersignatory. @@ -715,8 +719,12 @@ impl PartialEq for HolderCommitmentTransaction { } } -impl_writeable!(HolderCommitmentTransaction, 0, { - inner, counterparty_sig, counterparty_htlc_sigs, holder_sig_first +impl_writeable_tlv_based!(HolderCommitmentTransaction, { + (0, inner), + (2, counterparty_sig), + (4, holder_sig_first), +}, {}, { + (6, counterparty_htlc_sigs), }); impl HolderCommitmentTransaction { @@ -800,7 +808,10 @@ pub struct BuiltCommitmentTransaction { pub txid: Txid, } -impl_writeable!(BuiltCommitmentTransaction, 0, { transaction, txid }); +impl_writeable_tlv_based!(BuiltCommitmentTransaction, { + (0, transaction), + (2, txid) +}, {}, {}); impl BuiltCommitmentTransaction { /// Get the SIGHASH_ALL sighash value of the transaction. @@ -883,15 +894,15 @@ impl Readable for Vec { } } -impl_writeable!(CommitmentTransaction, 0, { - commitment_number, - to_broadcaster_value_sat, - to_countersignatory_value_sat, - feerate_per_kw, - htlcs, - keys, - built -}); +impl_writeable_tlv_based!(CommitmentTransaction, { + (0, commitment_number), + (2, to_broadcaster_value_sat), + (4, to_countersignatory_value_sat), + (6, feerate_per_kw), + (8, htlcs), + (10, keys), + (12, built), +}, {}, {}); impl CommitmentTransaction { /// Construct an object of the class while assigning transaction output indices to HTLCs. diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 9d4d55c406f..1a032eefea0 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -4316,9 +4316,9 @@ impl PersistenceNotifier { const SERIALIZATION_VERSION: u8 = 1; const MIN_SERIALIZATION_VERSION: u8 = 1; -impl Writeable for PendingHTLCInfo { +impl Writeable for PendingHTLCRouting { fn write(&self, writer: &mut W) -> Result<(), ::std::io::Error> { - match &self.routing { + match &self { &PendingHTLCRouting::Forward { ref onion_packet, ref short_channel_id } => { 0u8.write(writer)?; onion_packet.write(writer)?; @@ -4331,39 +4331,37 @@ impl Writeable for PendingHTLCInfo { incoming_cltv_expiry.write(writer)?; }, } - self.incoming_shared_secret.write(writer)?; - self.payment_hash.write(writer)?; - self.amt_to_forward.write(writer)?; - self.outgoing_cltv_value.write(writer)?; Ok(()) } } -impl Readable for PendingHTLCInfo { - fn read(reader: &mut R) -> Result { - Ok(PendingHTLCInfo { - routing: match Readable::read(reader)? { - 0u8 => PendingHTLCRouting::Forward { - onion_packet: Readable::read(reader)?, - short_channel_id: Readable::read(reader)?, - }, - 1u8 => PendingHTLCRouting::Receive { - payment_data: msgs::FinalOnionHopData { - payment_secret: Readable::read(reader)?, - total_msat: Readable::read(reader)?, - }, - incoming_cltv_expiry: Readable::read(reader)?, +impl Readable for PendingHTLCRouting { + fn read(reader: &mut R) -> Result { + match Readable::read(reader)? { + 0u8 => Ok(PendingHTLCRouting::Forward { + onion_packet: Readable::read(reader)?, + short_channel_id: Readable::read(reader)?, + }), + 1u8 => Ok(PendingHTLCRouting::Receive { + payment_data: msgs::FinalOnionHopData { + payment_secret: Readable::read(reader)?, + total_msat: Readable::read(reader)?, }, - _ => return Err(DecodeError::InvalidValue), - }, - incoming_shared_secret: Readable::read(reader)?, - payment_hash: Readable::read(reader)?, - amt_to_forward: Readable::read(reader)?, - outgoing_cltv_value: Readable::read(reader)?, - }) + incoming_cltv_expiry: Readable::read(reader)?, + }), + _ => Err(DecodeError::InvalidValue), + } } } +impl_writeable_tlv_based!(PendingHTLCInfo, { + (0, routing), + (2, incoming_shared_secret), + (4, payment_hash), + (6, amt_to_forward), + (8, outgoing_cltv_value) +}, {}, {}); + impl Writeable for HTLCFailureMsg { fn write(&self, writer: &mut W) -> Result<(), ::std::io::Error> { match self { @@ -4416,33 +4414,52 @@ impl Readable for PendingHTLCStatus { } } -impl_writeable!(HTLCPreviousHopData, 0, { - short_channel_id, - outpoint, - htlc_id, - incoming_packet_shared_secret -}); +impl_writeable_tlv_based!(HTLCPreviousHopData, { + (0, short_channel_id), + (2, outpoint), + (4, htlc_id), + (6, incoming_packet_shared_secret) +}, {}, {}); impl Writeable for ClaimableHTLC { fn write(&self, writer: &mut W) -> Result<(), ::std::io::Error> { - self.prev_hop.write(writer)?; - self.value.write(writer)?; - self.payment_data.payment_secret.write(writer)?; - self.payment_data.total_msat.write(writer)?; - self.cltv_expiry.write(writer) + write_tlv_fields!(writer, { + (0, self.prev_hop), + (2, self.value), + (4, self.payment_data.payment_secret), + (6, self.payment_data.total_msat), + (8, self.cltv_expiry) + }, {}); + Ok(()) } } impl Readable for ClaimableHTLC { fn read(reader: &mut R) -> Result { + let mut prev_hop = HTLCPreviousHopData { + short_channel_id: 0, htlc_id: 0, + incoming_packet_shared_secret: [0; 32], + outpoint: OutPoint::null(), + }; + let mut value = 0; + let mut payment_secret = PaymentSecret([0; 32]); + let mut total_msat = 0; + let mut cltv_expiry = 0; + read_tlv_fields!(reader, { + (0, prev_hop), + (2, value), + (4, payment_secret), + (6, total_msat), + (8, cltv_expiry) + }, {}); Ok(ClaimableHTLC { - prev_hop: Readable::read(reader)?, - value: Readable::read(reader)?, + prev_hop, + value, payment_data: msgs::FinalOnionHopData { - payment_secret: Readable::read(reader)?, - total_msat: Readable::read(reader)?, + payment_secret, + total_msat, }, - cltv_expiry: Readable::read(reader)?, + cltv_expiry, }) } } @@ -4547,13 +4564,13 @@ impl Readable for HTLCForwardInfo { } } -impl_writeable!(PendingInboundPayment, 0, { - payment_secret, - expiry_time, - user_payment_id, - payment_preimage, - min_value_msat -}); +impl_writeable_tlv_based!(PendingInboundPayment, { + (0, payment_secret), + (2, expiry_time), + (4, user_payment_id), + (6, payment_preimage), + (8, min_value_msat), +}, {}, {}); impl Writeable for ChannelManager where M::Target: chain::Watch, diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs index 94136a9ee65..6f4795c75e8 100644 --- a/lightning/src/ln/msgs.rs +++ b/lightning/src/ln/msgs.rs @@ -482,6 +482,17 @@ impl Readable for Result { } } +impl Readable for NetAddress { + fn read(reader: &mut R) -> Result { + match Readable::read(reader) { + Ok(Ok(res)) => Ok(res), + Ok(Err(_)) => Err(DecodeError::UnknownVersion), + Err(e) => Err(e), + } + } +} + + /// The unsigned part of a node_announcement #[derive(Clone, Debug, PartialEq)] pub struct UnsignedNodeAnnouncement { diff --git a/lightning/src/routing/network_graph.rs b/lightning/src/routing/network_graph.rs index 16e0a978a91..c52f55a9104 100644 --- a/lightning/src/routing/network_graph.rs +++ b/lightning/src/routing/network_graph.rs @@ -459,15 +459,15 @@ impl fmt::Display for DirectionalChannelInfo { } } -impl_writeable!(DirectionalChannelInfo, 0, { - last_update, - enabled, - cltv_expiry_delta, - htlc_minimum_msat, - htlc_maximum_msat, - fees, - last_update_message -}); +impl_writeable_tlv_based!(DirectionalChannelInfo, { + (0, last_update), + (2, enabled), + (4, cltv_expiry_delta), + (6, htlc_minimum_msat), + (8, htlc_maximum_msat), + (10, fees), + (12, last_update_message), +}, {}, {}); #[derive(Clone, Debug, PartialEq)] /// Details about a channel (both directions). @@ -500,15 +500,15 @@ impl fmt::Display for ChannelInfo { } } -impl_writeable!(ChannelInfo, 0, { - features, - node_one, - one_to_two, - node_two, - two_to_one, - capacity_sats, - announcement_message -}); +impl_writeable_tlv_based!(ChannelInfo, { + (0, features), + (2, node_one), + (4, one_to_two), + (6, node_two), + (8, two_to_one), + (10, capacity_sats), + (12, announcement_message), +}, {}, {}); /// Fees for routing via a given channel or a node @@ -521,24 +521,7 @@ pub struct RoutingFees { pub proportional_millionths: u32, } -impl Readable for RoutingFees{ - fn read(reader: &mut R) -> Result { - let base_msat: u32 = Readable::read(reader)?; - let proportional_millionths: u32 = Readable::read(reader)?; - Ok(RoutingFees { - base_msat, - proportional_millionths, - }) - } -} - -impl Writeable for RoutingFees { - fn write(&self, writer: &mut W) -> Result<(), ::std::io::Error> { - self.base_msat.write(writer)?; - self.proportional_millionths.write(writer)?; - Ok(()) - } -} +impl_writeable_tlv_based!(RoutingFees, {(0, base_msat), (2, proportional_millionths)}, {}, {}); #[derive(Clone, Debug, PartialEq)] /// Information received in the latest node_announcement from this node. @@ -563,48 +546,16 @@ pub struct NodeAnnouncementInfo { pub announcement_message: Option } -impl Writeable for NodeAnnouncementInfo { - fn write(&self, writer: &mut W) -> Result<(), ::std::io::Error> { - self.features.write(writer)?; - self.last_update.write(writer)?; - self.rgb.write(writer)?; - self.alias.write(writer)?; - (self.addresses.len() as u64).write(writer)?; - for ref addr in &self.addresses { - addr.write(writer)?; - } - self.announcement_message.write(writer)?; - Ok(()) - } -} - -impl Readable for NodeAnnouncementInfo { - fn read(reader: &mut R) -> Result { - let features = Readable::read(reader)?; - let last_update = Readable::read(reader)?; - let rgb = Readable::read(reader)?; - let alias = Readable::read(reader)?; - let addresses_count: u64 = Readable::read(reader)?; - let mut addresses = Vec::with_capacity(cmp::min(addresses_count, MAX_ALLOC_SIZE / 40) as usize); - for _ in 0..addresses_count { - match Readable::read(reader) { - Ok(Ok(addr)) => { addresses.push(addr); }, - Ok(Err(_)) => return Err(DecodeError::InvalidValue), - Err(DecodeError::ShortRead) => return Err(DecodeError::BadLengthDescriptor), - _ => unreachable!(), - } - } - let announcement_message = Readable::read(reader)?; - Ok(NodeAnnouncementInfo { - features, - last_update, - rgb, - alias, - addresses, - announcement_message - }) - } -} +impl_writeable_tlv_based!(NodeAnnouncementInfo, { + (0, features), + (2, last_update), + (4, rgb), + (6, alias), +}, { + (8, announcement_message), +}, { + (10, addresses), +}); #[derive(Clone, Debug, PartialEq)] /// Details about a node in the network, known from the network announcement. @@ -629,36 +580,12 @@ impl fmt::Display for NodeInfo { } } -impl Writeable for NodeInfo { - fn write(&self, writer: &mut W) -> Result<(), ::std::io::Error> { - (self.channels.len() as u64).write(writer)?; - for ref chan in self.channels.iter() { - chan.write(writer)?; - } - self.lowest_inbound_channel_fees.write(writer)?; - self.announcement_info.write(writer)?; - Ok(()) - } -} - -const MAX_ALLOC_SIZE: u64 = 64*1024; - -impl Readable for NodeInfo { - fn read(reader: &mut R) -> Result { - let channels_count: u64 = Readable::read(reader)?; - let mut channels = Vec::with_capacity(cmp::min(channels_count, MAX_ALLOC_SIZE / 8) as usize); - for _ in 0..channels_count { - channels.push(Readable::read(reader)?); - } - let lowest_inbound_channel_fees = Readable::read(reader)?; - let announcement_info = Readable::read(reader)?; - Ok(NodeInfo { - channels, - lowest_inbound_channel_fees, - announcement_info, - }) - } -} +impl_writeable_tlv_based!(NodeInfo, {}, { + (0, lowest_inbound_channel_fees), + (2, announcement_info), +}, { + (4, channels), +}); const SERIALIZATION_VERSION: u8 = 1; const MIN_SERIALIZATION_VERSION: u8 = 1; diff --git a/lightning/src/routing/router.rs b/lightning/src/routing/router.rs index 1f77681bf06..7688e70b36b 100644 --- a/lightning/src/routing/router.rs +++ b/lightning/src/routing/router.rs @@ -3857,8 +3857,8 @@ mod tests { use util::ser::Readable; /// Tries to open a network graph file, or panics with a URL to fetch it. pub(super) fn get_route_file() -> Result { - let res = File::open("net_graph-2021-02-12.bin") // By default we're run in RL/lightning - .or_else(|_| File::open("lightning/net_graph-2021-02-12.bin")) // We may be run manually in RL/ + let res = File::open("net_graph-2021-05-27.bin") // By default we're run in RL/lightning + .or_else(|_| File::open("lightning/net_graph-2021-05-27.bin")) // We may be run manually in RL/ .or_else(|_| { // Fall back to guessing based on the binary location // path is likely something like .../rust-lightning/target/debug/deps/lightning-... let mut path = std::env::current_exe().unwrap(); @@ -3867,7 +3867,7 @@ mod tests { path.pop(); // debug path.pop(); // target path.push("lightning"); - path.push("net_graph-2021-02-12.bin"); + path.push("net_graph-2021-05-27.bin"); eprintln!("{}", path.to_str().unwrap()); File::open(path) }); @@ -3890,7 +3890,7 @@ mod tests { let mut d = match get_route_file() { Ok(f) => f, Err(_) => { - eprintln!("Please fetch https://bitcoin.ninja/ldk-net_graph-05f0c5a0d772-2020-02-12.bin and place it at lightning/net_graph-2021-02-12.bin"); + eprintln!("Please fetch https://bitcoin.ninja/ldk-net_graph-45d86ead641d-2021-05-27.bin and place it at lightning/net_graph-2021-05-27.bin"); return; }, }; @@ -3917,7 +3917,7 @@ mod tests { let mut d = match get_route_file() { Ok(f) => f, Err(_) => { - eprintln!("Please fetch https://bitcoin.ninja/ldk-net_graph-05f0c5a0d772-2020-02-12.bin and place it at lightning/net_graph-2021-02-12.bin"); + eprintln!("Please fetch https://bitcoin.ninja/ldk-net_graph-45d86ead641d-2021-05-27.bin and place it at lightning/net_graph-2021-05-27.bin"); return; }, }; @@ -3955,7 +3955,7 @@ mod benches { #[bench] fn generate_routes(bench: &mut Bencher) { let mut d = tests::get_route_file() - .expect("Please fetch https://bitcoin.ninja/ldk-net_graph-05f0c5a0d772-2020-02-12.bin and place it at lightning/net_graph-2021-02-12.bin"); + .expect("Please fetch https://bitcoin.ninja/ldk-net_graph-45d86ead641d-2021-05-27.bin and place it at lightning/net_graph-2021-05-27.bin"); let graph = NetworkGraph::read(&mut d).unwrap(); // First, get 100 (source, destination) pairs for which route-getting actually succeeds... @@ -3987,7 +3987,7 @@ mod benches { #[bench] fn generate_mpp_routes(bench: &mut Bencher) { let mut d = tests::get_route_file() - .expect("Please fetch https://bitcoin.ninja/ldk-net_graph-05f0c5a0d772-2020-02-12.bin and place it at lightning/net_graph-2021-02-12.bin"); + .expect("Please fetch https://bitcoin.ninja/ldk-net_graph-45d86ead641d-2021-05-27.bin and place it at lightning/net_graph-2021-05-27.bin"); let graph = NetworkGraph::read(&mut d).unwrap(); // First, get 100 (source, destination) pairs for which route-getting actually succeeds... diff --git a/lightning/src/util/ser.rs b/lightning/src/util/ser.rs index 66c89aa8391..6a76cdfae54 100644 --- a/lightning/src/util/ser.rs +++ b/lightning/src/util/ser.rs @@ -222,6 +222,40 @@ pub trait MaybeReadable fn read(reader: &mut R) -> Result, DecodeError>; } +pub(crate) struct OptionDeserWrapper(pub Option); +impl Readable for OptionDeserWrapper { + fn read(reader: &mut R) -> Result { + Ok(Self(Some(Readable::read(reader)?))) + } +} + +const MAX_ALLOC_SIZE: u64 = 64*1024; + +pub(crate) struct VecWriteWrapper<'a, T: Writeable>(pub &'a Vec); +impl<'a, T: Writeable> Writeable for VecWriteWrapper<'a, T> { + fn write(&self, writer: &mut W) -> Result<(), ::std::io::Error> { + (self.0.len() as u64).write(writer)?; + for ref v in self.0.iter() { + v.write(writer)?; + } + Ok(()) + } +} +pub(crate) struct VecReadWrapper(pub Vec); +impl Readable for VecReadWrapper { + fn read(reader: &mut R) -> Result { + let count: u64 = Readable::read(reader)?; + let mut values = Vec::with_capacity(cmp::min(count, MAX_ALLOC_SIZE / (core::mem::size_of::() as u64)) as usize); + for _ in 0..count { + match Readable::read(reader) { + Ok(v) => { values.push(v); }, + Err(e) => return Err(e), + } + } + Ok(Self(values)) + } +} + pub(crate) struct U48(pub u64); impl Writeable for U48 { #[inline] diff --git a/lightning/src/util/ser_macros.rs b/lightning/src/util/ser_macros.rs index 87f990a6f06..db970df5f1d 100644 --- a/lightning/src/util/ser_macros.rs +++ b/lightning/src/util/ser_macros.rs @@ -115,8 +115,12 @@ macro_rules! decode_tlv { _ => {}, } // As we read types, make sure we hit every required type: - $(if (last_seen_type.is_none() || last_seen_type.unwrap() < $reqtype) && typ.0 > $reqtype { - Err(DecodeError::InvalidValue)? + $({ + #[allow(unused_comparisons)] // Note that $reqtype may be 0 making the second comparison always true + let invalid_order = (last_seen_type.is_none() || last_seen_type.unwrap() < $reqtype) && typ.0 > $reqtype; + if invalid_order { + Err(DecodeError::InvalidValue)? + } })* last_seen_type = Some(typ.0); @@ -146,8 +150,12 @@ macro_rules! decode_tlv { s.eat_remaining()?; } // Make sure we got to each required type after we've read every TLV: - $(if last_seen_type.is_none() || last_seen_type.unwrap() < $reqtype { - Err(DecodeError::InvalidValue)? + $({ + #[allow(unused_comparisons)] // Note that $reqtype may be 0 making the second comparison always true + let missing_req_type = last_seen_type.is_none() || last_seen_type.unwrap() < $reqtype; + if missing_req_type { + Err(DecodeError::InvalidValue)? + } })* } } } @@ -246,7 +254,7 @@ macro_rules! write_ver_prefix { /// This is the preferred method of adding new fields that old nodes can ignore and still function /// correctly. macro_rules! write_tlv_fields { - ($stream: expr, {$(($type: expr, $field: expr)),*}, {$(($optional_type: expr, $optional_field: expr)),*}) => { + ($stream: expr, {$(($type: expr, $field: expr)),* $(,)*}, {$(($optional_type: expr, $optional_field: expr)),* $(,)*}) => { encode_varint_length_prefixed_tlv!($stream, {$(($type, $field)),*} , {$(($optional_type, $optional_field)),*}); } } @@ -267,7 +275,7 @@ macro_rules! read_ver_prefix { /// Reads a suffix added by write_tlv_fields. macro_rules! read_tlv_fields { - ($stream: expr, {$(($reqtype: expr, $reqfield: ident)),*}, {$(($type: expr, $field: ident)),*}) => { { + ($stream: expr, {$(($reqtype: expr, $reqfield: ident)),* $(,)*}, {$(($type: expr, $field: ident)),* $(,)*}) => { { let tlv_len = ::util::ser::BigSize::read($stream)?; let mut rd = ::util::ser::FixedLengthReader::new($stream, tlv_len.0); decode_tlv!(&mut rd, {$(($reqtype, $reqfield)),*}, {$(($type, $field)),*}); @@ -275,6 +283,101 @@ macro_rules! read_tlv_fields { } } } +// If we naively create a struct in impl_writeable_tlv_based below, we may end up returning +// `Self { ,,vecfield: vecfield }` which is obviously incorrect. Instead, we have to match here to +// detect at least one empty field set and skip the potentially-extra comma. +macro_rules! _init_tlv_based_struct { + ({}, {$($field: ident),*}, {$($vecfield: ident),*}) => { + Ok(Self { + $($field),*, + $($vecfield: $vecfield.unwrap().0),* + }) + }; + ({$($reqfield: ident),*}, {}, {$($vecfield: ident),*}) => { + Ok(Self { + $($reqfield: $reqfield.0.unwrap()),*, + $($vecfield: $vecfield.unwrap().0),* + }) + }; + ({$($reqfield: ident),*}, {$($field: ident),*}, {}) => { + Ok(Self { + $($reqfield: $reqfield.0.unwrap()),*, + $($field),* + }) + }; + ({$($reqfield: ident),*}, {$($field: ident),*}, {$($vecfield: ident),*}) => { + Ok(Self { + $($reqfield: $reqfield.0.unwrap()),*, + $($field),*, + $($vecfield: $vecfield.unwrap().0),* + }) + } +} + +// If we don't have any optional types below, but do have some vec types, we end up calling +// `write_tlv_field!($stream, {..}, {, (vec_ty, vec_val)})`, which is obviously broken. +// Instead, for write and read we match the missing values and skip the extra comma. +macro_rules! _write_tlv_fields { + ($stream: expr, {$(($type: expr, $field: expr)),* $(,)*}, {}, {$(($optional_type: expr, $optional_field: expr)),* $(,)*}) => { + write_tlv_fields!($stream, {$(($type, $field)),*} , {$(($optional_type, $optional_field)),*}); + }; + ($stream: expr, {$(($type: expr, $field: expr)),* $(,)*}, {$(($optional_type: expr, $optional_field: expr)),* $(,)*}, {$(($optional_type_2: expr, $optional_field_2: expr)),* $(,)*}) => { + write_tlv_fields!($stream, {$(($type, $field)),*} , {$(($optional_type, $optional_field)),*, $(($optional_type_2, $optional_field_2)),*}); + } +} +macro_rules! _read_tlv_fields { + ($stream: expr, {$(($reqtype: expr, $reqfield: ident)),* $(,)*}, {}, {$(($type: expr, $field: ident)),* $(,)*}) => { + read_tlv_fields!($stream, {$(($reqtype, $reqfield)),*}, {$(($type, $field)),*}); + }; + ($stream: expr, {$(($reqtype: expr, $reqfield: ident)),* $(,)*}, {$(($type: expr, $field: ident)),* $(,)*}, {$(($type_2: expr, $field_2: ident)),* $(,)*}) => { + read_tlv_fields!($stream, {$(($reqtype, $reqfield)),*}, {$(($type, $field)),*, $(($type_2, $field_2)),*}); + } +} + +/// Implements Readable/Writeable for a struct storing it as a set of TLVs +/// First block includes all the required fields including a dummy value which is used during +/// deserialization but which will never be exposed to other code. +/// The second block includes optional fields. +/// The third block includes any Vecs which need to have their individual elements serialized. +macro_rules! impl_writeable_tlv_based { + ($st: ident, {$(($reqtype: expr, $reqfield: ident)),* $(,)*}, {$(($type: expr, $field: ident)),* $(,)*}, {$(($vectype: expr, $vecfield: ident)),* $(,)*}) => { + impl ::util::ser::Writeable for $st { + fn write(&self, writer: &mut W) -> Result<(), ::std::io::Error> { + _write_tlv_fields!(writer, { + $(($reqtype, self.$reqfield)),* + }, { + $(($type, self.$field)),* + }, { + $(($vectype, Some(::util::ser::VecWriteWrapper(&self.$vecfield)))),* + }); + Ok(()) + } + } + + impl ::util::ser::Readable for $st { + fn read(reader: &mut R) -> Result { + $( + let mut $reqfield = ::util::ser::OptionDeserWrapper(None); + )* + $( + let mut $field = None; + )* + $( + let mut $vecfield = Some(::util::ser::VecReadWrapper(Vec::new())); + )* + _read_tlv_fields!(reader, { + $(($reqtype, $reqfield)),* + }, { + $(($type, $field)),* + }, { + $(($vectype, $vecfield)),* + }); + _init_tlv_based_struct!({$($reqfield),*}, {$($field),*}, {$($vecfield),*}) + } + } + } +} + #[cfg(test)] mod tests { use std::io::{Cursor, Read};