From 5f21a0b72b0099ba4a4e6f0b64b37b2aef447a8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cramfox=E2=80=9D?= <“kasey@n0.computer”> Date: Tue, 10 Dec 2024 23:22:33 -0500 Subject: [PATCH 1/5] feat(iroh-net-report): add QUIC address discovery probe --- Cargo.lock | 1 + iroh-net-report/Cargo.toml | 1 + iroh-net-report/src/defaults.rs | 5 +- iroh-net-report/src/lib.rs | 30 +- iroh-net-report/src/reportgen.rs | 204 ++- iroh-net-report/src/reportgen/probes.rs | 56 +- iroh-relay/src/client.rs | 4 +- net-report-defaults.rs | 49 + net-report-lib.rs | 1383 ++++++++++++++++++++ probes.rs | 877 +++++++++++++ reportgen.rs | 1597 +++++++++++++++++++++++ 11 files changed, 4157 insertions(+), 50 deletions(-) create mode 100644 net-report-defaults.rs create mode 100644 net-report-lib.rs create mode 100644 probes.rs create mode 100644 reportgen.rs diff --git a/Cargo.lock b/Cargo.lock index 57adb256c06..60dcfbd6654 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2377,6 +2377,7 @@ dependencies = [ "hickory-resolver", "iroh-base", "iroh-metrics", + "iroh-quinn", "iroh-relay", "iroh-test 0.29.0", "netwatch", diff --git a/iroh-net-report/Cargo.toml b/iroh-net-report/Cargo.toml index d5b84c85999..78a20c7e9df 100644 --- a/iroh-net-report/Cargo.toml +++ b/iroh-net-report/Cargo.toml @@ -27,6 +27,7 @@ iroh-metrics = { version = "0.29.0", default-features = false } iroh-relay = { version = "0.29", path = "../iroh-relay" } netwatch = { version = "0.2.0" } portmapper = { version = "0.2.0", default-features = false } +quinn = { package = "iroh-quinn", version = "0.12.0" } rand = "0.8" reqwest = { version = "0.12", default-features = false } rustls = { version = "0.23", default-features = false } diff --git a/iroh-net-report/src/defaults.rs b/iroh-net-report/src/defaults.rs index 094697386d2..ef26f4cdc6a 100644 --- a/iroh-net-report/src/defaults.rs +++ b/iroh-net-report/src/defaults.rs @@ -1,9 +1,6 @@ //! Default values used in net_report. -/// The default STUN port used by the Relay server. -/// -/// The STUN port as defined by [RFC 8489]() -pub const DEFAULT_STUN_PORT: u16 = 3478; +pub(crate) use iroh_base::relay_map::{DEFAULT_RELAY_QUIC_PORT, DEFAULT_STUN_PORT}; /// Contains all timeouts that we use in `iroh-net-report`. pub(crate) mod timeouts { diff --git a/iroh-net-report/src/lib.rs b/iroh-net-report/src/lib.rs index 44cbb4b1222..02a6e66d367 100644 --- a/iroh-net-report/src/lib.rs +++ b/iroh-net-report/src/lib.rs @@ -20,6 +20,7 @@ use iroh_base::relay_map::{RelayMap, RelayNode, RelayUrl}; use iroh_metrics::inc; use iroh_relay::protos::stun; use netwatch::{IpFamily, UdpSocket}; +use reportgen::QuicConfig; use tokio::{ sync::{self, mpsc, oneshot}, time::{Duration, Instant}, @@ -245,8 +246,11 @@ impl Client { dm: RelayMap, stun_conn4: Option>, stun_conn6: Option>, + quic_config: Option, ) -> Result> { - let rx = self.get_report_channel(dm, stun_conn4, stun_conn6).await?; + let rx = self + .get_report_channel(dm, stun_conn4, stun_conn6, quic_config) + .await?; match rx.await { Ok(res) => res, Err(_) => Err(anyhow!("channel closed, actor awol")), @@ -259,6 +263,7 @@ impl Client { dm: RelayMap, stun_conn4: Option>, stun_conn6: Option>, + quic_config: Option, ) -> Result>>> { // TODO: consider if RelayMap should be made to easily clone? It seems expensive // right now. @@ -268,6 +273,7 @@ impl Client { relay_map: dm, stun_sock_v4: stun_conn4, stun_sock_v6: stun_conn6, + quic_config, response_tx: tx, }) .await?; @@ -307,6 +313,11 @@ pub(crate) enum Message { /// /// Like `stun_sock_v4` but for IPv6. stun_sock_v6: Option>, + /// Endpoint and client configuration to create a QUIC + /// connection to do QUIC address discovery. + /// + /// If not provided, will not do QUIC address discovery. + quic_config: Option, /// Channel to receive the response. response_tx: oneshot::Sender>>, }, @@ -446,9 +457,16 @@ impl Actor { relay_map, stun_sock_v4, stun_sock_v6, + quic_config, response_tx, } => { - self.handle_run_check(relay_map, stun_sock_v4, stun_sock_v6, response_tx); + self.handle_run_check( + relay_map, + stun_sock_v4, + stun_sock_v6, + quic_config, + response_tx, + ); } Message::ReportReady { report } => { self.handle_report_ready(report); @@ -476,6 +494,7 @@ impl Actor { relay_map: RelayMap, stun_sock_v4: Option>, stun_sock_v6: Option>, + quic_config: Option, response_tx: oneshot::Sender>>, ) { if self.current_report_run.is_some() { @@ -526,6 +545,7 @@ impl Actor { relay_map, stun_sock_v4, stun_sock_v6, + quic_config, self.dns_resolver.clone(), ); @@ -975,7 +995,7 @@ mod tests { // Note that the ProbePlan will change with each iteration. for i in 0..5 { println!("--round {}", i); - let r = client.get_report(dm.clone(), None, None).await?; + let r = client.get_report(dm.clone(), None, None, None).await?; assert!(r.udp, "want UDP"); assert_eq!( @@ -1016,7 +1036,7 @@ mod tests { let resolver = crate::dns::tests::resolver(); let mut client = Client::new(None, resolver.clone())?; - let r = client.get_report(dm, None, None).await?; + let r = client.get_report(dm, None, None, None).await?; let mut r: Report = (*r).clone(); r.portmap_probe = None; @@ -1285,7 +1305,7 @@ mod tests { ) }; - let r = client.get_report(dm, Some(sock), None).await?; + let r = client.get_report(dm, Some(sock), None, None).await?; dbg!(&r); assert_eq!(r.hair_pinning, Some(true)); diff --git a/iroh-net-report/src/reportgen.rs b/iroh-net-report/src/reportgen.rs index 25847b5ce63..f8d408a7934 100644 --- a/iroh-net-report/src/reportgen.rs +++ b/iroh-net-report/src/reportgen.rs @@ -45,7 +45,7 @@ use url::Host; use crate::Metrics; use crate::{ self as net_report, - defaults::DEFAULT_STUN_PORT, + defaults::{DEFAULT_RELAY_QUIC_PORT, DEFAULT_STUN_PORT}, dns::ResolverExt, ping::{PingError, Pinger}, RelayMap, RelayNode, RelayUrl, Report, @@ -83,6 +83,7 @@ impl Client { relay_map: RelayMap, stun_sock4: Option>, stun_sock6: Option>, + quic_config: Option, dns_resolver: DnsResolver, ) -> Self { let (msg_tx, msg_rx) = mpsc::channel(32); @@ -98,6 +99,7 @@ impl Client { relay_map, stun_sock4, stun_sock6, + quic_config, report: Report::default(), hairpin_actor: hairpin::Client::new(net_report, addr), outstanding_tasks: OutstandingTasks::default(), @@ -171,6 +173,8 @@ struct Actor { stun_sock4: Option>, /// Socket so send IPv6 STUN requests from. stun_sock6: Option>, + /// QUIC configuration to do QUIC address Discovery + quic_config: Option, // Internal state. /// The report being built. @@ -544,6 +548,7 @@ impl Actor { let reportstate = self.addr(); let stun_sock4 = self.stun_sock4.clone(); let stun_sock6 = self.stun_sock6.clone(); + let quic_config = self.quic_config.clone(); let relay_node = probe.node().clone(); let probe = probe.clone(); let net_report = self.net_report.clone(); @@ -555,6 +560,7 @@ impl Actor { reportstate, stun_sock4, stun_sock6, + quic_config, relay_node, probe.clone(), net_report, @@ -668,6 +674,15 @@ enum ProbeError { Error(anyhow::Error, Probe), } +/// Pieces needed to do QUIC address discovery. +#[derive(Debug, Clone)] +pub struct QuicConfig { + /// A QUIC Endpoint + pub ep: quinn::Endpoint, + /// A client config. + pub client_config: rustls::ClientConfig, +} + /// Executes a particular [`Probe`], including using a delayed start if needed. /// /// If *stun_sock4* and *stun_sock6* are `None` the STUN probes are disabled. @@ -676,6 +691,7 @@ async fn run_probe( reportstate: Addr, stun_sock4: Option>, stun_sock6: Option>, + quic_config: Option, relay_node: Arc, probe: Probe, net_report: net_report::Addr, @@ -762,6 +778,22 @@ async fn run_probe( } } } + + Probe::QuicIpv4 { ref node, .. } | Probe::QuicIpv6 { ref node, .. } => { + debug!("sending QUIC address discovery probe"); + let url = node.url.clone(); + match quic_config { + Some(quic_config) => { + run_quic_probe(quic_config, url, relay_addr, probe).await?; + } + None => { + return Err(ProbeError::AbortSet( + anyhow!("No QUIC endpoint for {}", probe.proto()), + probe.clone(), + )); + } + } + } } trace!("probe successful"); @@ -851,6 +883,39 @@ async fn run_stun_probe( } } +/// Run a QUIC address discovery probe. +// TODO(ramfox): if this probe is aborted, then the connection will never be +// properly closed, possibly causing errors on the server. +async fn run_quic_probe( + quic_config: QuicConfig, + url: RelayUrl, + relay_addr: SocketAddr, + probe: Probe, +) -> Result { + match probe.proto() { + ProbeProto::QuicIpv4 => debug_assert!(relay_addr.is_ipv4()), + ProbeProto::QuicIpv6 => debug_assert!(relay_addr.is_ipv6()), + _ => debug_assert!(false, "wrong probe"), + } + // TODO(ramfox): what to put here if no host is given? + let host = url.host_str().unwrap_or("localhost"); + let quic_client = iroh_relay::quic::QuicClient::new(quic_config.ep, quic_config.client_config) + .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; + let (addr, latency) = quic_client + .get_addr_and_latency(relay_addr, host) + .await + .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; + let mut result = ProbeReport::new(probe.clone()); + if matches!(probe, Probe::QuicIpv4 { .. }) { + result.ipv4_can_send = true; + } else { + result.ipv6_can_send = true; + } + result.addr = Some(addr); + result.latency = Some(latency); + Ok(result) +} + /// Reports whether or not we think the system is behind a /// captive portal, detected by making a request to a URL that we know should /// return a "204 No Content" response and checking if that's what we get. @@ -936,6 +1001,30 @@ async fn check_captive_portal( Ok(has_captive) } +/// Returns the proper port based on the protocol of the probe. +fn get_port(relay_node: &RelayNode, proto: &ProbeProto) -> Result { + match proto { + ProbeProto::QuicIpv4 | ProbeProto::QuicIpv6 => { + if let Some(ref quic) = relay_node.quic { + if quic.port == 0 { + Ok(DEFAULT_RELAY_QUIC_PORT) + } else { + Ok(quic.port) + } + } else { + bail!("Relay node not suitable for QUIC address discovery probes"); + } + } + _ => { + if relay_node.stun_port == 0 { + Ok(DEFAULT_STUN_PORT) + } else { + Ok(relay_node.stun_port) + } + } + } +} + /// Returns the IP address to use to communicate to this relay node. /// /// *proto* specifies the protocol of the probe. Depending on the protocol we may return @@ -946,50 +1035,49 @@ async fn get_relay_addr( relay_node: &RelayNode, proto: ProbeProto, ) -> Result { - let port = if relay_node.stun_port == 0 { - DEFAULT_STUN_PORT - } else { - relay_node.stun_port - }; - if relay_node.stun_only && !matches!(proto, ProbeProto::StunIpv4 | ProbeProto::StunIpv6) { bail!("Relay node not suitable for non-STUN probes"); } + let port = get_port(relay_node, &proto)?; match proto { - ProbeProto::StunIpv4 | ProbeProto::IcmpV4 => match relay_node.url.host() { - Some(url::Host::Domain(hostname)) => { - debug!(?proto, %hostname, "Performing DNS A lookup for relay addr"); - match dns_resolver.lookup_ipv4_staggered(hostname).await { - Ok(mut addrs) => addrs - .next() - .map(|ip| ip.to_canonical()) - .map(|addr| SocketAddr::new(addr, port)) - .ok_or(anyhow!("No suitable relay addr found")), - Err(err) => Err(err.context("No suitable relay addr found")), + ProbeProto::StunIpv4 | ProbeProto::IcmpV4 | ProbeProto::QuicIpv4 => { + match relay_node.url.host() { + Some(url::Host::Domain(hostname)) => { + debug!(?proto, %hostname, "Performing DNS A lookup for relay addr"); + match dns_resolver.lookup_ipv4_staggered(hostname).await { + Ok(mut addrs) => addrs + .next() + .map(|ip| ip.to_canonical()) + .map(|addr| SocketAddr::new(addr, port)) + .ok_or(anyhow!("No suitable relay addr found")), + Err(err) => Err(err.context("No suitable relay addr found")), + } } + Some(url::Host::Ipv4(addr)) => Ok(SocketAddr::new(addr.into(), port)), + Some(url::Host::Ipv6(_addr)) => Err(anyhow!("No suitable relay addr found")), + None => Err(anyhow!("No valid hostname in RelayUrl")), } - Some(url::Host::Ipv4(addr)) => Ok(SocketAddr::new(addr.into(), port)), - Some(url::Host::Ipv6(_addr)) => Err(anyhow!("No suitable relay addr found")), - None => Err(anyhow!("No valid hostname in RelayUrl")), - }, - - ProbeProto::StunIpv6 | ProbeProto::IcmpV6 => match relay_node.url.host() { - Some(url::Host::Domain(hostname)) => { - debug!(?proto, %hostname, "Performing DNS AAAA lookup for relay addr"); - match dns_resolver.lookup_ipv6_staggered(hostname).await { - Ok(mut addrs) => addrs - .next() - .map(|ip| ip.to_canonical()) - .map(|addr| SocketAddr::new(addr, port)) - .ok_or(anyhow!("No suitable relay addr found")), - Err(err) => Err(err.context("No suitable relay addr found")), + } + + ProbeProto::StunIpv6 | ProbeProto::IcmpV6 | ProbeProto::QuicIpv6 => { + match relay_node.url.host() { + Some(url::Host::Domain(hostname)) => { + debug!(?proto, %hostname, "Performing DNS AAAA lookup for relay addr"); + match dns_resolver.lookup_ipv6_staggered(hostname).await { + Ok(mut addrs) => addrs + .next() + .map(|ip| ip.to_canonical()) + .map(|addr| SocketAddr::new(addr, port)) + .ok_or(anyhow!("No suitable relay addr found")), + Err(err) => Err(err.context("No suitable relay addr found")), + } } + Some(url::Host::Ipv4(_addr)) => Err(anyhow!("No suitable relay addr found")), + Some(url::Host::Ipv6(addr)) => Ok(SocketAddr::new(addr.into(), port)), + None => Err(anyhow!("No valid hostname in RelayUrl")), } - Some(url::Host::Ipv4(_addr)) => Err(anyhow!("No suitable relay addr found")), - Some(url::Host::Ipv6(addr)) => Ok(SocketAddr::new(addr.into(), port)), - None => Err(anyhow!("No valid hostname in RelayUrl")), - }, + } ProbeProto::Https => Err(anyhow!("Not implemented")), } @@ -1114,7 +1202,10 @@ fn update_report(report: &mut Report, probe_report: ProbeReport) { if matches!( probe_report.probe.proto(), - ProbeProto::StunIpv4 | ProbeProto::StunIpv6 + ProbeProto::StunIpv4 + | ProbeProto::StunIpv6 + | ProbeProto::QuicIpv4 + | ProbeProto::QuicIpv6 ) { report.udp = true; @@ -1464,4 +1555,43 @@ mod tests { assert_eq!(ip, relay_url_ip); Ok(()) } + + #[tokio::test] + async fn test_quic_probe() -> TestResult { + let _logging_guard = iroh_test::logging::setup(); + let (server, relay) = test_utils::relay().await; + let client_config = iroh_relay::client::make_dangerous_client_config(); + let ep = quinn::Endpoint::client(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))?; + let client_addr = ep.local_addr()?; + let quic_addr_disc = QuicConfig { + ep: ep.clone(), + client_config, + }; + let url = relay.url.clone(); + let port = server.quic_addr().unwrap().port(); + let probe = Probe::QuicIpv4 { + delay: Duration::from_secs(0), + node: relay.clone(), + }; + let probe = match run_quic_probe( + quic_addr_disc, + url, + (Ipv4Addr::LOCALHOST, port).into(), + probe, + ) + .await + { + Ok(probe) => probe, + Err(e) => match e { + ProbeError::AbortSet(err, _) | ProbeError::Error(err, _) => { + return Err(err.into()); + } + }, + }; + assert!(probe.ipv4_can_send); + assert_eq!(probe.addr.unwrap(), client_addr); + ep.wait_idle().await; + server.shutdown().await?; + Ok(()) + } } diff --git a/iroh-net-report/src/reportgen/probes.rs b/iroh-net-report/src/reportgen/probes.rs index 08bbe1163b7..42b717cfad5 100644 --- a/iroh-net-report/src/reportgen/probes.rs +++ b/iroh-net-report/src/reportgen/probes.rs @@ -54,6 +54,10 @@ pub(super) enum ProbeProto { IcmpV4, /// ICMP IPv6 IcmpV6, + /// QUIC Address Discovery Ipv4 + QuicIpv4, + /// QUIC Address Discovery Ipv6 + QuicIpv6, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, derive_more::Display)] @@ -88,6 +92,16 @@ pub(super) enum Probe { delay: Duration, node: Arc, }, + #[display("QUIC Address Discovery Ivp4 after {delay:?} to {node}")] + QuicIpv4 { + delay: Duration, + node: Arc, + }, + #[display("QUIC Address Discovery Ivp6 after {delay:?} to {node}")] + QuicIpv6 { + delay: Duration, + node: Arc, + }, } impl Probe { @@ -97,7 +111,9 @@ impl Probe { | Probe::StunIpv6 { delay, .. } | Probe::Https { delay, .. } | Probe::IcmpV4 { delay, .. } - | Probe::IcmpV6 { delay, .. } => *delay, + | Probe::IcmpV6 { delay, .. } + | Probe::QuicIpv4 { delay, .. } + | Probe::QuicIpv6 { delay, .. } => *delay, } } @@ -108,6 +124,8 @@ impl Probe { Probe::Https { .. } => ProbeProto::Https, Probe::IcmpV4 { .. } => ProbeProto::IcmpV4, Probe::IcmpV6 { .. } => ProbeProto::IcmpV6, + Probe::QuicIpv4 { .. } => ProbeProto::QuicIpv4, + Probe::QuicIpv6 { .. } => ProbeProto::QuicIpv6, } } @@ -117,7 +135,9 @@ impl Probe { | Probe::StunIpv6 { node, .. } | Probe::Https { node, .. } | Probe::IcmpV4 { node, .. } - | Probe::IcmpV6 { node, .. } => node, + | Probe::IcmpV6 { node, .. } + | Probe::QuicIpv4 { node, .. } + | Probe::QuicIpv6 { node, .. } => node, } } } @@ -206,6 +226,8 @@ impl ProbePlan { for relay_node in relay_map.nodes() { let mut stun_ipv4_probes = ProbeSet::new(ProbeProto::StunIpv4); let mut stun_ipv6_probes = ProbeSet::new(ProbeProto::StunIpv6); + let mut quic_ipv4_probes = ProbeSet::new(ProbeProto::QuicIpv4); + let mut quic_ipv6_probes = ProbeSet::new(ProbeProto::QuicIpv6); for attempt in 0..3 { let delay = DEFAULT_INITIAL_RETRANSMIT * attempt as u32; @@ -217,6 +239,12 @@ impl ProbePlan { node: relay_node.clone(), }) .expect("adding StunIpv4 probe to a StunIpv4 probe set"); + quic_ipv4_probes + .push(Probe::QuicIpv4 { + delay, + node: relay_node.clone(), + }) + .expect("adding QuicIpv4 probe to a QuicIpv4 probe set"); } if if_state.have_v6 { stun_ipv6_probes @@ -226,9 +254,17 @@ impl ProbePlan { }) .expect("adding StunIpv6 probe to a StunIpv6 probe set"); } + quic_ipv6_probes + .push(Probe::QuicIpv6 { + delay, + node: relay_node.clone(), + }) + .expect("adding QuicIpv6 probe to a QuicAddrIpv6 probe set"); } plan.add(stun_ipv4_probes); plan.add(stun_ipv6_probes); + plan.add(quic_ipv4_probes); + plan.add(quic_ipv6_probes); // The HTTP and ICMP probes only start after the STUN probes have had a chance. let mut https_probes = ProbeSet::new(ProbeProto::Https); @@ -327,6 +363,8 @@ impl ProbePlan { let mut stun_ipv4_probes = ProbeSet::new(ProbeProto::StunIpv4); let mut stun_ipv6_probes = ProbeSet::new(ProbeProto::StunIpv6); + let mut quic_ipv4_probes = ProbeSet::new(ProbeProto::QuicIpv4); + let mut quic_ipv6_probes = ProbeSet::new(ProbeProto::QuicIpv6); for attempt in 0..attempts { let delay = (retransmit_delay * attempt as u32) @@ -338,6 +376,12 @@ impl ProbePlan { node: relay_node.clone(), }) .expect("Pushing StunIpv4 Probe to StunIpv4 ProbeSet"); + quic_ipv4_probes + .push(Probe::QuicIpv4 { + delay, + node: relay_node.clone(), + }) + .expect("adding QuicIpv4 probe to a QuicAddrIpv4 probe set"); } if do6 { stun_ipv6_probes @@ -346,10 +390,18 @@ impl ProbePlan { node: relay_node.clone(), }) .expect("Pushing StunIpv6 Probe to StunIpv6 ProbeSet"); + quic_ipv6_probes + .push(Probe::QuicIpv6 { + delay, + node: relay_node.clone(), + }) + .expect("adding QuicIpv6 probe to a QuicAddrIpv6 probe set"); } } plan.add(stun_ipv4_probes); plan.add(stun_ipv6_probes); + plan.add(quic_ipv4_probes); + plan.add(quic_ipv6_probes); // The HTTP and ICMP probes only start after the STUN probes have had a chance. let mut https_probes = ProbeSet::new(ProbeProto::Https); diff --git a/iroh-relay/src/client.rs b/iroh-relay/src/client.rs index beb9819a7f4..3c8ecc2fe4b 100644 --- a/iroh-relay/src/client.rs +++ b/iroh-relay/src/client.rs @@ -345,11 +345,11 @@ impl ClientBuilder { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-utils"))] /// Creates a client config that trusts any servers without verifying their TLS certificate. /// /// Should be used for testing local relay setups only. -pub(crate) fn make_dangerous_client_config() -> rustls::ClientConfig { +pub fn make_dangerous_client_config() -> rustls::ClientConfig { warn!( "Insecure config: SSL certificates from relay servers will be trusted without verification" ); diff --git a/net-report-defaults.rs b/net-report-defaults.rs new file mode 100644 index 00000000000..26f62274156 --- /dev/null +++ b/net-report-defaults.rs @@ -0,0 +1,49 @@ +//! Default values used in net_report. + +/// The default STUN port used by the Relay server. +/// +/// The STUN port as defined by [RFC 8489]() +pub use iroh_base::relay_map::DEFAULT_STUN_PORT; + +/// The default QUIC port used by the Relay server to accept QUIC connections +/// for QUIC address discovery +/// +/// The port is "QUIC" typed on a phone keypad. +pub use iroh_base::relay_map::DEFAULT_QUIC_PORT; + +/// Contains all timeouts that we use in `iroh-net_report`. +pub(crate) mod timeouts { + use std::time::Duration; + + // Timeouts for net_report + + /// The maximum amount of time net_report will spend gathering a single report. + pub(crate) const OVERALL_REPORT_TIMEOUT: Duration = Duration::from_secs(5); + + /// The total time we wait for all the probes. + /// + /// This includes the STUN, ICMP and HTTPS probes, which will all + /// start at different times based on the ProbePlan. + pub(crate) const PROBES_TIMEOUT: Duration = Duration::from_secs(3); + + /// How long to await for a captive-portal result. + /// + /// This delay is chosen so it starts after good-working STUN probes + /// would have finished, but not too long so the delay is bearable if + /// STUN is blocked. + pub(crate) const CAPTIVE_PORTAL_DELAY: Duration = Duration::from_millis(200); + + /// Timeout for captive portal checks + /// + /// Must be lower than [`OVERALL_REPORT_TIMEOUT`] minus + /// [`CAPTIVE_PORTAL_DELAY`]. + pub(crate) const CAPTIVE_PORTAL_TIMEOUT: Duration = Duration::from_secs(2); + + pub(crate) const DNS_TIMEOUT: Duration = Duration::from_secs(3); + + /// The amount of time we wait for a hairpinned packet to come back. + pub(crate) const HAIRPIN_CHECK_TIMEOUT: Duration = Duration::from_millis(100); + + /// Default Pinger timeout + pub(crate) const DEFAULT_PINGER_TIMEOUT: Duration = Duration::from_secs(5); +} diff --git a/net-report-lib.rs b/net-report-lib.rs new file mode 100644 index 00000000000..53986322ae8 --- /dev/null +++ b/net-report-lib.rs @@ -0,0 +1,1383 @@ +//! Checks the network conditions from the current host. +//! +//! NetReport is responsible for finding out the network conditions of the current host, like +//! whether it is connected to the internet via IPv4 and/or IPv6, what the NAT situation is +//! etc and reachability to the configured relays. +// Based on + +use std::{ + collections::{BTreeMap, HashMap}, + fmt::{self, Debug}, + net::{SocketAddr, SocketAddrV4, SocketAddrV6}, + sync::Arc, +}; + +use anyhow::{anyhow, Context as _, Result}; +use bytes::Bytes; +use hickory_resolver::TokioAsyncResolver as DnsResolver; +use iroh_base::relay_map::{RelayMap, RelayNode, RelayUrl}; +#[cfg(feature = "metrics")] +use iroh_metrics::inc; +use iroh_relay::protos::stun; +use netwatch::{IpFamily, UdpSocket}; +pub use reportgen::QuicAddressDiscovery; +use tokio::{ + sync::{self, mpsc, oneshot}, + time::{Duration, Instant}, +}; +use tokio_util::{sync::CancellationToken, task::AbortOnDropHandle}; +use tracing::{debug, error, info_span, trace, warn, Instrument}; + +mod defaults; +mod dns; +#[cfg(feature = "metrics")] +mod metrics; +mod ping; +mod reportgen; + +#[cfg(feature = "metrics")] +pub use metrics::Metrics; + +const FULL_REPORT_INTERVAL: Duration = Duration::from_secs(5 * 60); + +/// The maximum latency of all nodes, if none are found yet. +/// +/// Normally the max latency of all nodes is computed, but if we don't yet know any nodes +/// latencies we return this as default. This is the value of the initial STUN probe +/// delays. It is only used as time to wait for further latencies to arrive, which *should* +/// never happen unless there already is at least one latency. Yet here we are, defining a +/// default which will never be used. +const DEFAULT_MAX_LATENCY: Duration = Duration::from_millis(100); + +/// A net_report report. +/// +/// Can be obtained by calling [`Client::get_report`]. +#[derive(Default, Debug, PartialEq, Eq, Clone)] +pub struct Report { + /// A UDP STUN round trip completed. + pub udp: bool, + /// An IPv6 STUN round trip completed. + pub ipv6: bool, + /// An IPv4 STUN round trip completed. + pub ipv4: bool, + /// An IPv6 packet was able to be sent + pub ipv6_can_send: bool, + /// an IPv4 packet was able to be sent + pub ipv4_can_send: bool, + /// could bind a socket to ::1 + pub os_has_ipv6: bool, + /// An ICMPv4 round trip completed, `None` if not checked. + pub icmpv4: Option, + /// An ICMPv6 round trip completed, `None` if not checked. + pub icmpv6: Option, + /// Whether STUN results depend on which STUN server you're talking to (on IPv4). + pub mapping_varies_by_dest_ip: Option, + /// Whether STUN results depend on which STUN server you're talking to (on IPv6). + /// + /// Note that we don't really expect this to happen and are merely logging this if + /// detecting rather than using it. For now. + pub mapping_varies_by_dest_ipv6: Option, + /// Whether the router supports communicating between two local devices through the NATted + /// public IP address (on IPv4). + pub hair_pinning: Option, + /// Probe indicating the presence of port mapping protocols on the LAN. + pub portmap_probe: Option, + /// `None` for unknown + pub preferred_relay: Option, + /// keyed by relay Url + pub relay_latency: RelayLatencies, + /// keyed by relay Url + pub relay_v4_latency: RelayLatencies, + /// keyed by relay Url + pub relay_v6_latency: RelayLatencies, + /// ip:port of global IPv4 + pub global_v4: Option, + /// `[ip]:port` of global IPv6 + pub global_v6: Option, + /// CaptivePortal is set when we think there's a captive portal that is + /// intercepting HTTP traffic. + pub captive_portal: Option, +} + +impl fmt::Display for Report { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } +} + +/// Latencies per relay node. +#[derive(Debug, Default, PartialEq, Eq, Clone)] +pub struct RelayLatencies(BTreeMap); + +impl RelayLatencies { + fn new() -> Self { + Default::default() + } + + /// Updates a relay's latency, if it is faster than before. + fn update_relay(&mut self, url: RelayUrl, latency: Duration) { + let val = self.0.entry(url).or_insert(latency); + if latency < *val { + *val = latency; + } + } + + /// Merges another [`RelayLatencies`] into this one. + /// + /// For each relay the latency is updated using [`RelayLatencies::update_relay`]. + fn merge(&mut self, other: &RelayLatencies) { + for (url, latency) in other.iter() { + self.update_relay(url.clone(), latency); + } + } + + /// Returns the maximum latency for all relays. + /// + /// If there are not yet any latencies this will return [`DEFAULT_MAX_LATENCY`]. + fn max_latency(&self) -> Duration { + self.0 + .values() + .max() + .copied() + .unwrap_or(DEFAULT_MAX_LATENCY) + } + + /// Returns an iterator over all the relays and their latencies. + pub fn iter(&self) -> impl Iterator + '_ { + self.0.iter().map(|(k, v)| (k, *v)) + } + + fn len(&self) -> usize { + self.0.len() + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn get(&self, url: &RelayUrl) -> Option { + self.0.get(url).copied() + } +} + +/// Client to run net_reports. +/// +/// Creating this creates a net_report actor which runs in the background. Most of the time +/// it is idle unless [`Client::get_report`] is called, which is the main interface. +/// +/// The [`Client`] struct can be cloned and results multiple handles to the running actor. +/// If all [`Client`]s are dropped the actor stops running. +/// +/// While running the net_report actor expects to be passed all received stun packets using +/// `Addr::receive_stun_packet`. +#[derive(Debug)] +pub struct Client { + /// Channel to send message to the [`Actor`]. + /// + /// If all senders are dropped, in other words all clones of this struct are dropped, + /// the actor will terminate. + addr: Addr, + /// Ensures the actor is terminated when the client is dropped. + _drop_guard: Arc>, +} + +#[derive(Debug)] +struct Reports { + /// Do a full relay scan, even if last is `Some`. + next_full: bool, + /// Some previous reports. + prev: HashMap>, + /// Most recent report. + last: Option>, + /// Time of last full (non-incremental) report. + last_full: Instant, +} + +impl Default for Reports { + fn default() -> Self { + Self { + next_full: Default::default(), + prev: Default::default(), + last: Default::default(), + last_full: Instant::now(), + } + } +} + +impl Client { + /// Creates a new net_report client. + /// + /// This starts a connected actor in the background. Once the client is dropped it will + /// stop running. + pub fn new(port_mapper: Option, dns_resolver: DnsResolver) -> Result { + let mut actor = Actor::new(port_mapper, dns_resolver)?; + let addr = actor.addr(); + let task = tokio::spawn( + async move { actor.run().await }.instrument(info_span!("net_report.actor")), + ); + let drop_guard = AbortOnDropHandle::new(task); + Ok(Client { + addr, + _drop_guard: Arc::new(drop_guard), + }) + } + + /// Returns a new address to send messages to this actor. + /// + /// Unlike the client itself the returned [`Addr`] does not own the actor task, it only + /// allows sending messages to the actor. + pub fn addr(&self) -> Addr { + self.addr.clone() + } + + /// Runs a net_report, returning the report. + /// + /// It may not be called concurrently with itself, `&mut self` takes care of that. + /// + /// The *stun_conn4* and *stun_conn6* endpoints are bound UDP sockets to use to send out + /// STUN packets. This function **will not read from the sockets**, as they may be + /// receiving other traffic as well, normally they are the sockets carrying the real + /// traffic. Thus all stun packets received on those sockets should be passed to + /// `Addr::receive_stun_packet` in order for this function to receive the stun + /// responses and function correctly. + /// + /// If these are not passed in this will bind sockets for STUN itself, though results + /// may not be as reliable. + pub async fn get_report( + &mut self, + dm: RelayMap, + stun_conn4: Option>, + stun_conn6: Option>, + quic_addr_disc: Option, + ) -> Result> { + let rx = self + .get_report_channel(dm, stun_conn4, stun_conn6, quic_addr_disc) + .await?; + match rx.await { + Ok(res) => res, + Err(_) => Err(anyhow!("channel closed, actor awol")), + } + } + + /// Get report with channel + pub async fn get_report_channel( + &mut self, + dm: RelayMap, + stun_conn4: Option>, + stun_conn6: Option>, + quic_addr_disc: Option, + ) -> Result>>> { + // TODO: consider if RelayMap should be made to easily clone? It seems expensive + // right now. + let (tx, rx) = oneshot::channel(); + self.addr + .send(Message::RunCheck { + relay_map: dm, + stun_sock_v4: stun_conn4, + stun_sock_v6: stun_conn6, + quic_addr_disc, + response_tx: tx, + }) + .await?; + Ok(rx) + } +} + +#[derive(Debug)] +pub(crate) struct Inflight { + /// The STUN transaction ID. + txn: stun::TransactionId, + /// The time the STUN probe was sent. + start: Instant, + /// Response to send STUN results: latency of STUN response and the discovered address. + s: sync::oneshot::Sender<(Duration, SocketAddr)>, +} + +/// Messages to send to the [`Actor`]. +#[derive(Debug)] +pub(crate) enum Message { + /// Run a net_report. + /// + /// Only one net_report can be run at a time, trying to run multiple concurrently will + /// fail. + RunCheck { + /// The relay configuration. + relay_map: RelayMap, + /// Socket to send IPv4 STUN probes from. + /// + /// Responses are never read from this socket, they must be passed in via the + /// [`Message::StunPacket`] message since the socket is also used to receive + /// other packets from in the magicsocket (`MagicSock`). + /// + /// If not provided this will attempt to bind a suitable socket itself. + stun_sock_v4: Option>, + /// Socket to send IPv6 STUN probes from. + /// + /// Like `stun_sock_v4` but for IPv6. + stun_sock_v6: Option>, + /// Endpoint and client configuration to create a QUIC + /// connection to do QUIC address discovery. + /// + /// If not provided, will not do QUIC address discovery. + quic_addr_disc: Option, + /// Channel to receive the response. + response_tx: oneshot::Sender>>, + }, + /// A report produced by the [`reportgen`] actor. + ReportReady { report: Box }, + /// The [`reportgen`] actor failed to produce a report. + ReportAborted { err: anyhow::Error }, + /// An incoming STUN packet to parse. + StunPacket { + /// The raw UDP payload. + payload: Bytes, + /// The address this was claimed to be received from. + from_addr: SocketAddr, + }, + /// A probe wants to register an in-flight STUN request. + /// + /// The sender is signalled once the STUN packet is registered with the actor and will + /// correctly accept the STUN response. + InFlightStun(Inflight, oneshot::Sender<()>), +} + +/// Sender to the main service. +/// +/// Unlike [`Client`] this is the raw channel to send messages over. Keeping this alive +/// will not keep the actor alive, which makes this handy to pass to internal tasks. +#[derive(Debug, Clone)] +pub struct Addr { + sender: mpsc::Sender, +} + +impl Addr { + /// Pass a received STUN packet to the net_reporter. + /// + /// Normally the UDP sockets to send STUN messages from are passed in so that STUN + /// packets are sent from the sockets that carry the real traffic. However because + /// these sockets carry real traffic they will also receive non-STUN traffic, thus the + /// net_report actor does not read from the sockets directly. If you receive a STUN + /// packet on the socket you should pass it to this method. + /// + /// It is safe to call this even when the net_report actor does not currently have any + /// in-flight STUN probes. The actor will simply ignore any stray STUN packets. + /// + /// There is an implicit queue here which may drop packets if the actor does not keep up + /// consuming them. + pub fn receive_stun_packet(&self, payload: Bytes, src: SocketAddr) { + if let Err(mpsc::error::TrySendError::Full(_)) = self.sender.try_send(Message::StunPacket { + payload, + from_addr: src, + }) { + #[cfg(feature = "metrics")] + inc!(Metrics, stun_packets_dropped); + warn!("dropping stun packet from {}", src); + } + } + + async fn send(&self, msg: Message) -> Result<(), mpsc::error::SendError> { + self.sender.send(msg).await.inspect_err(|_| { + error!("net_report actor lost"); + }) + } +} + +/// The net_report actor. +/// +/// This actor runs for the entire duration there's a [`Client`] connected. +#[derive(Debug)] +struct Actor { + // Actor plumbing. + /// Actor messages channel. + /// + /// If there are no more senders the actor stops. + receiver: mpsc::Receiver, + /// The sender side of the messages channel. + /// + /// This allows creating new [`Addr`]s from the actor. + sender: mpsc::Sender, + /// A collection of previously generated reports. + /// + /// Sometimes it is useful to look at past reports to decide what to do. + reports: Reports, + + // Actor configuration. + /// The port mapper client, if those are requested. + /// + /// The port mapper is responsible for talking to routers via UPnP and the like to try + /// and open ports. + port_mapper: Option, + + // Actor state. + /// Information about the currently in-flight STUN requests. + /// + /// This is used to complete the STUN probe when receiving STUN packets. + in_flight_stun_requests: HashMap, + /// The [`reportgen`] actor currently generating a report. + current_report_run: Option, + + /// The DNS resolver to use for probes that need to perform DNS lookups + dns_resolver: DnsResolver, +} + +impl Actor { + /// Creates a new actor. + /// + /// This does not start the actor, see [`Actor::run`] for this. You should not + /// normally create this directly but rather create a [`Client`]. + fn new(port_mapper: Option, dns_resolver: DnsResolver) -> Result { + // TODO: consider an instrumented flume channel so we have metrics. + let (sender, receiver) = mpsc::channel(32); + Ok(Self { + receiver, + sender, + reports: Default::default(), + port_mapper, + in_flight_stun_requests: Default::default(), + current_report_run: None, + dns_resolver, + }) + } + + /// Returns the channel to send messages to the actor. + fn addr(&self) -> Addr { + Addr { + sender: self.sender.clone(), + } + } + + /// Run the actor. + /// + /// It will now run and handle messages. Once the connected [`Client`] (including all + /// its clones) is dropped this will terminate. + async fn run(&mut self) { + debug!("net_report actor starting"); + while let Some(msg) = self.receiver.recv().await { + trace!(?msg, "handling message"); + match msg { + Message::RunCheck { + relay_map, + stun_sock_v4, + stun_sock_v6, + quic_addr_disc, + response_tx, + } => { + self.handle_run_check( + relay_map, + stun_sock_v4, + stun_sock_v6, + quic_addr_disc, + response_tx, + ); + } + Message::ReportReady { report } => { + self.handle_report_ready(report); + } + Message::ReportAborted { err } => { + self.handle_report_aborted(err); + } + Message::StunPacket { payload, from_addr } => { + self.handle_stun_packet(&payload, from_addr); + } + Message::InFlightStun(inflight, response_tx) => { + self.handle_in_flight_stun(inflight, response_tx); + } + } + } + } + + /// Starts a check run as requested by the [`Message::RunCheck`] message. + /// + /// If *stun_sock_v4* or *stun_sock_v6* are not provided this will bind the sockets + /// itself. This is not ideal since really you want to send STUN probes from the + /// sockets you will be using. + fn handle_run_check( + &mut self, + relay_map: RelayMap, + stun_sock_v4: Option>, + stun_sock_v6: Option>, + quic_addr_disc: Option, + response_tx: oneshot::Sender>>, + ) { + if self.current_report_run.is_some() { + response_tx + .send(Err(anyhow!( + "ignoring RunCheck request: reportgen actor already running" + ))) + .ok(); + return; + } + + let now = Instant::now(); + + let cancel_token = CancellationToken::new(); + let stun_sock_v4 = match stun_sock_v4 { + Some(sock) => Some(sock), + None => bind_local_stun_socket(IpFamily::V4, self.addr(), cancel_token.clone()), + }; + let stun_sock_v6 = match stun_sock_v6 { + Some(sock) => Some(sock), + None => bind_local_stun_socket(IpFamily::V6, self.addr(), cancel_token.clone()), + }; + let mut do_full = self.reports.next_full + || now.duration_since(self.reports.last_full) > FULL_REPORT_INTERVAL; + + // If the last report had a captive portal and reported no UDP access, + // it's possible that we didn't get a useful net_report due to the + // captive portal blocking us. If so, make this report a full (non-incremental) one. + if !do_full { + if let Some(ref last) = self.reports.last { + do_full = !last.udp && last.captive_portal.unwrap_or_default(); + } + } + if do_full { + self.reports.last = None; // causes ProbePlan::new below to do a full (initial) plan + self.reports.next_full = false; + self.reports.last_full = now; + #[cfg(feature = "metrics")] + inc!(Metrics, reports_full); + } + #[cfg(feature = "metrics")] + inc!(Metrics, reports); + + let actor = reportgen::Client::new( + self.addr(), + self.reports.last.clone(), + self.port_mapper.clone(), + relay_map, + stun_sock_v4, + stun_sock_v6, + quic_addr_disc, + self.dns_resolver.clone(), + ); + + self.current_report_run = Some(ReportRun { + _reportgen: actor, + _drop_guard: cancel_token.drop_guard(), + report_tx: response_tx, + }); + } + + fn handle_report_ready(&mut self, report: Box) { + let report = self.finish_and_store_report(*report); + self.in_flight_stun_requests.clear(); + if let Some(ReportRun { report_tx, .. }) = self.current_report_run.take() { + report_tx.send(Ok(report)).ok(); + } + } + + fn handle_report_aborted(&mut self, err: anyhow::Error) { + self.in_flight_stun_requests.clear(); + if let Some(ReportRun { report_tx, .. }) = self.current_report_run.take() { + report_tx.send(Err(err.context("report aborted"))).ok(); + } + } + + /// Handles [`Message::StunPacket`]. + /// + /// If there are currently no in-flight stun requests registered this is dropped, + /// otherwise forwarded to the probe. + fn handle_stun_packet(&mut self, pkt: &[u8], src: SocketAddr) { + trace!(%src, "received STUN packet"); + if self.in_flight_stun_requests.is_empty() { + return; + } + + #[cfg(feature = "metrics")] + match &src { + SocketAddr::V4(_) => { + inc!(Metrics, stun_packets_recv_ipv4); + } + SocketAddr::V6(_) => { + inc!(Metrics, stun_packets_recv_ipv6); + } + } + + match stun::parse_response(pkt) { + Ok((txn, addr_port)) => match self.in_flight_stun_requests.remove(&txn) { + Some(inf) => { + debug!(%src, %txn, "received known STUN packet"); + let elapsed = inf.start.elapsed(); + inf.s.send((elapsed, addr_port)).ok(); + } + None => { + debug!(%src, %txn, "received unexpected STUN message response"); + } + }, + Err(err) => { + match stun::parse_binding_request(pkt) { + Ok(txn) => { + // Is this our hairpin request? + match self.in_flight_stun_requests.remove(&txn) { + Some(inf) => { + debug!(%src, %txn, "received our hairpin STUN request"); + let elapsed = inf.start.elapsed(); + inf.s.send((elapsed, src)).ok(); + } + None => { + debug!(%src, %txn, "unknown STUN request"); + } + } + } + Err(_) => { + debug!(%src, "received invalid STUN response: {err:#}"); + } + } + } + } + } + + /// Handles [`Message::InFlightStun`]. + /// + /// The in-flight request is added to [`Actor::in_flight_stun_requests`] so that + /// [`Actor::handle_stun_packet`] can forward packets correctly. + /// + /// *response_tx* is to signal the actor message has been handled. + fn handle_in_flight_stun(&mut self, inflight: Inflight, response_tx: oneshot::Sender<()>) { + self.in_flight_stun_requests.insert(inflight.txn, inflight); + response_tx.send(()).ok(); + } + + fn finish_and_store_report(&mut self, report: Report) -> Arc { + let report = self.add_report_history_and_set_preferred_relay(report); + debug!("{report:?}"); + report + } + + /// Adds `r` to the set of recent Reports and mutates `r.preferred_relay` to contain the best recent one. + /// `r` is stored ref counted and a reference is returned. + fn add_report_history_and_set_preferred_relay(&mut self, mut r: Report) -> Arc { + let mut prev_relay = None; + if let Some(ref last) = self.reports.last { + prev_relay.clone_from(&last.preferred_relay); + } + let now = Instant::now(); + const MAX_AGE: Duration = Duration::from_secs(5 * 60); + + // relay ID => its best recent latency in last MAX_AGE + let mut best_recent = RelayLatencies::new(); + + // chain the current report as we are still mutating it + let prevs_iter = self + .reports + .prev + .iter() + .map(|(a, b)| -> (&Instant, &Report) { (a, b) }) + .chain(std::iter::once((&now, &r))); + + let mut to_remove = Vec::new(); + for (t, pr) in prevs_iter { + if now.duration_since(*t) > MAX_AGE { + to_remove.push(*t); + continue; + } + best_recent.merge(&pr.relay_latency); + } + + for t in to_remove { + self.reports.prev.remove(&t); + } + + // Then, pick which currently-alive relay server from the + // current report has the best latency over the past MAX_AGE. + let mut best_any = Duration::default(); + let mut old_relay_cur_latency = Duration::default(); + { + for (url, duration) in r.relay_latency.iter() { + if Some(url) == prev_relay.as_ref() { + old_relay_cur_latency = duration; + } + if let Some(best) = best_recent.get(url) { + if r.preferred_relay.is_none() || best < best_any { + best_any = best; + r.preferred_relay.replace(url.clone()); + } + } + } + + // If we're changing our preferred relay but the old one's still + // accessible and the new one's not much better, just stick with + // where we are. + if prev_relay.is_some() + && r.preferred_relay != prev_relay + && !old_relay_cur_latency.is_zero() + && best_any > old_relay_cur_latency / 3 * 2 + { + r.preferred_relay = prev_relay; + } + } + + let r = Arc::new(r); + self.reports.prev.insert(now, r.clone()); + self.reports.last = Some(r.clone()); + + r + } +} + +/// State the net_report actor needs for an in-progress report generation. +#[derive(Debug)] +struct ReportRun { + /// The handle of the [`reportgen`] actor, cancels the actor on drop. + _reportgen: reportgen::Client, + /// Drop guard to optionally kill workers started by net_report to support reportgen. + _drop_guard: tokio_util::sync::DropGuard, + /// Where to send the completed report. + report_tx: oneshot::Sender>>, +} + +/// Attempts to bind a local socket to send STUN packets from. +/// +/// If successful this returns the bound socket and will forward STUN responses to the +/// provided *actor_addr*. The *cancel_token* serves to stop the packet forwarding when the +/// socket is no longer needed. +fn bind_local_stun_socket( + network: IpFamily, + actor_addr: Addr, + cancel_token: CancellationToken, +) -> Option> { + let sock = match UdpSocket::bind(network, 0) { + Ok(sock) => Arc::new(sock), + Err(err) => { + debug!("failed to bind STUN socket: {}", err); + return None; + } + }; + let span = info_span!( + "stun_udp_listener", + local_addr = sock + .local_addr() + .map(|a| a.to_string()) + .unwrap_or(String::from("-")), + ); + { + let sock = sock.clone(); + tokio::spawn( + async move { + debug!("udp stun socket listener started"); + // TODO: Can we do better for buffers here? Probably doesn't matter much. + let mut buf = vec![0u8; 64 << 10]; + loop { + tokio::select! { + biased; + _ = cancel_token.cancelled() => break, + res = recv_stun_once(&sock, &mut buf, &actor_addr) => { + if let Err(err) = res { + warn!(%err, "stun recv failed"); + break; + } + } + } + } + debug!("udp stun socket listener stopped"); + } + .instrument(span), + ); + } + Some(sock) +} + +/// Receive STUN response from a UDP socket, pass it to the actor. +async fn recv_stun_once(sock: &UdpSocket, buf: &mut [u8], actor_addr: &Addr) -> Result<()> { + let (count, mut from_addr) = sock + .recv_from(buf) + .await + .context("Error reading from stun socket")?; + let payload = &buf[..count]; + from_addr.set_ip(from_addr.ip().to_canonical()); + let msg = Message::StunPacket { + payload: Bytes::from(payload.to_vec()), + from_addr, + }; + actor_addr.send(msg).await.context("actor stopped") +} + +/// Test if IPv6 works at all, or if it's been hard disabled at the OS level. +pub fn os_has_ipv6() -> bool { + UdpSocket::bind_local_v6(0).is_ok() +} + +#[cfg(test)] +mod test_utils { + //! Creates a relay server against which to perform tests + + use std::sync::Arc; + + use iroh_relay::server::{ + self, + testing::{quic_config, relay_config}, + ServerConfig, + }; + + use crate::RelayNode; + + pub(crate) async fn relay() -> (server::Server, Arc) { + let server = server::Server::spawn(server::testing::server_config()) + .await + .expect("should serve relay"); + let node_desc = RelayNode { + url: server.https_url().expect("should work as relay"), + stun_only: false, // the checks above and below guarantee both stun and relay + stun_port: server.stun_addr().expect("server should serve stun").port(), + quic_only: false, + quic_port: server + .quic_addr() + .expect("server should serve quic address discovery") + .port(), + }; + + (server, Arc::new(node_desc)) + } + + pub(crate) async fn relay_with_quic() -> (server::Server, Arc) { + let server_config = ServerConfig { + relay: Some(relay_config()), + stun: None, + quic: Some(quic_config()), + metrics_addr: None, + }; + let server = server::Server::spawn(server_config) + .await + .expect("should serve relay"); + let node_desc = RelayNode { + url: server.https_url().expect("should work as relay"), + stun_only: false, + stun_port: 0, + quic_only: false, + quic_port: server + .quic_addr() + .expect("server should serve quic address discovery") + .port(), + }; + + (server, Arc::new(node_desc)) + } + + /// Create a [`crate::RelayMap`] of the given size. + /// + /// This function uses [`relay`]. Note that the returned map uses internal order that will + /// often _not_ match the order of the servers. + pub(crate) async fn relay_map(relays: usize) -> (Vec, crate::RelayMap) { + let mut servers = Vec::with_capacity(relays); + let mut nodes = Vec::with_capacity(relays); + for _ in 0..relays { + let (relay_server, node) = relay().await; + servers.push(relay_server); + nodes.push(node); + } + let map = crate::RelayMap::from_nodes(nodes).expect("unuque urls"); + (servers, map) + } +} + +#[cfg(test)] +mod tests { + use std::net::Ipv4Addr; + + use bytes::BytesMut; + use tokio::time; + use tracing::info; + + use super::*; + use crate::ping::Pinger; + + mod stun_utils { + //! Utils for testing that expose a simple stun server. + + use std::{net::IpAddr, sync::Arc}; + + use anyhow::Result; + use tokio::{ + net, + sync::{oneshot, Mutex}, + }; + use tracing::{debug, trace}; + + use super::*; + use crate::{RelayMap, RelayNode, RelayUrl}; + + /// A drop guard to clean up test infrastructure. + /// + /// After dropping the test infrastructure will asynchronously shutdown and release its + /// resources. + // Nightly sees the sender as dead code currently, but we only rely on Drop of the + // sender. + #[derive(Debug)] + pub struct CleanupDropGuard { + _guard: oneshot::Sender<()>, + } + + // (read_ipv4, read_ipv6) + #[derive(Debug, Default, Clone)] + pub struct StunStats(Arc>); + + impl StunStats { + pub async fn total(&self) -> usize { + let s = self.0.lock().await; + s.0 + s.1 + } + } + + pub fn relay_map_of(stun: impl Iterator) -> RelayMap { + relay_map_of_opts(stun.map(|addr| (addr, true))) + } + + pub fn relay_map_of_opts(stun: impl Iterator) -> RelayMap { + let nodes = stun.map(|(addr, stun_only)| { + let host = addr.ip(); + let port = addr.port(); + + let url: RelayUrl = format!("http://{host}:{port}").parse().unwrap(); + RelayNode { + url, + stun_port: port, + stun_only, + quic_only: false, + quic_port: 0, + } + }); + RelayMap::from_nodes(nodes).expect("generated invalid nodes") + } + + /// Sets up a simple STUN server binding to `0.0.0.0:0`. + /// + /// See [`serve`] for more details. + pub(crate) async fn serve_v4() -> Result<(SocketAddr, StunStats, CleanupDropGuard)> { + serve(std::net::Ipv4Addr::UNSPECIFIED.into()).await + } + + /// Sets up a simple STUN server. + pub(crate) async fn serve(ip: IpAddr) -> Result<(SocketAddr, StunStats, CleanupDropGuard)> { + let stats = StunStats::default(); + + let pc = net::UdpSocket::bind((ip, 0)).await?; + let mut addr = pc.local_addr()?; + match addr.ip() { + IpAddr::V4(ip) => { + if ip.octets() == [0, 0, 0, 0] { + addr.set_ip("127.0.0.1".parse().unwrap()); + } + } + _ => unreachable!("using ipv4"), + } + + println!("STUN listening on {}", addr); + let (_guard, r) = oneshot::channel(); + let stats_c = stats.clone(); + tokio::task::spawn(async move { + run_stun(pc, stats_c, r).await; + }); + + Ok((addr, stats, CleanupDropGuard { _guard })) + } + + async fn run_stun(pc: net::UdpSocket, stats: StunStats, mut done: oneshot::Receiver<()>) { + let mut buf = vec![0u8; 64 << 10]; + loop { + trace!("read loop"); + tokio::select! { + _ = &mut done => { + debug!("shutting down"); + break; + } + res = pc.recv_from(&mut buf) => match res { + Ok((n, addr)) => { + trace!("read packet {}bytes from {}", n, addr); + let pkt = &buf[..n]; + if !stun::is(pkt) { + debug!("received non STUN pkt"); + continue; + } + if let Ok(txid) = stun::parse_binding_request(pkt) { + debug!("received binding request"); + let mut s = stats.0.lock().await; + if addr.is_ipv4() { + s.0 += 1; + } else { + s.1 += 1; + } + drop(s); + + let res = stun::response(txid, addr); + if let Err(err) = pc.send_to(&res, addr).await { + eprintln!("STUN server write failed: {:?}", err); + } + } + } + Err(err) => { + eprintln!("failed to read: {:?}", err); + } + } + } + } + } + } + + #[tokio::test] + async fn test_basic() -> Result<()> { + let _guard = iroh_test::logging::setup(); + let (stun_addr, stun_stats, _cleanup_guard) = + stun_utils::serve("127.0.0.1".parse().unwrap()).await?; + + let resolver = crate::dns::tests::resolver(); + let mut client = Client::new(None, resolver.clone())?; + let dm = stun_utils::relay_map_of([stun_addr].into_iter()); + + // Note that the ProbePlan will change with each iteration. + for i in 0..5 { + println!("--round {}", i); + let r = client.get_report(dm.clone(), None, None, None).await?; + + assert!(r.udp, "want UDP"); + assert_eq!( + r.relay_latency.len(), + 1, + "expected 1 key in RelayLatency; got {}", + r.relay_latency.len() + ); + assert!( + r.relay_latency.iter().next().is_some(), + "expected key 1 in RelayLatency; got {:?}", + r.relay_latency + ); + assert!(r.global_v4.is_some(), "expected globalV4 set"); + assert!(r.preferred_relay.is_some(),); + } + + assert!( + stun_stats.total().await >= 5, + "expected at least 5 stun, got {}", + stun_stats.total().await, + ); + + Ok(()) + } + + #[tokio::test] + async fn test_udp_blocked() -> Result<()> { + let _guard = iroh_test::logging::setup(); + + // Create a "STUN server", which will never respond to anything. This is how UDP to + // the STUN server being blocked will look like from the client's perspective. + let blackhole = tokio::net::UdpSocket::bind("127.0.0.1:0").await?; + let stun_addr = blackhole.local_addr()?; + let dm = stun_utils::relay_map_of_opts([(stun_addr, false)].into_iter()); + + // Now create a client and generate a report. + let resolver = crate::dns::tests::resolver(); + let mut client = Client::new(None, resolver.clone())?; + + let r = client.get_report(dm, None, None, None).await?; + let mut r: Report = (*r).clone(); + r.portmap_probe = None; + + // This test wants to ensure that the ICMP part of the probe works when UDP is + // blocked. Unfortunately on some systems we simply don't have permissions to + // create raw ICMP pings and we'll have to silently accept this test is useless (if + // we could, this would be a skip instead). + let pinger = Pinger::new(); + let can_ping = pinger.send(Ipv4Addr::LOCALHOST.into(), b"aa").await.is_ok(); + let want_icmpv4 = match can_ping { + true => Some(true), + false => None, + }; + + let want = Report { + // The ICMP probe sets the can_ping flag. + ipv4_can_send: can_ping, + // OS IPv6 test is irrelevant here, accept whatever the current machine has. + os_has_ipv6: r.os_has_ipv6, + // Captive portal test is irrelevant; accept what the current report has. + captive_portal: r.captive_portal, + // If we can ping we expect to have this. + icmpv4: want_icmpv4, + // If we had a pinger, we'll have some latencies filled in and a preferred relay + relay_latency: can_ping + .then(|| r.relay_latency.clone()) + .unwrap_or_default(), + preferred_relay: can_ping + .then_some(r.preferred_relay.clone()) + .unwrap_or_default(), + ..Default::default() + }; + + assert_eq!(r, want); + + Ok(()) + } + + #[tokio::test(flavor = "current_thread", start_paused = true)] + async fn test_add_report_history_set_preferred_relay() -> Result<()> { + fn relay_url(i: u16) -> RelayUrl { + format!("http://{i}.com").parse().unwrap() + } + + // report returns a *Report from (relay host, Duration)+ pairs. + fn report(a: impl IntoIterator) -> Option> { + let mut report = Report::default(); + for (s, d) in a { + assert!(s.starts_with('d'), "invalid relay server key"); + let id: u16 = s[1..].parse().unwrap(); + report + .relay_latency + .0 + .insert(relay_url(id), Duration::from_secs(d)); + } + + Some(Arc::new(report)) + } + struct Step { + /// Delay in seconds + after: u64, + r: Option>, + } + struct Test { + name: &'static str, + steps: Vec, + /// want PreferredRelay on final step + want_relay: Option, + // wanted len(c.prev) + want_prev_len: usize, + } + + let tests = [ + Test { + name: "first_reading", + steps: vec![Step { + after: 0, + r: report([("d1", 2), ("d2", 3)]), + }], + want_prev_len: 1, + want_relay: Some(relay_url(1)), + }, + Test { + name: "with_two", + steps: vec![ + Step { + after: 0, + r: report([("d1", 2), ("d2", 3)]), + }, + Step { + after: 1, + r: report([("d1", 4), ("d2", 3)]), + }, + ], + want_prev_len: 2, + want_relay: Some(relay_url(1)), // t0's d1 of 2 is still best + }, + Test { + name: "but_now_d1_gone", + steps: vec![ + Step { + after: 0, + r: report([("d1", 2), ("d2", 3)]), + }, + Step { + after: 1, + r: report([("d1", 4), ("d2", 3)]), + }, + Step { + after: 2, + r: report([("d2", 3)]), + }, + ], + want_prev_len: 3, + want_relay: Some(relay_url(2)), // only option + }, + Test { + name: "d1_is_back", + steps: vec![ + Step { + after: 0, + r: report([("d1", 2), ("d2", 3)]), + }, + Step { + after: 1, + r: report([("d1", 4), ("d2", 3)]), + }, + Step { + after: 2, + r: report([("d2", 3)]), + }, + Step { + after: 3, + r: report([("d1", 4), ("d2", 3)]), + }, // same as 2 seconds ago + ], + want_prev_len: 4, + want_relay: Some(relay_url(1)), // t0's d1 of 2 is still best + }, + Test { + name: "things_clean_up", + steps: vec![ + Step { + after: 0, + r: report([("d1", 1), ("d2", 2)]), + }, + Step { + after: 1, + r: report([("d1", 1), ("d2", 2)]), + }, + Step { + after: 2, + r: report([("d1", 1), ("d2", 2)]), + }, + Step { + after: 3, + r: report([("d1", 1), ("d2", 2)]), + }, + Step { + after: 10 * 60, + r: report([("d3", 3)]), + }, + ], + want_prev_len: 1, // t=[0123]s all gone. (too old, older than 10 min) + want_relay: Some(relay_url(3)), // only option + }, + Test { + name: "preferred_relay_hysteresis_no_switch", + steps: vec![ + Step { + after: 0, + r: report([("d1", 4), ("d2", 5)]), + }, + Step { + after: 1, + r: report([("d1", 4), ("d2", 3)]), + }, + ], + want_prev_len: 2, + want_relay: Some(relay_url(1)), // 2 didn't get fast enough + }, + Test { + name: "preferred_relay_hysteresis_do_switch", + steps: vec![ + Step { + after: 0, + r: report([("d1", 4), ("d2", 5)]), + }, + Step { + after: 1, + r: report([("d1", 4), ("d2", 1)]), + }, + ], + want_prev_len: 2, + want_relay: Some(relay_url(2)), // 2 got fast enough + }, + ]; + let resolver = crate::dns::tests::resolver(); + for mut tt in tests { + println!("test: {}", tt.name); + let mut actor = Actor::new(None, resolver.clone()).unwrap(); + for s in &mut tt.steps { + // trigger the timer + time::advance(Duration::from_secs(s.after)).await; + let r = Arc::try_unwrap(s.r.take().unwrap()).unwrap(); + s.r = Some(actor.add_report_history_and_set_preferred_relay(r)); + } + let last_report = tt.steps.last().unwrap().r.clone().unwrap(); + let got = actor.reports.prev.len(); + let want = tt.want_prev_len; + assert_eq!(got, want, "prev length"); + let got = &last_report.preferred_relay; + let want = &tt.want_relay; + assert_eq!(got, want, "preferred_relay"); + } + + Ok(()) + } + + #[tokio::test] + async fn test_hairpin() -> Result<()> { + // Hairpinning is initiated after we discover our own IPv4 socket address (IP + + // port) via STUN, so the test needs to have a STUN server and perform STUN over + // IPv4 first. Hairpinning detection works by sending a STUN *request* to **our own + // public socket address** (IP + port). If the router supports hairpinning the STUN + // request is returned back to us and received on our public address. This doesn't + // need to be a STUN request, but STUN already has a unique transaction ID which we + // can easily use to identify the packet. + + // Setup STUN server and create relay_map. + let (stun_addr, _stun_stats, _done) = stun_utils::serve_v4().await?; + let dm = stun_utils::relay_map_of([stun_addr].into_iter()); + dbg!(&dm); + + let resolver = crate::dns::tests::resolver().clone(); + let mut client = Client::new(None, resolver)?; + + // Set up an external socket to send STUN requests from, this will be discovered as + // our public socket address by STUN. We send back any packets received on this + // socket to the net_report client using Client::receive_stun_packet. Once we sent + // the hairpin STUN request (from a different randomly bound socket) we are sending + // it to this socket, which is forwarnding it back to our net_report client, because + // this dumb implementation just forwards anything even if it would be garbage. + // Thus hairpinning detection will declare hairpinning to work. + let sock = UdpSocket::bind_local(IpFamily::V4, 0)?; + let sock = Arc::new(sock); + info!(addr=?sock.local_addr().unwrap(), "Using local addr"); + let task = { + let sock = sock.clone(); + let addr = client.addr.clone(); + tokio::spawn( + async move { + let mut buf = BytesMut::zeroed(64 << 10); + loop { + let (count, src) = sock.recv_from(&mut buf).await.unwrap(); + info!( + addr=?sock.local_addr().unwrap(), + %count, + "Forwarding payload to net_report client", + ); + let payload = buf.split_to(count).freeze(); + addr.receive_stun_packet(payload, src); + } + } + .instrument(info_span!("pkt-fwd")), + ) + }; + + let r = client.get_report(dm, Some(sock), None, None).await?; + dbg!(&r); + assert_eq!(r.hair_pinning, Some(true)); + + task.abort(); + Ok(()) + } + + #[tokio::test] + async fn test_quic_basic() -> Result<()> { + let _logging_guard = iroh_test::logging::setup(); + // set up relay server that has quic enabled, but not stun + let (server, relay) = test_utils::relay_with_quic().await; + + // set up quic client endpoint to use in the report + let client_config = iroh_relay::client::make_dangerous_client_config(); + let client_config = quinn::ClientConfig::new(Arc::new(client_config)); + let ep = quinn::Endpoint::client(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))?; + let addr = match ep.local_addr()? { + SocketAddr::V4(ipp) => ipp, + SocketAddr::V6(_) => unreachable!(), + }; + let quic_addr_disc = QuicAddressDiscovery { + ep: ep.clone(), + client_config, + }; + + // create a net report client + let resolver = crate::dns::tests::resolver(); + let mut client = Client::new(None, resolver.clone())?; + + let relay_map = RelayMap::from_nodes(vec![relay])?; + let r = client + .get_report(relay_map, None, None, Some(quic_addr_disc)) + .await?; + assert!(r.ipv4); + assert!(r.ipv4_can_send); + assert_eq!(r.global_v4, Some(addr)); + + // cleanup + ep.wait_idle().await; + server.shutdown().await?; + Ok(()) + } +} diff --git a/probes.rs b/probes.rs new file mode 100644 index 00000000000..f2a46180cbc --- /dev/null +++ b/probes.rs @@ -0,0 +1,877 @@ +//! The relay probes. +//! +//! All the probes try and establish the latency to the relay servers. Preferably the STUN +//! probes work and we also learn about our public IP addresses and ports. But fallback +//! probes for HTTPS and ICMP exist as well. + +use std::{collections::BTreeSet, fmt, sync::Arc}; + +use anyhow::{ensure, Result}; +use netwatch::interfaces; +use tokio::time::Duration; + +use crate::{RelayMap, RelayNode, RelayUrl, Report}; + +/// The retransmit interval used when net_report first runs. +/// +/// We have no past context to work with, and we want answers relatively quickly, so it's +/// biased slightly more aggressive than [`DEFAULT_ACTIVE_RETRANSMIT_DELAY`]. A few extra +/// packets at startup is fine. +const DEFAULT_INITIAL_RETRANSMIT: Duration = Duration::from_millis(100); + +/// The retransmit interval used when a previous report exists but is missing latency. +/// +/// When in an active steady-state, i.e. a previous report exists, we use the latency of the +/// previous report to determine the retransmit interval. However when this previous relay +/// latency is missing this default is used. +/// +/// This is a somewhat conservative guess because if we have no data, likely the relay node +/// is very far away and we have no data because we timed out the last time we probed it. +const DEFAULT_ACTIVE_RETRANSMIT_DELAY: Duration = Duration::from_millis(200); + +/// The extra time to add to retransmits if a previous report exists. +/// +/// When in an active steady-state, i.e. a previous report exists, we add this delay +/// multiplied with the attempt to probe retries to give later attempts increasingly more +/// time. +const ACTIVE_RETRANSMIT_EXTRA_DELAY: Duration = Duration::from_millis(50); + +/// The number of fastest relays to periodically re-query during incremental net_report +/// reports. (During a full report, all relay servers are scanned.) +const NUM_INCREMENTAL_RELAYS: usize = 3; + +/// The protocol used to time a node's latency. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, derive_more::Display)] +#[repr(u8)] +pub(super) enum ProbeProto { + /// STUN IPv4 + StunIpv4, + /// STUN IPv6 + StunIpv6, + /// HTTPS + Https, + /// ICMP IPv4 + IcmpV4, + /// ICMP IPv6 + IcmpV6, + /// QUIC Address Discovery IPv4 + QuicAddrIpv4, + /// QUIC Address Discovery IPv6 + QuicAddrIpv6, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, derive_more::Display)] +pub(super) enum Probe { + #[display("STUN Ipv4 after {delay:?} to {node}")] + StunIpv4 { + /// When the probe is started, relative to the time that `get_report` is called. + /// One probe in each `ProbePlan` should have a delay of 0. Non-zero values + /// are for retries on UDP loss or timeout. + delay: Duration, + + /// The relay server to send this probe to. + node: Arc, + }, + #[display("STUN Ipv6 after {delay:?} to {node}")] + StunIpv6 { + delay: Duration, + node: Arc, + }, + #[display("HTTPS after {delay:?} to {node}")] + Https { + delay: Duration, + node: Arc, + }, + #[display("ICMPv4 after {delay:?} to {node}")] + IcmpV4 { + delay: Duration, + node: Arc, + }, + #[display("ICMPv6 after {delay:?} to {node}")] + IcmpV6 { + delay: Duration, + node: Arc, + }, + #[display("QUIC Addr Ivp4 after {delay:?} to {node}")] + QuicAddrIpv4 { + delay: Duration, + node: Arc, + }, + #[display("QUIC Addr Ivp6 after {delay:?} to {node}")] + QuicAddrIpv6 { + delay: Duration, + node: Arc, + }, +} + +impl Probe { + pub(super) fn delay(&self) -> Duration { + match self { + Probe::StunIpv4 { delay, .. } + | Probe::StunIpv6 { delay, .. } + | Probe::Https { delay, .. } + | Probe::IcmpV4 { delay, .. } + | Probe::IcmpV6 { delay, .. } + | Probe::QuicAddrIpv4 { delay, .. } + | Probe::QuicAddrIpv6 { delay, .. } => *delay, + } + } + + pub(super) fn proto(&self) -> ProbeProto { + match self { + Probe::StunIpv4 { .. } => ProbeProto::StunIpv4, + Probe::StunIpv6 { .. } => ProbeProto::StunIpv6, + Probe::Https { .. } => ProbeProto::Https, + Probe::IcmpV4 { .. } => ProbeProto::IcmpV4, + Probe::IcmpV6 { .. } => ProbeProto::IcmpV6, + Probe::QuicAddrIpv4 { .. } => ProbeProto::QuicAddrIpv4, + Probe::QuicAddrIpv6 { .. } => ProbeProto::QuicAddrIpv6, + } + } + + pub(super) fn node(&self) -> &Arc { + match self { + Probe::StunIpv4 { node, .. } + | Probe::StunIpv6 { node, .. } + | Probe::Https { node, .. } + | Probe::IcmpV4 { node, .. } + | Probe::IcmpV6 { node, .. } + | Probe::QuicAddrIpv4 { node, .. } + | Probe::QuicAddrIpv6 { node, .. } => node, + } + } +} + +/// A probe set is a sequence of similar [`Probe`]s with delays between them. +/// +/// The probes are to the same Relayer and of the same [`ProbeProto`] but will have different +/// delays. The delays are effectively retries, though they do not wait for the previous +/// probe to be finished. The first successful probe will cancel all other probes in the +/// set. +/// +/// This is a lot of type-safety by convention. It would be so much nicer to have this +/// compile-time checked but that introduces a giant mess of generics and traits and +/// associated exploding types. +/// +/// A [`ProbeSet`] implements [`IntoIterator`] similar to how [`Vec`] does. +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct ProbeSet { + /// The [`ProbeProto`] all the probes in this set have. + proto: ProbeProto, + /// The probes in the set. + probes: Vec, +} + +impl ProbeSet { + fn new(proto: ProbeProto) -> Self { + Self { + probes: Vec::new(), + proto, + } + } + + fn push(&mut self, probe: Probe) -> Result<()> { + ensure!(probe.proto() == self.proto, "mismatching probe proto"); + self.probes.push(probe); + Ok(()) + } + + fn is_empty(&self) -> bool { + self.probes.is_empty() + } +} + +impl<'a> IntoIterator for &'a ProbeSet { + type Item = &'a Probe; + + type IntoIter = std::slice::Iter<'a, Probe>; + + fn into_iter(self) -> Self::IntoIter { + self.probes.iter() + } +} + +impl fmt::Display for ProbeSet { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, r#"ProbeSet("{}") {{"#, self.proto)?; + for probe in self.probes.iter() { + writeln!(f, " {probe},")?; + } + writeln!(f, "}}") + } +} + +/// A probe plan. +/// +/// A probe plan contains a number of [`ProbeSet`]s containing probes to be executed. +/// Generally the first probe of of a set which completes aborts the remaining probes of a +/// set. Sometimes a failing probe can also abort the remaining probes of a set. +/// +/// The [`reportgen`] actor will also abort all the remaining [`ProbeSet`]s once it has +/// sufficient information for a report. +/// +/// [`reportgen`]: crate::reportgen +#[derive(Debug, PartialEq, Eq)] +pub(super) struct ProbePlan(BTreeSet); + +impl ProbePlan { + /// Creates an initial probe plan. + pub(super) fn initial(relay_map: &RelayMap, if_state: &interfaces::State) -> Self { + let mut plan = Self(BTreeSet::new()); + + // The first time we need add probes after the STUN we record this delay, so that + // further relay server can reuse this delay. + let mut max_stun_delay: Option = None; + + for relay_node in relay_map.nodes() { + let mut stun_ipv4_probes = ProbeSet::new(ProbeProto::StunIpv4); + let mut stun_ipv6_probes = ProbeSet::new(ProbeProto::StunIpv6); + let mut quic_ipv4_probes = ProbeSet::new(ProbeProto::QuicAddrIpv4); + let mut quic_ipv6_probes = ProbeSet::new(ProbeProto::QuicAddrIpv6); + + for attempt in 0..3 { + let delay = DEFAULT_INITIAL_RETRANSMIT * attempt as u32; + + if if_state.have_v4 { + stun_ipv4_probes + .push(Probe::StunIpv4 { + delay, + node: relay_node.clone(), + }) + .expect("adding StunIpv4 probe to a StunIpv4 probe set"); + quic_ipv4_probes + .push(Probe::QuicAddrIpv4 { + delay, + node: relay_node.clone(), + }) + .expect("adding QuicAddrIpv4 probe to a QuicAddrIpv4 probe set"); + } + if if_state.have_v6 { + stun_ipv6_probes + .push(Probe::StunIpv6 { + delay, + node: relay_node.clone(), + }) + .expect("adding StunIpv6 probe to a StunIpv6 probe set"); + quic_ipv6_probes + .push(Probe::QuicAddrIpv6 { + delay, + node: relay_node.clone(), + }) + .expect("adding QuicAddrIpv6 probe to a QuicAddrIpv6 probe set"); + } + } + plan.add(stun_ipv4_probes); + plan.add(stun_ipv6_probes); + plan.add(quic_ipv4_probes); + plan.add(quic_ipv6_probes); + + // The HTTP and ICMP probes only start after the STUN and QUIC probes have had a chance. + let mut https_probes = ProbeSet::new(ProbeProto::Https); + let mut icmp_probes_ipv4 = ProbeSet::new(ProbeProto::IcmpV4); + let mut icmp_probes_ipv6 = ProbeSet::new(ProbeProto::IcmpV6); + for attempt in 0..3 { + let start = *max_stun_delay.get_or_insert_with(|| plan.max_delay()) + + DEFAULT_INITIAL_RETRANSMIT; + let delay = start + DEFAULT_INITIAL_RETRANSMIT * attempt as u32; + + https_probes + .push(Probe::Https { + delay, + node: relay_node.clone(), + }) + .expect("adding Https probe to a Https probe set"); + if if_state.have_v4 { + icmp_probes_ipv4 + .push(Probe::IcmpV4 { + delay, + node: relay_node.clone(), + }) + .expect("adding Icmp probe to an Icmp probe set"); + } + if if_state.have_v6 { + icmp_probes_ipv6 + .push(Probe::IcmpV6 { + delay, + node: relay_node.clone(), + }) + .expect("adding IcmpIpv6 probe to and IcmpIpv6 probe set"); + } + } + plan.add(https_probes); + plan.add(icmp_probes_ipv4); + plan.add(icmp_probes_ipv6); + } + plan + } + + /// Creates a follow up probe plan using a previous net_report report. + pub(super) fn with_last_report( + relay_map: &RelayMap, + if_state: &interfaces::State, + last_report: &Report, + ) -> Self { + if last_report.relay_latency.is_empty() { + return Self::initial(relay_map, if_state); + } + let mut plan = Self(Default::default()); + + // The first time we need add probes after the STUN we record this delay, so that + // further relay servers can reuse this delay. + let mut max_stun_delay: Option = None; + + let had_ipv4 = !last_report.relay_v4_latency.is_empty(); + let had_ipv6 = !last_report.relay_v6_latency.is_empty(); + let had_both = if_state.have_v6 && had_ipv4 && had_ipv6; + let sorted_relays = sort_relays(relay_map, last_report); + for (ri, (url, relay_node)) in sorted_relays.into_iter().enumerate() { + if ri == NUM_INCREMENTAL_RELAYS { + break; + } + let mut do4 = if_state.have_v4; + let mut do6 = if_state.have_v6; + + // By default, each node only gets one STUN packet sent, + // except the fastest two from the previous round. + let mut attempts = 1; + let is_fastest_two = ri < 2; + + if is_fastest_two { + attempts = 2; + } else if had_both { + // For dual stack machines, make the 3rd & slower nodes alternate between + // IPv4 and IPv6 for STUN and ICMP probes. + if ri % 2 == 0 { + (do4, do6) = (true, false); + } else { + (do4, do6) = (false, true); + } + } + if !is_fastest_two && !had_ipv6 { + do6 = false; + } + if Some(url) == last_report.preferred_relay.as_ref() { + // But if we already had a relay home, try extra hard to + // make sure it's there so we don't flip flop around. + attempts = 4; + } + let retransmit_delay = last_report + .relay_latency + .get(url) + .map(|l| l * 120 / 100) // increases latency by 20%, why? + .unwrap_or(DEFAULT_ACTIVE_RETRANSMIT_DELAY); + + let mut stun_ipv4_probes = ProbeSet::new(ProbeProto::StunIpv4); + let mut stun_ipv6_probes = ProbeSet::new(ProbeProto::StunIpv6); + let mut quic_ipv4_probes = ProbeSet::new(ProbeProto::QuicAddrIpv4); + let mut quic_ipv6_probes = ProbeSet::new(ProbeProto::QuicAddrIpv6); + + for attempt in 0..attempts { + let delay = (retransmit_delay * attempt as u32) + + (ACTIVE_RETRANSMIT_EXTRA_DELAY * attempt as u32); + if do4 { + stun_ipv4_probes + .push(Probe::StunIpv4 { + delay, + node: relay_node.clone(), + }) + .expect("Pushing StunIpv4 Probe to StunIpv4 ProbeSet"); + quic_ipv4_probes + .push(Probe::QuicAddrIpv4 { + delay, + node: relay_node.clone(), + }) + .expect("adding QuicAddrIpv4 probe to a QuicAddrIpv4 probe set"); + } + if do6 { + stun_ipv6_probes + .push(Probe::StunIpv6 { + delay, + node: relay_node.clone(), + }) + .expect("Pushing StunIpv6 Probe to StunIpv6 ProbeSet"); + quic_ipv6_probes + .push(Probe::QuicAddrIpv6 { + delay, + node: relay_node.clone(), + }) + .expect("adding QuicAddrIpv6 probe to a QuicAddrIpv6 probe set"); + } + } + plan.add(stun_ipv4_probes); + plan.add(stun_ipv6_probes); + plan.add(quic_ipv4_probes); + plan.add(quic_ipv6_probes); + + // The HTTP and ICMP probes only start after the STUN probes have had a chance. + let mut https_probes = ProbeSet::new(ProbeProto::Https); + let mut icmp_v4_probes = ProbeSet::new(ProbeProto::IcmpV4); + let mut icmp_v6_probes = ProbeSet::new(ProbeProto::IcmpV6); + let start = *max_stun_delay.get_or_insert_with(|| plan.max_delay()); + for attempt in 0..attempts { + let delay = start + + (retransmit_delay * attempt as u32) + + (ACTIVE_RETRANSMIT_EXTRA_DELAY * (attempt as u32 + 1)); + https_probes + .push(Probe::Https { + delay, + node: relay_node.clone(), + }) + .expect("Pushing Https Probe to an Https ProbeSet"); + if do4 { + icmp_v4_probes + .push(Probe::IcmpV4 { + delay, + node: relay_node.clone(), + }) + .expect("Pushing IcmpV4 Probe to an Icmp ProbeSet"); + } + if do6 { + icmp_v6_probes + .push(Probe::IcmpV6 { + delay, + node: relay_node.clone(), + }) + .expect("Pusying IcmpV6 Probe to an IcmpV6 ProbeSet"); + } + } + plan.add(https_probes); + plan.add(icmp_v4_probes); + plan.add(icmp_v6_probes); + } + plan + } + + /// Returns an iterator over the [`ProbeSet`]s in this plan. + pub(super) fn iter(&self) -> impl Iterator { + self.0.iter() + } + + /// Adds a [`ProbeSet`] if it contains probes. + fn add(&mut self, set: ProbeSet) { + if !set.is_empty() { + self.0.insert(set); + } + } + + /// Returns the delay of the last probe in the probe plan. + fn max_delay(&self) -> Duration { + self.0 + .iter() + .flatten() + .map(|probe| probe.delay()) + .max() + .unwrap_or_default() + } +} + +impl fmt::Display for ProbePlan { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "ProbePlan {{")?; + for probe_set in self.0.iter() { + writeln!(f, r#" ProbeSet("{}") {{"#, probe_set.proto)?; + for probe in probe_set.probes.iter() { + writeln!(f, " {probe},")?; + } + writeln!(f, " }}")?; + } + writeln!(f, "}}") + } +} + +impl FromIterator for ProbePlan { + fn from_iter>(iter: T) -> Self { + Self(iter.into_iter().collect()) + } +} + +/// Sorts the nodes in the [`RelayMap`] from fastest to slowest. +/// +/// This uses the latencies from the last report to determine the order. Relay Nodes with no +/// data are at the end. +fn sort_relays<'a>( + relay_map: &'a RelayMap, + last_report: &Report, +) -> Vec<(&'a RelayUrl, &'a Arc)> { + let mut prev: Vec<_> = relay_map.nodes().collect(); + prev.sort_by(|a, b| { + let latencies_a = last_report.relay_latency.get(&a.url); + let latencies_b = last_report.relay_latency.get(&b.url); + match (latencies_a, latencies_b) { + (Some(_), None) => { + // Non-zero sorts before zero. + std::cmp::Ordering::Less + } + (None, Some(_)) => { + // Zero can't sort before anything else. + std::cmp::Ordering::Greater + } + (None, None) => { + // For both empty latencies sort by relay_id. + a.url.cmp(&b.url) + } + (Some(_), Some(_)) => match latencies_a.cmp(&latencies_b) { + std::cmp::Ordering::Equal => a.url.cmp(&b.url), + x => x, + }, + } + }); + + prev.into_iter().map(|n| (&n.url, n)).collect() +} + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + + use super::*; + use crate::{test_utils, RelayLatencies}; + + /// Shorthand which declares a new ProbeSet. + /// + /// `$kind`: The `ProbeProto`. + /// `$node`: Expression which will be an `Arc`. + /// `$delays`: A `Vec` of the delays for this probe. + macro_rules! probeset { + (proto: ProbeProto::$kind:ident, relay: $node:expr, delays: $delays:expr,) => { + ProbeSet { + proto: ProbeProto::$kind, + probes: $delays + .iter() + .map(|delay| Probe::$kind { + delay: *delay, + node: $node, + }) + .collect(), + } + }; + } + + #[tokio::test] + async fn test_initial_probeplan() { + let (_servers, relay_map) = test_utils::relay_map(2).await; + let relay_node_1 = relay_map.nodes().next().unwrap(); + let relay_node_2 = relay_map.nodes().nth(1).unwrap(); + let if_state = interfaces::State::fake(); + let plan = ProbePlan::initial(&relay_map, &if_state); + + let expected_plan: ProbePlan = [ + probeset! { + proto: ProbeProto::StunIpv4, + relay: relay_node_1.clone(), + delays: [Duration::ZERO, + Duration::from_millis(100), + Duration::from_millis(200)], + }, + probeset! { + proto: ProbeProto::StunIpv6, + relay: relay_node_1.clone(), + delays: [Duration::ZERO, + Duration::from_millis(100), + Duration::from_millis(200)], + }, + probeset! { + proto: ProbeProto::Https, + relay: relay_node_1.clone(), + delays: [Duration::from_millis(300), + Duration::from_millis(400), + Duration::from_millis(500)], + }, + probeset! { + proto: ProbeProto::IcmpV4, + relay: relay_node_1.clone(), + delays: [Duration::from_millis(300), + Duration::from_millis(400), + Duration::from_millis(500)], + }, + probeset! { + proto: ProbeProto::IcmpV6, + relay: relay_node_1.clone(), + delays: [Duration::from_millis(300), + Duration::from_millis(400), + Duration::from_millis(500)], + }, + probeset! { + proto: ProbeProto::StunIpv4, + relay: relay_node_2.clone(), + delays: [Duration::ZERO, + Duration::from_millis(100), + Duration::from_millis(200)], + }, + probeset! { + proto: ProbeProto::StunIpv6, + relay: relay_node_2.clone(), + delays: [Duration::ZERO, + Duration::from_millis(100), + Duration::from_millis(200)], + }, + probeset! { + proto: ProbeProto::Https, + relay: relay_node_2.clone(), + delays: [Duration::from_millis(300), + Duration::from_millis(400), + Duration::from_millis(500)], + }, + probeset! { + proto: ProbeProto::IcmpV4, + relay: relay_node_2.clone(), + delays: [Duration::from_millis(300), + Duration::from_millis(400), + Duration::from_millis(500)], + }, + probeset! { + proto: ProbeProto::IcmpV6, + relay: relay_node_2.clone(), + delays: [Duration::from_millis(300), + Duration::from_millis(400), + Duration::from_millis(500)], + }, + ] + .into_iter() + .collect(); + + println!("expected:"); + println!("{expected_plan}"); + println!("actual:"); + println!("{plan}"); + // The readable error: + assert_eq!(plan.to_string(), expected_plan.to_string()); + // Just in case there's a bug in the Display impl: + assert_eq!(plan, expected_plan); + } + + #[tokio::test] + async fn test_plan_with_report() { + let _logging = iroh_test::logging::setup(); + let (_servers, relay_map) = test_utils::relay_map(2).await; + let relay_node_1 = relay_map.nodes().next().unwrap().clone(); + let relay_node_2 = relay_map.nodes().nth(1).unwrap().clone(); + let if_state = interfaces::State::fake(); + + for i in 0..10 { + println!("round {}", i); + let mut latencies = RelayLatencies::new(); + latencies.update_relay(relay_node_1.url.clone(), Duration::from_millis(2)); + latencies.update_relay(relay_node_2.url.clone(), Duration::from_millis(2)); + let last_report = Report { + udp: true, + ipv6: true, + ipv4: true, + ipv6_can_send: true, + ipv4_can_send: true, + os_has_ipv6: true, + icmpv4: None, + icmpv6: None, + mapping_varies_by_dest_ip: Some(false), + mapping_varies_by_dest_ipv6: Some(false), + hair_pinning: Some(true), + portmap_probe: None, + preferred_relay: Some(relay_node_1.url.clone()), + relay_latency: latencies.clone(), + relay_v4_latency: latencies.clone(), + relay_v6_latency: latencies.clone(), + global_v4: None, + global_v6: None, + captive_portal: None, + }; + let plan = ProbePlan::with_last_report(&relay_map, &if_state, &last_report); + let expected_plan: ProbePlan = [ + probeset! { + proto: ProbeProto::StunIpv4, + relay: relay_node_1.clone(), + delays: [Duration::ZERO, + Duration::from_micros(52_400), + Duration::from_micros(104_800), + Duration::from_micros(157_200)], + }, + probeset! { + proto: ProbeProto::StunIpv6, + relay: relay_node_1.clone(), + delays: [Duration::ZERO, + Duration::from_micros(52_400), + Duration::from_micros(104_800), + Duration::from_micros(157_200)], + }, + probeset! { + proto: ProbeProto::Https, + relay: relay_node_1.clone(), + delays: [Duration::from_micros(207_200), + Duration::from_micros(259_600), + Duration::from_micros(312_000), + Duration::from_micros(364_400)], + }, + probeset! { + proto: ProbeProto::IcmpV4, + relay: relay_node_1.clone(), + delays: [Duration::from_micros(207_200), + Duration::from_micros(259_600), + Duration::from_micros(312_000), + Duration::from_micros(364_400)], + }, + probeset! { + proto: ProbeProto::IcmpV6, + relay: relay_node_1.clone(), + delays: [Duration::from_micros(207_200), + Duration::from_micros(259_600), + Duration::from_micros(312_000), + Duration::from_micros(364_400)], + }, + probeset! { + proto: ProbeProto::StunIpv4, + relay: relay_node_2.clone(), + delays: [Duration::ZERO, + Duration::from_micros(52_400)], + }, + probeset! { + proto: ProbeProto::StunIpv6, + relay: relay_node_2.clone(), + delays: [Duration::ZERO, + Duration::from_micros(52_400)], + }, + probeset! { + proto: ProbeProto::Https, + relay: relay_node_2.clone(), + delays: [Duration::from_micros(207_200), + Duration::from_micros(259_600)], + }, + probeset! { + proto: ProbeProto::IcmpV4, + relay: relay_node_2.clone(), + delays: [Duration::from_micros(207_200), + Duration::from_micros(259_600)], + }, + probeset! { + proto: ProbeProto::IcmpV6, + relay: relay_node_2.clone(), + delays: [Duration::from_micros(207_200), + Duration::from_micros(259_600)], + }, + ] + .into_iter() + .collect(); + + println!("{} round", i); + println!("expected:"); + println!("{expected_plan}"); + println!("actual:"); + println!("{plan}"); + // The readable error: + assert_eq!(plan.to_string(), expected_plan.to_string(), "{}", i); + // Just in case there's a bug in the Display impl: + assert_eq!(plan, expected_plan, "{}", i); + } + } + + fn create_last_report( + url_1: &RelayUrl, + latency_1: Option, + url_2: &RelayUrl, + latency_2: Option, + ) -> Report { + let mut latencies = RelayLatencies::new(); + if let Some(latency_1) = latency_1 { + latencies.update_relay(url_1.clone(), latency_1); + } + if let Some(latency_2) = latency_2 { + latencies.update_relay(url_2.clone(), latency_2); + } + Report { + udp: true, + ipv6: true, + ipv4: true, + ipv6_can_send: true, + ipv4_can_send: true, + os_has_ipv6: true, + icmpv4: None, + icmpv6: None, + mapping_varies_by_dest_ip: Some(false), + mapping_varies_by_dest_ipv6: Some(false), + hair_pinning: Some(true), + portmap_probe: None, + preferred_relay: Some(url_1.clone()), + relay_latency: latencies.clone(), + relay_v4_latency: latencies.clone(), + relay_v6_latency: latencies.clone(), + global_v4: None, + global_v6: None, + captive_portal: None, + } + } + + #[tokio::test] + async fn test_relay_sort_two_latencies() { + let _logging = iroh_test::logging::setup(); + let (_servers, relay_map) = test_utils::relay_map(2).await; + let r1 = relay_map.nodes().next().unwrap(); + let r2 = relay_map.nodes().nth(1).unwrap(); + let last_report = create_last_report( + &r1.url, + Some(Duration::from_millis(1)), + &r2.url, + Some(Duration::from_millis(2)), + ); + let sorted: Vec<_> = sort_relays(&relay_map, &last_report) + .iter() + .map(|(url, _reg)| *url) + .collect(); + assert_eq!(sorted, vec![&r1.url, &r2.url]); + } + + #[tokio::test] + async fn test_relay_sort_equal_latencies() { + let _logging = iroh_test::logging::setup(); + let (_servers, relay_map) = test_utils::relay_map(2).await; + let r1 = relay_map.nodes().next().unwrap(); + let r2 = relay_map.nodes().nth(1).unwrap(); + let last_report = create_last_report( + &r1.url, + Some(Duration::from_millis(2)), + &r2.url, + Some(Duration::from_millis(2)), + ); + let sorted: Vec<_> = sort_relays(&relay_map, &last_report) + .iter() + .map(|(url, _)| *url) + .collect(); + assert_eq!(sorted, vec![&r1.url, &r2.url]); + } + + #[tokio::test] + async fn test_relay_sort_missing_latency() { + let (_servers, relay_map) = test_utils::relay_map(2).await; + let r1 = relay_map.nodes().next().unwrap(); + let r2 = relay_map.nodes().nth(1).unwrap(); + + let last_report = + create_last_report(&r1.url, None, &r2.url, Some(Duration::from_millis(2))); + let sorted: Vec<_> = sort_relays(&relay_map, &last_report) + .iter() + .map(|(url, _)| *url) + .collect(); + assert_eq!(sorted, vec![&r2.url, &r1.url]); + + let last_report = + create_last_report(&r1.url, Some(Duration::from_millis(2)), &r2.url, None); + let sorted: Vec<_> = sort_relays(&relay_map, &last_report) + .iter() + .map(|(url, _)| *url) + .collect(); + assert_eq!(sorted, vec![&r1.url, &r2.url]); + } + + #[tokio::test] + async fn test_relay_sort_no_latency() { + let _logging = iroh_test::logging::setup(); + let (_servers, relay_map) = test_utils::relay_map(2).await; + let r1 = relay_map.nodes().next().unwrap(); + let r2 = relay_map.nodes().nth(1).unwrap(); + + let last_report = create_last_report(&r1.url, None, &r2.url, None); + let sorted: Vec<_> = sort_relays(&relay_map, &last_report) + .iter() + .map(|(url, _)| *url) + .collect(); + // sorted by relay url only + assert_eq!(sorted, vec![&r1.url, &r2.url]); + } +} diff --git a/reportgen.rs b/reportgen.rs new file mode 100644 index 00000000000..11a8328ff29 --- /dev/null +++ b/reportgen.rs @@ -0,0 +1,1597 @@ +//! The reportgen actor is responsible for generating a single net_report report. +//! +//! It is implemented as an actor with [`Client`] as handle. +//! +//! The actor starts generating the report as soon as it is created, it does not receive any +//! messages from the client. It follows roughly these steps: +//! +//! - Determines host IPv6 support. +//! - Creates hairpin actor. +//! - Creates portmapper future. +//! - Creates captive portal detection future. +//! - Creates Probe Set futures. +//! - These send messages to the reportgen actor. +//! - Loops driving the futures and handling actor messages: +//! - Disables futures as they are completed or aborted. +//! - Stop if there are no outstanding tasks/futures, or on timeout. +//! - Sends the completed report to the net_report actor. + +use std::{ + future::Future, + net::{IpAddr, SocketAddr}, + pin::Pin, + sync::Arc, + task::{Context, Poll}, + time::Duration, +}; + +use anyhow::{anyhow, bail, Context as _, Result}; +use hickory_resolver::TokioAsyncResolver as DnsResolver; +#[cfg(feature = "metrics")] +use iroh_metrics::inc; +use iroh_relay::{http::RELAY_PROBE_PATH, protos::stun}; +use netwatch::{interfaces, UdpSocket}; +use rand::seq::IteratorRandom; +use tokio::{ + sync::{mpsc, oneshot}, + task::JoinSet, + time::{self, Instant}, +}; +use tokio_util::task::AbortOnDropHandle; +use tracing::{debug, debug_span, error, info_span, trace, warn, Instrument, Span}; +use url::Host; + +#[cfg(feature = "metrics")] +use crate::Metrics; +use crate::{ + self as net_report, + defaults::{DEFAULT_QUIC_PORT, DEFAULT_STUN_PORT}, + dns::ResolverExt, + ping::{PingError, Pinger}, + RelayMap, RelayNode, RelayUrl, Report, +}; + +mod hairpin; +mod probes; + +use probes::{Probe, ProbePlan, ProbeProto}; + +use crate::defaults::timeouts::{ + CAPTIVE_PORTAL_DELAY, CAPTIVE_PORTAL_TIMEOUT, OVERALL_REPORT_TIMEOUT, PROBES_TIMEOUT, +}; + +const ENOUGH_NODES: usize = 3; + +/// Holds the state for a single invocation of [`net_report::Client::get_report`]. +/// +/// Dropping this will cancel the actor and stop the report generation. +#[derive(Debug)] +pub(super) struct Client { + // Addr is currently only used by child actors, so not yet exposed here. + _drop_guard: AbortOnDropHandle<()>, +} + +impl Client { + /// Creates a new actor generating a single report. + /// + /// The actor starts running immediately and only generates a single report, after which + /// it shuts down. Dropping this handle will abort the actor. + pub(super) fn new( + net_report: net_report::Addr, + last_report: Option>, + port_mapper: Option, + relay_map: RelayMap, + stun_sock4: Option>, + stun_sock6: Option>, + quic_addr_disc: Option, + dns_resolver: DnsResolver, + ) -> Self { + let (msg_tx, msg_rx) = mpsc::channel(32); + let addr = Addr { + sender: msg_tx.clone(), + }; + let mut actor = Actor { + msg_tx, + msg_rx, + net_report: net_report.clone(), + last_report, + port_mapper, + relay_map, + stun_sock4, + stun_sock6, + quic_addr_disc, + report: Report::default(), + hairpin_actor: hairpin::Client::new(net_report, addr), + outstanding_tasks: OutstandingTasks::default(), + dns_resolver, + }; + let task = tokio::spawn( + async move { actor.run().await }.instrument(info_span!("reportgen.actor")), + ); + Self { + _drop_guard: AbortOnDropHandle::new(task), + } + } +} + +/// The address of the reportstate [`Actor`]. +/// +/// Unlike the [`Client`] struct itself this is the raw channel to send message over. +/// Keeping this alive will not keep the actor alive, which makes this handy to pass to +/// internal tasks. +#[derive(Debug, Clone)] +pub(super) struct Addr { + sender: mpsc::Sender, +} + +impl Addr { + /// Blocking send to the actor, to be used from a non-actor future. + async fn send(&self, msg: Message) -> Result<(), mpsc::error::SendError> { + trace!( + "sending {:?} to channel with cap {}", + msg, + self.sender.capacity() + ); + self.sender.send(msg).await + } +} + +/// Messages to send to the reportstate [`Actor`]. +#[derive(Debug)] +enum Message { + /// Set the hairpinning availability in the report. + HairpinResult(bool), + /// Check whether executing a probe would still help. + // TODO: Ideally we remove the need for this message and the logic is inverted: once we + // get a probe result we cancel all probes that are no longer needed. But for now it's + // this way around to ease conversion. + ProbeWouldHelp(Probe, Arc, oneshot::Sender), + /// Abort all remaining probes. + AbortProbes, +} + +/// The reportstate actor. +/// +/// This actor starts, generates a single report and exits. +#[derive(Debug)] +struct Actor { + /// The sender of the message channel, so we can give out [`Addr`]. + msg_tx: mpsc::Sender, + /// The receiver of the message channel. + msg_rx: mpsc::Receiver, + /// The address of the net_report actor. + net_report: super::Addr, + + // Provided state + /// The previous report, if it exists. + last_report: Option>, + /// The portmapper client, if there is one. + port_mapper: Option, + /// The relay configuration. + relay_map: RelayMap, + /// Socket to send IPv4 STUN requests from. + stun_sock4: Option>, + /// Socket so send IPv6 STUN requests from. + stun_sock6: Option>, + /// QUIC configuration to do QUIC address Discovery + quic_addr_disc: Option, + + // Internal state. + /// The report being built. + report: Report, + /// The hairpin actor. + hairpin_actor: hairpin::Client, + /// Which tasks the [`Actor`] is still waiting on. + /// + /// This is essentially the summary of all the work the [`Actor`] is doing. + outstanding_tasks: OutstandingTasks, + /// The DNS resolver to use for probes that need to resolve DNS records. + dns_resolver: DnsResolver, +} + +impl Actor { + fn addr(&self) -> Addr { + Addr { + sender: self.msg_tx.clone(), + } + } + + async fn run(&mut self) { + match self.run_inner().await { + Ok(_) => debug!("reportgen actor finished"), + Err(err) => { + self.net_report + .send(net_report::Message::ReportAborted { err }) + .await + .ok(); + } + } + } + + /// Runs the main reportgen actor logic. + /// + /// This actor runs by: + /// + /// - Creates a hairpin actor. + /// - Creates a captive portal future. + /// - Creates ProbeSet futures in a group of futures. + /// - Runs a main loop: + /// - Drives all the above futures. + /// - Receives actor messages (sent by those futures). + /// - Updates the report, cancels unneeded futures. + /// - Sends the report to the net_report actor. + async fn run_inner(&mut self) -> Result<()> { + debug!( + port_mapper = %self.port_mapper.is_some(), + "reportstate actor starting", + ); + + self.report.os_has_ipv6 = super::os_has_ipv6(); + + let mut port_mapping = self.prepare_portmapper_task(); + let mut captive_task = self.prepare_captive_portal_task(); + let mut probes = self.spawn_probes_task().await?; + + let total_timer = tokio::time::sleep(OVERALL_REPORT_TIMEOUT); + tokio::pin!(total_timer); + let probe_timer = tokio::time::sleep(PROBES_TIMEOUT); + tokio::pin!(probe_timer); + + loop { + trace!(awaiting = ?self.outstanding_tasks, "tick; awaiting tasks"); + if self.outstanding_tasks.all_done() { + debug!("all tasks done"); + break; + } + tokio::select! { + biased; + _ = &mut total_timer => { + trace!("tick: total_timer expired"); + bail!("report timed out"); + } + + _ = &mut probe_timer => { + warn!("tick: probes timed out"); + // Set new timeout to not go into this branch multiple times. We need + // the abort to finish all probes normally. PROBES_TIMEOUT is + // sufficiently far in the future. + probe_timer.as_mut().reset(Instant::now() + PROBES_TIMEOUT); + probes.abort_all(); + self.handle_abort_probes(); + } + + // Drive the portmapper. + pm = &mut port_mapping, if self.outstanding_tasks.port_mapper => { + debug!(report=?pm, "tick: portmapper probe report"); + self.report.portmap_probe = pm; + port_mapping.inner = None; + self.outstanding_tasks.port_mapper = false; + } + + // Check for probes finishing. + set_result = probes.join_next(), if self.outstanding_tasks.probes => { + trace!("tick: probes done: {:?}", set_result); + match set_result { + Some(Ok(Ok(report))) => self.handle_probe_report(report), + Some(Ok(Err(_))) => (), + Some(Err(e)) => { + warn!("probes task error: {:?}", e); + } + None => { + self.handle_abort_probes(); + } + } + trace!("tick: probes handled"); + } + + // Drive the captive task. + found = &mut captive_task, if self.outstanding_tasks.captive_task => { + trace!("tick: captive portal task done"); + self.report.captive_portal = found; + captive_task.inner = None; + self.outstanding_tasks.captive_task = false; + } + + // Handle actor messages. + msg = self.msg_rx.recv() => { + trace!("tick: msg recv: {:?}", msg); + match msg { + Some(msg) => self.handle_message(msg), + None => bail!("msg_rx closed, reportgen client must be dropped"), + } + } + } + } + + if !probes.is_empty() { + debug!( + "aborting {} probe sets, already have enough reports", + probes.len() + ); + drop(probes); + } + + debug!("Sending report to net_report actor"); + self.net_report + .send(net_report::Message::ReportReady { + report: Box::new(self.report.clone()), + }) + .await?; + + Ok(()) + } + + /// Handles an actor message. + /// + /// Returns `true` if all the probes need to be aborted. + fn handle_message(&mut self, msg: Message) { + trace!(?msg, "handling message"); + match msg { + Message::HairpinResult(works) => { + self.report.hair_pinning = Some(works); + self.outstanding_tasks.hairpin = false; + } + Message::ProbeWouldHelp(probe, relay_node, response_tx) => { + let res = self.probe_would_help(probe, relay_node); + if response_tx.send(res).is_err() { + debug!("probe dropped before ProbeWouldHelp response sent"); + } + } + Message::AbortProbes => { + self.handle_abort_probes(); + } + } + } + + fn handle_probe_report(&mut self, probe_report: ProbeReport) { + debug!(?probe_report, "finished probe"); + update_report(&mut self.report, probe_report); + + // When we discover the first IPv4 address we want to start the hairpin actor. + if let Some(ref addr) = self.report.global_v4 { + if !self.hairpin_actor.has_started() { + self.hairpin_actor.start_check(*addr); + self.outstanding_tasks.hairpin = true; + } + } + + // Once we've heard from enough relay servers (3), start a timer to give up on the other + // probes. The timer's duration is a function of whether this is our initial full + // probe or an incremental one. For incremental ones, wait for the duration of the + // slowest relay. For initial ones, double that. + let enough_relays = std::cmp::min(self.relay_map.len(), ENOUGH_NODES); + if self.report.relay_latency.len() == enough_relays { + let timeout = self.report.relay_latency.max_latency(); + let timeout = match self.last_report.is_some() { + true => timeout, + false => timeout * 2, + }; + let reportcheck = self.addr(); + debug!( + reports=self.report.relay_latency.len(), + delay=?timeout, + "Have enough probe reports, aborting further probes soon", + ); + tokio::spawn( + async move { + time::sleep(timeout).await; + // Because we do this after a timeout it is entirely normal that the + // actor is no longer there by the time we send this message. + reportcheck + .send(Message::AbortProbes) + .await + .map_err(|err| trace!("Failed to abort all probes: {err:#}")) + .ok(); + } + .instrument(Span::current()), + ); + } + } + + /// Whether running this probe would still improve our report. + fn probe_would_help(&mut self, probe: Probe, relay_node: Arc) -> bool { + // If the probe is for a relay we don't yet know about, that would help. + if self.report.relay_latency.get(&relay_node.url).is_none() { + return true; + } + + // If the probe is for IPv6 and we don't yet have an IPv6 report, that would help. + if probe.proto() == ProbeProto::StunIpv6 && self.report.relay_v6_latency.is_empty() { + return true; + } + + // For IPv4, we need at least two IPv4 results overall to + // determine whether we're behind a NAT that shows us as + // different source IPs and/or ports depending on who we're + // talking to. If we don't yet have two results yet + // (`mapping_varies_by_dest_ip` is blank), then another IPv4 probe + // would be good. + if probe.proto() == ProbeProto::StunIpv4 && self.report.mapping_varies_by_dest_ip.is_none() + { + return true; + } + + // Otherwise not interesting. + false + } + + /// Stops further probes. + /// + /// This makes sure that no further probes are run and also cancels the captive portal + /// and portmapper tasks if there were successful probes. Be sure to only handle this + /// after all the required [`ProbeReport`]s have been processed. + fn handle_abort_probes(&mut self) { + trace!("handle abort probes"); + self.outstanding_tasks.probes = false; + if self.report.udp { + self.outstanding_tasks.port_mapper = false; + self.outstanding_tasks.captive_task = false; + } + } + + /// Creates the future which will perform the portmapper task. + /// + /// The returned future will run the portmapper, if enabled, resolving to it's result. + fn prepare_portmapper_task( + &mut self, + ) -> MaybeFuture>>>> { + let mut port_mapping = MaybeFuture::default(); + if let Some(port_mapper) = self.port_mapper.clone() { + port_mapping.inner = Some(Box::pin(async move { + match port_mapper.probe().await { + Ok(Ok(res)) => Some(res), + Ok(Err(err)) => { + debug!("skipping port mapping: {err:?}"); + None + } + Err(recv_err) => { + warn!("skipping port mapping: {recv_err:?}"); + None + } + } + })); + self.outstanding_tasks.port_mapper = true; + } + port_mapping + } + + /// Creates the future which will perform the captive portal check. + fn prepare_captive_portal_task( + &mut self, + ) -> MaybeFuture>>>> { + // If we're doing a full probe, also check for a captive portal. We + // delay by a bit to wait for UDP STUN to finish, to avoid the probe if + // it's unnecessary. + if self.last_report.is_none() { + // Even if we're doing a non-incremental update, we may want to try our + // preferred relay for captive portal detection. + let preferred_relay = self + .last_report + .as_ref() + .and_then(|l| l.preferred_relay.clone()); + + let dns_resolver = self.dns_resolver.clone(); + let dm = self.relay_map.clone(); + self.outstanding_tasks.captive_task = true; + MaybeFuture { + inner: Some(Box::pin(async move { + tokio::time::sleep(CAPTIVE_PORTAL_DELAY).await; + debug!("Captive portal check started after {CAPTIVE_PORTAL_DELAY:?}"); + let captive_portal_check = tokio::time::timeout( + CAPTIVE_PORTAL_TIMEOUT, + check_captive_portal(&dns_resolver, &dm, preferred_relay) + .instrument(debug_span!("captive-portal")), + ); + match captive_portal_check.await { + Ok(Ok(found)) => Some(found), + Ok(Err(err)) => { + let err: Result = err.downcast(); + match err { + Ok(req_err) if req_err.is_connect() => { + debug!("check_captive_portal failed: {req_err:#}"); + } + Ok(req_err) => warn!("check_captive_portal error: {req_err:#}"), + Err(any_err) => warn!("check_captive_portal error: {any_err:#}"), + } + None + } + Err(_) => { + warn!("check_captive_portal timed out"); + None + } + } + })), + } + } else { + self.outstanding_tasks.captive_task = false; + MaybeFuture::default() + } + } + + /// Prepares the future which will run all the probes as per generated ProbePlan. + /// + /// Probes operate like the following: + /// + /// - A future is created for each probe in all probe sets. + /// - All probes in a set are grouped in [`JoinSet`]. + /// - All those probe sets are grouped in one overall [`JoinSet`]. + /// - This future is polled by the main actor loop to make progress. + /// - Once a probe future is polled: + /// - Many probes start with a delay, they sleep during this time. + /// - When a probe starts it first asks the reportgen [`Actor`] if it is still useful + /// to run. If not it aborts the entire probe set. + /// - When a probe finishes, its [`ProbeReport`] is yielded to the reportgen actor. + /// - Probes get aborted in several ways: + /// - A running it can fail and abort the entire probe set if it deems the + /// failure permanent. Probes in a probe set are essentially retries. + /// - Once there are [`ProbeReport`]s from enough nodes, all remaining probes are + /// aborted. That is, the main actor loop stops polling them. + async fn spawn_probes_task(&mut self) -> Result>> { + let if_state = interfaces::State::new().await; + debug!(%if_state, "Local interfaces"); + let plan = match self.last_report { + Some(ref report) => ProbePlan::with_last_report(&self.relay_map, &if_state, report), + None => ProbePlan::initial(&self.relay_map, &if_state), + }; + trace!(%plan, "probe plan"); + + // The pinger is created here so that any sockets that might be bound for it are + // shared between the probes that use it. It binds sockets lazily, so we can always + // create it. + let pinger = Pinger::new(); + + // A collection of futures running probe sets. + let mut probes = JoinSet::default(); + for probe_set in plan.iter() { + let mut set = JoinSet::default(); + for probe in probe_set { + let reportstate = self.addr(); + let stun_sock4 = self.stun_sock4.clone(); + let stun_sock6 = self.stun_sock6.clone(); + let quic_addr_disc = self.quic_addr_disc.clone(); + let relay_node = probe.node().clone(); + let probe = probe.clone(); + let net_report = self.net_report.clone(); + let pinger = pinger.clone(); + let dns_resolver = self.dns_resolver.clone(); + + set.spawn( + run_probe( + reportstate, + stun_sock4, + stun_sock6, + quic_addr_disc, + relay_node, + probe.clone(), + net_report, + pinger, + dns_resolver, + ) + .instrument(debug_span!("run_probe", %probe)), + ); + } + + // Add the probe set to all futures of probe sets. Handle aborting a probe set + // if needed, only normal errors means the set continues. + probes.spawn( + async move { + // Hack because ProbeSet is not it's own type yet. + let mut probe_proto = None; + while let Some(res) = set.join_next().await { + match res { + Ok(Ok(report)) => return Ok(report), + Ok(Err(ProbeError::Error(err, probe))) => { + probe_proto = Some(probe.proto()); + warn!(?probe, "probe failed: {:#}", err); + continue; + } + Ok(Err(ProbeError::AbortSet(err, probe))) => { + debug!(?probe, "probe set aborted: {:#}", err); + set.abort_all(); + return Err(err); + } + Err(err) => { + warn!("fatal probe set error, aborting: {:#}", err); + continue; + } + } + } + warn!(?probe_proto, "no successful probes in ProbeSet"); + Err(anyhow!("All probes in ProbeSet failed")) + } + .instrument(info_span!("probe")), + ); + } + self.outstanding_tasks.probes = true; + + Ok(probes) + } +} + +/// Tasks on which the reportgen [`Actor`] is still waiting. +/// +/// There is no particular progression, e.g. hairpin starts `false`, moves to `true` when a +/// check is started and then becomes `false` again once it is finished. +#[derive(Debug, Default)] +struct OutstandingTasks { + probes: bool, + port_mapper: bool, + captive_task: bool, + hairpin: bool, +} + +impl OutstandingTasks { + fn all_done(&self) -> bool { + !(self.probes || self.port_mapper || self.captive_task || self.hairpin) + } +} + +/// The success result of [`run_probe`]. +#[derive(Debug, Clone)] +struct ProbeReport { + /// Whether we can send IPv4 UDP packets. + ipv4_can_send: bool, + /// Whether we can send IPv6 UDP packets. + ipv6_can_send: bool, + /// Whether we can send ICMPv4 packets, `None` if not checked. + icmpv4: Option, + /// Whether we can send ICMPv6 packets, `None` if not checked. + icmpv6: Option, + /// The latency to the relay node. + latency: Option, + /// The probe that generated this report. + probe: Probe, + /// The discovered public address. + addr: Option, +} + +impl ProbeReport { + fn new(probe: Probe) -> Self { + ProbeReport { + probe, + ipv4_can_send: false, + ipv6_can_send: false, + icmpv4: None, + icmpv6: None, + latency: None, + addr: None, + } + } +} + +/// Errors for [`run_probe`]. +/// +/// The main purpose is to signal whether other probes in this probe set should still be +/// run. Recall that a probe set is normally a set of identical probes with delays, +/// effectively creating retries, and the first successful probe of a probe set will cancel +/// the others in the set. So this allows an unsuccessful probe to cancel the remainder of +/// the set or not. +#[derive(Debug)] +enum ProbeError { + /// Abort the current set. + AbortSet(anyhow::Error, Probe), + /// Continue the other probes in the set. + Error(anyhow::Error, Probe), +} + +/// Pieces needed to do QUIC address discovery. +#[derive(Debug, Clone)] +pub struct QuicAddressDiscovery { + /// A QUIC Endpoint + pub ep: quinn::Endpoint, + /// A client config. + pub client_config: quinn::ClientConfig, +} + +/// Executes a particular [`Probe`], including using a delayed start if needed. +/// +/// If *stun_sock4* and *stun_sock6* are `None` the STUN probes are disabled. +#[allow(clippy::too_many_arguments)] +async fn run_probe( + reportstate: Addr, + stun_sock4: Option>, + stun_sock6: Option>, + quic_addr_disc: Option, + relay_node: Arc, + probe: Probe, + net_report: net_report::Addr, + pinger: Pinger, + dns_resolver: DnsResolver, +) -> Result { + if !probe.delay().is_zero() { + trace!("delaying probe"); + tokio::time::sleep(probe.delay()).await; + } + debug!("starting probe"); + + let (would_help_tx, would_help_rx) = oneshot::channel(); + if let Err(err) = reportstate + .send(Message::ProbeWouldHelp( + probe.clone(), + relay_node.clone(), + would_help_tx, + )) + .await + { + // this happens on shutdown or if the report is already finished + debug!("Failed to check if probe would help: {err:#}"); + return Err(ProbeError::AbortSet(err.into(), probe.clone())); + } + + if !would_help_rx.await.map_err(|_| { + ProbeError::AbortSet( + anyhow!("ReportCheck actor dropped sender while waiting for ProbeWouldHelp response"), + probe.clone(), + ) + })? { + return Err(ProbeError::AbortSet( + anyhow!("ReportCheck says probe set no longer useful"), + probe, + )); + } + + let relay_addr = get_relay_addr(&dns_resolver, &relay_node, probe.proto()) + .await + .context("no relay node addr") + .map_err(|e| ProbeError::AbortSet(e, probe.clone()))?; + + let mut result = ProbeReport::new(probe.clone()); + match probe { + Probe::StunIpv4 { .. } | Probe::StunIpv6 { .. } => { + let maybe_sock = if matches!(probe, Probe::StunIpv4 { .. }) { + stun_sock4.as_ref() + } else { + stun_sock6.as_ref() + }; + match maybe_sock { + Some(sock) => { + result = run_stun_probe(sock, relay_addr, net_report, probe).await?; + } + None => { + return Err(ProbeError::AbortSet( + anyhow!("No socket for {}, aborting probeset", probe.proto()), + probe.clone(), + )); + } + } + } + Probe::IcmpV4 { .. } | Probe::IcmpV6 { .. } => { + result = run_icmp_probe(probe, relay_addr, pinger).await? + } + Probe::Https { ref node, .. } => { + debug!("sending probe HTTPS"); + match measure_https_latency(&dns_resolver, node, None).await { + Ok((latency, ip)) => { + result.latency = Some(latency); + // We set these IPv4 and IPv6 but they're not really used + // and we don't necessarily set them both. If UDP is blocked + // and both IPv4 and IPv6 are available over TCP, it's basically + // random which fields end up getting set here. + // Since they're not needed, that's fine for now. + match ip { + IpAddr::V4(_) => result.ipv4_can_send = true, + IpAddr::V6(_) => result.ipv6_can_send = true, + } + } + Err(err) => { + warn!("https latency measurement failed: {:?}", err); + } + } + } + Probe::QuicAddrIpv4 { ref node, .. } | Probe::QuicAddrIpv6 { ref node, .. } => { + debug!("sending QUIC address discovery prob"); + let url = node.url.clone(); + match quic_addr_disc { + Some(quic_addr_disc) => { + result = run_quic_probe(quic_addr_disc, url, relay_addr, probe).await? + } + None => { + return Err(ProbeError::AbortSet( + anyhow!("No QUIC endpoint for {}, aborting probeset", probe.proto()), + probe.clone(), + )); + } + } + } + } + + trace!("probe successful"); + Ok(result) +} + +/// Run a QUIC address discovery probe. +// TODO(ramfox): if this probe is aborted, then the connection will never be +// properly, possibly causing errors on the server. +async fn run_quic_probe( + quic_addr_disc: QuicAddressDiscovery, + url: RelayUrl, + relay_addr: SocketAddr, + probe: Probe, +) -> Result { + match probe.proto() { + ProbeProto::QuicAddrIpv4 => debug_assert!(relay_addr.is_ipv4()), + ProbeProto::QuicAddrIpv6 => debug_assert!(relay_addr.is_ipv6()), + _ => debug_assert!(false, "wrong probe"), + } + // TODO(ramfox): what to put here if no host is given? + let host = url.host_str().unwrap_or("localhost"); + let quic_client = + iroh_relay::quic::QuicClient::new(quic_addr_disc.ep, quic_addr_disc.client_config); + let (addr, latency) = quic_client + .get_addr_and_latency(relay_addr, host) + .await + .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; + let mut result = ProbeReport::new(probe.clone()); + if matches!(probe, Probe::QuicAddrIpv4 { .. }) { + result.ipv4_can_send = true; + } else { + result.ipv6_can_send = true; + } + result.addr = Some(addr); + result.latency = Some(latency); + Ok(result) +} + +/// Run a STUN IPv4 or IPv6 probe. +async fn run_stun_probe( + sock: &Arc, + relay_addr: SocketAddr, + net_report: net_report::Addr, + probe: Probe, +) -> Result { + match probe.proto() { + ProbeProto::StunIpv4 => debug_assert!(relay_addr.is_ipv4()), + ProbeProto::StunIpv6 => debug_assert!(relay_addr.is_ipv6()), + _ => debug_assert!(false, "wrong probe"), + } + let txid = stun::TransactionId::default(); + let req = stun::request(txid); + + // Setup net_report to give us back the incoming STUN response. + let (stun_tx, stun_rx) = oneshot::channel(); + let (inflight_ready_tx, inflight_ready_rx) = oneshot::channel(); + net_report + .send(net_report::Message::InFlightStun( + net_report::Inflight { + txn: txid, + start: Instant::now(), + s: stun_tx, + }, + inflight_ready_tx, + )) + .await + .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; + inflight_ready_rx + .await + .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; + + // Send the probe. + match sock.send_to(&req, relay_addr).await { + Ok(n) if n == req.len() => { + debug!(%relay_addr, %txid, "sending {} probe", probe.proto()); + let mut result = ProbeReport::new(probe.clone()); + + if matches!(probe, Probe::StunIpv4 { .. }) { + result.ipv4_can_send = true; + #[cfg(feature = "metrics")] + inc!(Metrics, stun_packets_sent_ipv4); + } else { + result.ipv6_can_send = true; + #[cfg(feature = "metrics")] + inc!(Metrics, stun_packets_sent_ipv6); + } + let (delay, addr) = stun_rx + .await + .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; + result.latency = Some(delay); + result.addr = Some(addr); + Ok(result) + } + Ok(n) => { + let err = anyhow!("Failed to send full STUN request: {}", probe.proto()); + error!(%relay_addr, sent_len=n, req_len=req.len(), "{err:#}"); + Err(ProbeError::Error(err, probe.clone())) + } + Err(err) => { + let kind = err.kind(); + let err = anyhow::Error::new(err) + .context(format!("Failed to send STUN request: {}", probe.proto())); + + // It is entirely normal that we are on a dual-stack machine with no + // routed IPv6 network. So silence that case. + // NetworkUnreachable and HostUnreachable are still experimental (io_error_more + // #86442) but it is already emitted. So hack around this. + match format!("{kind:?}").as_str() { + "NetworkUnreachable" | "HostUnreachable" => { + debug!(%relay_addr, "{err:#}"); + Err(ProbeError::AbortSet(err, probe.clone())) + } + _ => { + // No need to log this, our caller does already log this. + Err(ProbeError::Error(err, probe.clone())) + } + } + } + } +} + +/// Reports whether or not we think the system is behind a +/// captive portal, detected by making a request to a URL that we know should +/// return a "204 No Content" response and checking if that's what we get. +/// +/// The boolean return is whether we think we have a captive portal. +async fn check_captive_portal( + dns_resolver: &DnsResolver, + dm: &RelayMap, + preferred_relay: Option, +) -> Result { + // If we have a preferred relay node and we can use it for non-STUN requests, try that; + // otherwise, pick a random one suitable for non-STUN requests. + let preferred_relay = preferred_relay.and_then(|url| match dm.get_node(&url) { + Some(node) if node.stun_only => Some(url), + _ => None, + }); + + let url = match preferred_relay { + Some(url) => url, + None => { + let urls: Vec<_> = dm + .nodes() + .filter(|n| !n.stun_only) + .map(|n| n.url.clone()) + .collect(); + if urls.is_empty() { + debug!("No suitable relay node for captive portal check"); + return Ok(false); + } + + let i = (0..urls.len()) + .choose(&mut rand::thread_rng()) + .unwrap_or_default(); + urls[i].clone() + } + }; + + let mut builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); + if let Some(Host::Domain(domain)) = url.host() { + // Use our own resolver rather than getaddrinfo + // + // Be careful, a non-zero port will override the port in the URI. + // + // Ideally we would try to resolve **both** IPv4 and IPv6 rather than purely race + // them. But our resolver doesn't support that yet. + let addrs: Vec<_> = dns_resolver + .lookup_ipv4_ipv6_staggered(domain) + .await? + .map(|ipaddr| SocketAddr::new(ipaddr, 0)) + .collect(); + builder = builder.resolve_to_addrs(domain, &addrs); + } + let client = builder.build()?; + + // Note: the set of valid characters in a challenge and the total + // length is limited; see is_challenge_char in bin/iroh-relay for more + // details. + + let host_name = url.host_str().unwrap_or_default(); + let challenge = format!("ts_{}", host_name); + let portal_url = format!("http://{}/generate_204", host_name); + let res = client + .request(reqwest::Method::GET, portal_url) + .header("X-Tailscale-Challenge", &challenge) + .send() + .await?; + + let expected_response = format!("response {challenge}"); + let is_valid_response = res + .headers() + .get("X-Tailscale-Response") + .map(|s| s.to_str().unwrap_or_default()) + == Some(&expected_response); + + debug!( + "check_captive_portal url={} status_code={} valid_response={}", + res.url(), + res.status(), + is_valid_response, + ); + let has_captive = res.status() != 204 || !is_valid_response; + + Ok(has_captive) +} + +fn get_port(relay_node: &RelayNode, proto: &ProbeProto) -> u16 { + match proto { + ProbeProto::QuicAddrIpv4 | ProbeProto::QuicAddrIpv6 => { + if relay_node.quic_port == 0 { + DEFAULT_QUIC_PORT + } else { + relay_node.quic_port + } + } + _ => { + if relay_node.stun_port == 0 { + DEFAULT_STUN_PORT + } else { + relay_node.stun_port + } + } + } +} + +/// Returns the IP address to use to communicate to this relay node. +/// +/// *proto* specifies the protocol of the probe. Depending on the protocol we may return +/// different results. Obviously IPv4 vs IPv6 but a [`RelayNode`] may also have disabled +/// some protocols. +async fn get_relay_addr( + dns_resolver: &DnsResolver, + relay_node: &RelayNode, + proto: ProbeProto, +) -> Result { + let mut port: u16 = 0; + if relay_node.stun_only && !matches!(proto, ProbeProto::StunIpv4 | ProbeProto::StunIpv6) { + bail!("Relay node not suitable for non-STUN probes"); + + } + if relay_node.quic_only && !matches!(proto, ProbeProto::QuicAddrIpv4 | ProbeProto::QuicAddrIpv6) + { + bail!("Relay node not suitable for non-QUIC address discovery probes"); + } + + match proto { + ProbeProto::StunIpv4 | ProbeProto::IcmpV4 | ProbeProto::QuicAddrIpv4 => { + match relay_node.url.host() { + Some(url::Host::Domain(hostname)) => { + debug!(?proto, %hostname, "Performing DNS A lookup for relay addr"); + match dns_resolver.lookup_ipv4_staggered(hostname).await { + Ok(mut addrs) => addrs + .next() + .map(|ip| ip.to_canonical()) + .map(|addr| SocketAddr::new(addr, port)) + .ok_or(anyhow!("No suitable relay addr found")), + Err(err) => Err(err.context("No suitable relay addr found")), + } + } + Some(url::Host::Ipv4(addr)) => Ok(SocketAddr::new(addr.into(), port)), + Some(url::Host::Ipv6(_addr)) => Err(anyhow!("No suitable relay addr found")), + None => Err(anyhow!("No valid hostname in RelayUrl")), + } + } + + ProbeProto::StunIpv6 | ProbeProto::IcmpV6 | ProbeProto::QuicAddrIpv6 => { + match relay_node.url.host() { + Some(url::Host::Domain(hostname)) => { + debug!(?proto, %hostname, "Performing DNS AAAA lookup for relay addr"); + match dns_resolver.lookup_ipv6_staggered(hostname).await { + Ok(mut addrs) => addrs + .next() + .map(|ip| ip.to_canonical()) + .map(|addr| SocketAddr::new(addr, port)) + .ok_or(anyhow!("No suitable relay addr found")), + Err(err) => Err(err.context("No suitable relay addr found")), + } + } + Some(url::Host::Ipv4(_addr)) => Err(anyhow!("No suitable relay addr found")), + Some(url::Host::Ipv6(addr)) => Ok(SocketAddr::new(addr.into(), port)), + None => Err(anyhow!("No valid hostname in RelayUrl")), + } + } + + ProbeProto::Https => Err(anyhow!("Not implemented")), + } +} + +/// Runs an ICMP IPv4 or IPv6 probe. +/// +/// The `pinger` is passed in so the ping sockets are only bound once +/// for the probe set. +async fn run_icmp_probe( + probe: Probe, + relay_addr: SocketAddr, + pinger: Pinger, +) -> Result { + match probe.proto() { + ProbeProto::IcmpV4 => debug_assert!(relay_addr.is_ipv4()), + ProbeProto::IcmpV6 => debug_assert!(relay_addr.is_ipv6()), + _ => debug_assert!(false, "wrong probe"), + } + const DATA: &[u8; 15] = b"iroh icmp probe"; + debug!(dst = %relay_addr, len = DATA.len(), "ICMP Ping started"); + let latency = pinger + .send(relay_addr.ip(), DATA) + .await + .map_err(|err| match err { + PingError::Client(err) => ProbeError::AbortSet( + anyhow!("Failed to create pinger ({err:#}), aborting probeset"), + probe.clone(), + ), + PingError::Ping(err) => ProbeError::Error(err.into(), probe.clone()), + })?; + debug!(dst = %relay_addr, len = DATA.len(), ?latency, "ICMP ping done"); + let mut report = ProbeReport::new(probe); + report.latency = Some(latency); + match relay_addr { + SocketAddr::V4(_) => { + report.ipv4_can_send = true; + report.icmpv4 = Some(true); + } + SocketAddr::V6(_) => { + report.ipv6_can_send = true; + report.icmpv6 = Some(true); + } + } + Ok(report) +} + +/// Executes an HTTPS probe. +/// +/// If `certs` is provided they will be added to the trusted root certificates, allowing the +/// use of self-signed certificates for servers. Currently this is used for testing. +#[allow(clippy::unused_async)] +async fn measure_https_latency( + dns_resolver: &DnsResolver, + node: &RelayNode, + certs: Option>>, +) -> Result<(Duration, IpAddr)> { + let url = node.url.join(RELAY_PROBE_PATH)?; + + // This should also use same connection establishment as relay client itself, which + // needs to be more configurable so users can do more crazy things: + // https://github.com/n0-computer/iroh/issues/2901 + let mut builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); + if let Some(Host::Domain(domain)) = url.host() { + // Use our own resolver rather than getaddrinfo + // + // Be careful, a non-zero port will override the port in the URI. + // + // The relay Client uses `.lookup_ipv4_ipv6` to connect, so use the same function + // but staggered for reliability. Ideally this tries to resolve **both** IPv4 and + // IPv6 though. But our resolver does not have a function for that yet. + let addrs: Vec<_> = dns_resolver + .lookup_ipv4_ipv6_staggered(domain) + .await? + .map(|ipaddr| SocketAddr::new(ipaddr, 0)) + .collect(); + builder = builder.resolve_to_addrs(domain, &addrs); + } + if let Some(certs) = certs { + for cert in certs { + let cert = reqwest::Certificate::from_der(&cert)?; + builder = builder.add_root_certificate(cert); + } + } + let client = builder.build()?; + + let start = Instant::now(); + let mut response = client.request(reqwest::Method::GET, url).send().await?; + let latency = start.elapsed(); + if response.status().is_success() { + // Drain the response body to be nice to the server, up to a limit. + const MAX_BODY_SIZE: usize = 8 << 10; // 8 KiB + let mut body_size = 0; + while let Some(chunk) = response.chunk().await? { + body_size += chunk.len(); + if body_size >= MAX_BODY_SIZE { + break; + } + } + + // Only `None` if a different hyper HttpConnector in the request. + let remote_ip = response + .remote_addr() + .context("missing HttpInfo from HttpConnector")? + .ip(); + Ok((latency, remote_ip)) + } else { + Err(anyhow!( + "Error response from server: '{}'", + response.status().canonical_reason().unwrap_or_default() + )) + } +} + +/// Updates a net_report [`Report`] with a new [`ProbeReport`]. +fn update_report(report: &mut Report, probe_report: ProbeReport) { + let relay_node = probe_report.probe.node(); + if let Some(latency) = probe_report.latency { + report + .relay_latency + .update_relay(relay_node.url.clone(), latency); + + if matches!( + probe_report.probe.proto(), + ProbeProto::StunIpv4 + | ProbeProto::StunIpv6 + | ProbeProto::QuicAddrIpv4 + | ProbeProto::QuicAddrIpv6 + ) { + report.udp = true; + + match probe_report.addr { + Some(SocketAddr::V4(ipp)) => { + report.ipv4 = true; + report + .relay_v4_latency + .update_relay(relay_node.url.clone(), latency); + if report.global_v4.is_none() { + report.global_v4 = Some(ipp); + } else if report.global_v4 != Some(ipp) { + report.mapping_varies_by_dest_ip = Some(true); + } else if report.mapping_varies_by_dest_ip.is_none() { + report.mapping_varies_by_dest_ip = Some(false); + } + } + Some(SocketAddr::V6(ipp)) => { + report.ipv6 = true; + report + .relay_v6_latency + .update_relay(relay_node.url.clone(), latency); + if report.global_v6.is_none() { + report.global_v6 = Some(ipp); + } else if report.global_v6 != Some(ipp) { + report.mapping_varies_by_dest_ipv6 = Some(true); + warn!("IPv6 Address detected by STUN varies by destination"); + } else if report.mapping_varies_by_dest_ipv6.is_none() { + report.mapping_varies_by_dest_ipv6 = Some(false); + } + } + None => { + // If we are here we had a relay server latency reported from a STUN probe. + // Thus we must have a reported address. + debug_assert!(probe_report.addr.is_some()); + } + } + } + } + report.ipv4_can_send |= probe_report.ipv4_can_send; + report.ipv6_can_send |= probe_report.ipv6_can_send; + report.icmpv4 = report + .icmpv4 + .map(|val| val || probe_report.icmpv4.unwrap_or_default()) + .or(probe_report.icmpv4); + report.icmpv6 = report + .icmpv6 + .map(|val| val || probe_report.icmpv6.unwrap_or_default()) + .or(probe_report.icmpv6); +} + +/// Resolves to pending if the inner is `None`. +#[derive(Debug)] +pub(crate) struct MaybeFuture { + /// Future to be polled. + pub inner: Option, +} + +// NOTE: explicit implementation to bypass derive unnecessary bounds +impl Default for MaybeFuture { + fn default() -> Self { + MaybeFuture { inner: None } + } +} + +impl Future for MaybeFuture { + type Output = T::Output; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match self.inner { + Some(ref mut t) => Pin::new(t).poll(cx), + None => Poll::Pending, + } + } +} + +#[cfg(test)] +mod tests { + use std::net::{Ipv4Addr, Ipv6Addr}; + + use testresult::TestResult; + + use super::{super::test_utils, *}; + + #[tokio::test] + async fn test_update_report_stun_working() { + let _logging = iroh_test::logging::setup(); + let (_server_a, relay_a) = test_utils::relay().await; + let (_server_b, relay_b) = test_utils::relay().await; + + let mut report = Report::default(); + + // A STUN IPv4 probe from the the first relay server. + let probe_report_a = ProbeReport { + ipv4_can_send: true, + ipv6_can_send: false, + icmpv4: None, + icmpv6: None, + latency: Some(Duration::from_millis(5)), + probe: Probe::StunIpv4 { + delay: Duration::ZERO, + node: relay_a.clone(), + }, + addr: Some((Ipv4Addr::new(203, 0, 113, 1), 1234).into()), + }; + update_report(&mut report, probe_report_a.clone()); + + assert!(report.udp); + assert_eq!( + report.relay_latency.get(&relay_a.url).unwrap(), + Duration::from_millis(5) + ); + assert_eq!( + report.relay_v4_latency.get(&relay_a.url).unwrap(), + Duration::from_millis(5) + ); + assert!(report.ipv4_can_send); + assert!(!report.ipv6_can_send); + + // A second STUN IPv4 probe, same external IP detected but slower. + let probe_report_b = ProbeReport { + latency: Some(Duration::from_millis(8)), + probe: Probe::StunIpv4 { + delay: Duration::ZERO, + node: relay_b.clone(), + }, + ..probe_report_a + }; + update_report(&mut report, probe_report_b); + + assert!(report.udp); + assert_eq!( + report.relay_latency.get(&relay_a.url).unwrap(), + Duration::from_millis(5) + ); + assert_eq!( + report.relay_v4_latency.get(&relay_a.url).unwrap(), + Duration::from_millis(5) + ); + assert!(report.ipv4_can_send); + assert!(!report.ipv6_can_send); + + // A STUN IPv6 probe, this one is faster. + let probe_report_a_ipv6 = ProbeReport { + ipv4_can_send: false, + ipv6_can_send: true, + icmpv4: None, + icmpv6: None, + latency: Some(Duration::from_millis(4)), + probe: Probe::StunIpv6 { + delay: Duration::ZERO, + node: relay_a.clone(), + }, + addr: Some((Ipv6Addr::new(2001, 0xdb8, 0, 0, 0, 0, 0, 1), 1234).into()), + }; + update_report(&mut report, probe_report_a_ipv6); + + assert!(report.udp); + assert_eq!( + report.relay_latency.get(&relay_a.url).unwrap(), + Duration::from_millis(4) + ); + assert_eq!( + report.relay_v6_latency.get(&relay_a.url).unwrap(), + Duration::from_millis(4) + ); + assert!(report.ipv4_can_send); + assert!(report.ipv6_can_send); + } + + #[tokio::test] + async fn test_update_report_icmp() { + let _logging = iroh_test::logging::setup(); + let (_server_a, relay_a) = test_utils::relay().await; + let (_server_b, relay_b) = test_utils::relay().await; + + let mut report = Report::default(); + + // An ICMPv4 probe from the EU relay server. + let probe_report_eu = ProbeReport { + ipv4_can_send: true, + ipv6_can_send: false, + icmpv4: Some(true), + icmpv6: None, + latency: Some(Duration::from_millis(5)), + probe: Probe::IcmpV4 { + delay: Duration::ZERO, + node: relay_a.clone(), + }, + addr: Some((Ipv4Addr::new(203, 0, 113, 1), 1234).into()), + }; + update_report(&mut report, probe_report_eu.clone()); + + assert!(!report.udp); + assert!(report.ipv4_can_send); + assert_eq!(report.icmpv4, Some(true)); + + // A second ICMPv4 probe which did not work. + let probe_report_na = ProbeReport { + ipv4_can_send: false, + ipv6_can_send: false, + icmpv4: Some(false), + icmpv6: None, + latency: None, + probe: Probe::IcmpV4 { + delay: Duration::ZERO, + node: relay_b.clone(), + }, + addr: None, + }; + update_report(&mut report, probe_report_na); + + assert_eq!(report.icmpv4, Some(true)); + + // Behold, a STUN probe arrives! + let probe_report_eu_stun = ProbeReport { + ipv4_can_send: true, + ipv6_can_send: false, + icmpv4: None, + icmpv6: None, + latency: Some(Duration::from_millis(5)), + probe: Probe::StunIpv4 { + delay: Duration::ZERO, + node: relay_a.clone(), + }, + addr: Some((Ipv4Addr::new(203, 0, 113, 1), 1234).into()), + }; + update_report(&mut report, probe_report_eu_stun); + + assert!(report.udp); + assert_eq!(report.icmpv4, Some(true)); + } + + // # ICMP permissions on Linux + // + // ## Using capabilities: CAP_NET_RAW + // + // To run ICMP tests on Linux you need CAP_NET_RAW capabilities. When running tests + // this means you first need to build the binary, set the capabilities and finally run + // the tests. + // + // Build the test binary: + // + // cargo nextest run -p iroh_net net_report::reportgen::tests --no-run + // + // Find out the test binary location: + // + // cargo nextest list --message-format json -p iroh-net net_report::reportgen::tests \ + // | jq '."rust-suites"."iroh-net"."binary-path"' | tr -d \" + // + // Set the CAP_NET_RAW permission, note that nextest runs each test in a child process + // so the capabilities need to be inherited: + // + // sudo setcap CAP_NET_RAW=eip target/debug/deps/iroh_net-abc123 + // + // Finally run the test: + // + // cargo nextest run -p iroh_net net_report::reportgen::tests + // + // This allows the pinger to create a SOCK_RAW socket for IPPROTO_ICMP. + // + // + // ## Using sysctl + // + // Now you know the hard way, you can also get this permission a little easier, but + // slightly less secure, by allowing any process running with your group ID to create a + // SOCK_DGRAM for IPPROTO_ICMP. + // + // First find out your group ID: + // + // id --group + // + // Then allow this group to send pings. Note that this is an inclusive range: + // + // sudo sysctl net.ipv4.ping_group_range="1234 1234" + // + // Note that this does not survive a reboot usually, commonly you need to edit + // /etc/sysctl.conf or /etc/sysctl.d/* to persist this across reboots. + // + // TODO: Not sure what about IPv6 pings using sysctl. + #[tokio::test] + async fn test_icmpk_probe() { + let _logging_guard = iroh_test::logging::setup(); + let pinger = Pinger::new(); + let (server, node) = test_utils::relay().await; + let addr = server.stun_addr().expect("test relay serves stun"); + let probe = Probe::IcmpV4 { + delay: Duration::from_secs(0), + node, + }; + + // A single ICMP packet might get lost. Try several and take the first. + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let mut tasks = JoinSet::new(); + for i in 0..8 { + let probe = probe.clone(); + let pinger = pinger.clone(); + let tx = tx.clone(); + tasks.spawn(async move { + time::sleep(Duration::from_millis(i * 100)).await; + let res = run_icmp_probe(probe, addr, pinger).await; + tx.send(res).ok(); + }); + } + let mut last_err = None; + while let Some(res) = rx.recv().await { + match res { + Ok(report) => { + dbg!(&report); + assert_eq!(report.icmpv4, Some(true)); + assert!( + report.latency.expect("should have a latency") > Duration::from_secs(0) + ); + break; + } + Err(ProbeError::Error(err, _probe)) => { + last_err = Some(err); + } + Err(ProbeError::AbortSet(_err, _probe)) => { + // We don't have permission, too bad. + // panic!("no ping permission: {err:#}"); + break; + } + } + } + if let Some(err) = last_err { + panic!("Ping error: {err:#}"); + } + } + + #[tokio::test] + async fn test_measure_https_latency() -> TestResult { + let _logging_guard = iroh_test::logging::setup(); + let (server, relay) = test_utils::relay().await; + let dns_resolver = crate::dns::tests::resolver(); + tracing::info!(relay_url = ?relay.url , "RELAY_URL"); + let (latency, ip) = + measure_https_latency(dns_resolver, &relay, server.certificates()).await?; + + assert!(latency > Duration::ZERO); + + let relay_url_ip = relay + .url + .host_str() + .context("host")? + .parse::()?; + assert_eq!(ip, relay_url_ip); + Ok(()) + } + + #[tokio::test] + async fn test_quic_probe() -> TestResult { + let _logging_guard = iroh_test::logging::setup(); + let (server, relay) = test_utils::relay().await; + let client_config = iroh_relay::client::make_dangerous_client_config(); + let client_config = quinn::ClientConfig::new(Arc::new(client_config)); + let ep = quinn::Endpoint::client(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))?; + let client_addr = ep.local_addr()?; + let quic_addr_disc = QuicAddressDiscovery { + ep: ep.clone(), + client_config, + }; + let url = relay.url.clone(); + let port = server.quic_addr().unwrap().port(); + let probe = Probe::QuicAddrIpv4 { + delay: Duration::from_secs(0), + node: relay.clone(), + }; + let probe = match run_quic_probe( + quic_addr_disc, + url, + (Ipv4Addr::LOCALHOST, port).into(), + probe, + ) + .await + { + Ok(probe) => probe, + Err(e) => match e { + ProbeError::AbortSet(err, _) | ProbeError::Error(err, _) => { + return Err(err.into()); + } + }, + }; + assert!(probe.ipv4_can_send); + assert_eq!(probe.addr.unwrap(), client_addr); + ep.wait_idle().await; + server.shutdown().await?; + Ok(()) + } +} From 96220c9419c10b742276c7b2c2ec485631b2897c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cramfox=E2=80=9D?= <“kasey@n0.computer”> Date: Wed, 11 Dec 2024 00:04:03 -0500 Subject: [PATCH 2/5] adjust magicsock to use new net-report, with QAD still disabled --- iroh-net-report/src/lib.rs | 2 +- iroh/src/magicsock.rs | 4 +- magicsock.rs | 3939 ++++++++++++++++++++++++++++++++++++ 3 files changed, 3943 insertions(+), 2 deletions(-) create mode 100644 magicsock.rs diff --git a/iroh-net-report/src/lib.rs b/iroh-net-report/src/lib.rs index 02a6e66d367..3898a45cf9d 100644 --- a/iroh-net-report/src/lib.rs +++ b/iroh-net-report/src/lib.rs @@ -20,7 +20,6 @@ use iroh_base::relay_map::{RelayMap, RelayNode, RelayUrl}; use iroh_metrics::inc; use iroh_relay::protos::stun; use netwatch::{IpFamily, UdpSocket}; -use reportgen::QuicConfig; use tokio::{ sync::{self, mpsc, oneshot}, time::{Duration, Instant}, @@ -35,6 +34,7 @@ mod ping; mod reportgen; pub use metrics::Metrics; +pub use reportgen::QuicConfig; const FULL_REPORT_INTERVAL: Duration = Duration::from_secs(5 * 60); diff --git a/iroh/src/magicsock.rs b/iroh/src/magicsock.rs index cd4c3067f2c..92b4c5eb7c3 100644 --- a/iroh/src/magicsock.rs +++ b/iroh/src/magicsock.rs @@ -2336,10 +2336,12 @@ impl Actor { let pconn4 = Some(self.pconn4.clone()); let pconn6 = self.pconn6.clone(); + let quic_config = None; + debug!("requesting net_report report"); match self .net_reporter - .get_report_channel(relay_map, pconn4, pconn6) + .get_report_channel(relay_map, pconn4, pconn6, quic_config) .await { Ok(rx) => { diff --git a/magicsock.rs b/magicsock.rs new file mode 100644 index 00000000000..b025cfbaf58 --- /dev/null +++ b/magicsock.rs @@ -0,0 +1,3939 @@ + +//! Implements a socket that can change its communication path while in use, actively searching for the best way to communicate. +//! +//! Based on tailscale/wgengine/magicsock +//! +//! ### `DEV_RELAY_ONLY` env var: +//! When present at *compile time*, this env var will force all packets +//! to be sent over the relay connection, regardless of whether or +//! not we have a direct UDP address for the given node. +//! +//! The intended use is for testing the relay protocol inside the MagicSock +//! to ensure that we can rely on the relay to send packets when two nodes +//! are unable to find direct UDP connections to each other. +//! +//! This also prevent this node from attempting to hole punch and prevents it +//! from responding to any hole punching attempts. This node will still, +//! however, read any packets that come off the UDP sockets. + +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + fmt::Display, + io, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, + pin::Pin, + sync::{ + atomic::{AtomicBool, AtomicU16, AtomicU64, Ordering}, + Arc, RwLock, + }, + task::{Context, Poll, Waker}, + time::{Duration, Instant}, +}; + +use anyhow::{anyhow, Context as _, Result}; +use bytes::Bytes; +use futures_lite::{FutureExt, Stream, StreamExt}; +use futures_util::stream::BoxStream; +use iroh_base::key::NodeId; +use iroh_metrics::{inc, inc_by}; +use iroh_relay::protos::stun; +use netwatch::{interfaces, ip::LocalAddresses, netmon}; +use quinn::AsyncUdpSocket; +use rand::{seq::SliceRandom, Rng, SeedableRng}; +use smallvec::{smallvec, SmallVec}; +use tokio::{ + sync::{self, mpsc, Mutex}, + task::JoinSet, + time, +}; +use tokio_util::sync::CancellationToken; +use tracing::{ + debug, error, error_span, event, info, info_span, instrument, trace, trace_span, warn, + Instrument, Level, Span, +}; +use url::Url; +use watchable::Watchable; + +use self::{ + metrics::Metrics as MagicsockMetrics, + node_map::{NodeMap, PingAction, PingRole, SendPing}, + relay_actor::{RelayActor, RelayActorMessage, RelayReadResult}, + udp_conn::UdpConn, +}; +use crate::{ + defaults::timeouts::NET_REPORT_TIMEOUT, + disco::{self, CallMeMaybe, SendAddr}, + discovery::{Discovery, DiscoveryItem}, + dns::DnsResolver, + endpoint::NodeAddr, + key::{PublicKey, SecretKey, SharedSecret}, + AddrInfo, RelayMap, RelayUrl, +}; + +mod metrics; +mod node_map; +mod relay_actor; +mod timer; +mod udp_conn; + +pub use node_map::Source; + +pub(super) use self::timer::Timer; +pub use self::{ + metrics::Metrics, + node_map::{ConnectionType, ConnectionTypeStream, ControlMsg, DirectAddrInfo, RemoteInfo}, +}; + +/// How long we consider a STUN-derived endpoint valid for. UDP NAT mappings typically +/// expire at 30 seconds, so this is a few seconds shy of that. +const ENDPOINTS_FRESH_ENOUGH_DURATION: Duration = Duration::from_secs(27); + +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); + +/// Contains options for `MagicSock::listen`. +#[derive(derive_more::Debug)] +pub(crate) struct Options { + /// The IPv4 address to listen on. + /// + /// If set to `None` it will choose a random port and listen on `0.0.0.0:0`. + pub(crate) addr_v4: Option, + /// The IPv6 address to listen on. + /// + /// If set to `None` it will choose a random port and listen on `[::]:0`. + pub(crate) addr_v6: Option, + + /// Secret key for this node. + pub(crate) secret_key: SecretKey, + + /// The [`RelayMap`] to use, leave empty to not use a relay server. + pub(crate) relay_map: RelayMap, + + /// An optional [`NodeMap`], to restore information about nodes. + pub(crate) node_map: Option>, + + /// Optional node discovery mechanism. + pub(crate) discovery: Option>, + + /// A DNS resolver to use for resolving relay URLs. + /// + /// You can use [`crate::dns::default_resolver`] for a resolver that uses the system's DNS + /// configuration. + pub(crate) dns_resolver: DnsResolver, + + /// Proxy configuration. + pub(crate) proxy_url: Option, + + /// Skip verification of SSL certificates from relay servers + /// + /// May only be used in tests. + #[cfg(any(test, feature = "test-utils"))] + #[cfg_attr(iroh_docsrs, doc(cfg(any(test, feature = "test-utils"))))] + pub(crate) insecure_skip_relay_cert_verify: bool, +} + +impl Default for Options { + fn default() -> Self { + Options { + addr_v4: None, + addr_v6: None, + secret_key: SecretKey::generate(), + relay_map: RelayMap::empty(), + node_map: None, + discovery: None, + proxy_url: None, + dns_resolver: crate::dns::default_resolver().clone(), + #[cfg(any(test, feature = "test-utils"))] + insecure_skip_relay_cert_verify: false, + } + } +} + +/// Contents of a relay message. Use a SmallVec to avoid allocations for the very +/// common case of a single packet. +type RelayContents = SmallVec<[Bytes; 1]>; + +/// Handle for [`MagicSock`]. +/// +/// Dereferences to [`MagicSock`], and handles closing. +#[derive(Clone, Debug, derive_more::Deref)] +pub(crate) struct Handle { + #[deref(forward)] + msock: Arc, + // Empty when closed + actor_tasks: Arc>>, +} + +/// Iroh connectivity layer. +/// +/// This is responsible for routing packets to nodes based on node IDs, it will initially +/// route packets via a relay and transparently try and establish a node-to-node +/// connection and upgrade to it. It will also keep looking for better connections as the +/// network details of both nodes change. +/// +/// It is usually only necessary to use a single [`MagicSock`] instance in an application, it +/// means any QUIC endpoints on top will be sharing as much information about nodes as +/// possible. +#[derive(derive_more::Debug)] +pub(crate) struct MagicSock { + actor_sender: mpsc::Sender, + relay_actor_sender: mpsc::Sender, + /// String representation of the node_id of this node. + me: String, + /// Proxy + proxy_url: Option, + + /// Used for receiving relay messages. + relay_recv_receiver: parking_lot::Mutex>, + /// Stores wakers, to be called when relay_recv_ch receives new data. + network_recv_wakers: parking_lot::Mutex>, + network_send_wakers: Arc>>, + + /// The DNS resolver to be used in this magicsock. + dns_resolver: DnsResolver, + + /// Key for this node. + secret_key: SecretKey, + + /// Cached version of the Ipv4 and Ipv6 addrs of the current connection. + local_addrs: std::sync::RwLock<(SocketAddr, Option)>, + + /// Preferred port from `Options::port`; 0 means auto. + port: AtomicU16, + + /// Close is in progress (or done) + closing: AtomicBool, + /// Close was called. + closed: AtomicBool, + /// If the last net_report report, reports IPv6 to be available. + ipv6_reported: Arc, + + /// None (or zero nodes) means relay is disabled. + relay_map: RelayMap, + /// Nearest relay node ID; 0 means none/unknown. + my_relay: Watchable>, + /// Tracks the networkmap node entity for each node discovery key. + node_map: NodeMap, + /// UDP IPv4 socket + pconn4: UdpConn, + /// UDP IPv6 socket + pconn6: Option, + /// NetReport client + net_reporter: net_report::Addr, + /// Handle to the underlying quinn::Endpoint. + /// + /// Used in netcheck for QUIC address discovery. + quic_endpoint: Arc>>, + /// The state for an active DiscoKey. + disco_secrets: DiscoSecrets, + + /// UDP disco (ping) queue + udp_disco_sender: mpsc::Sender<(SocketAddr, PublicKey, disco::Message)>, + + /// Optional discovery service + discovery: Option>, + + /// Our discovered direct addresses. + direct_addrs: DiscoveredDirectAddrs, + + /// List of CallMeMaybe disco messages that should be sent out after the next endpoint update + /// completes + pending_call_me_maybes: parking_lot::Mutex>, + + /// Indicates the direct addr update state. + direct_addr_update_state: DirectAddrUpdateState, + + /// Skip verification of SSL certificates from relay servers + /// + /// May only be used in tests. + #[cfg(any(test, feature = "test-utils"))] + #[cfg_attr(iroh_docsrs, doc(cfg(any(test, feature = "test-utils"))))] + insecure_skip_relay_cert_verify: bool, +} + +impl MagicSock { + /// Creates a magic [`MagicSock`] listening on [`Options::addr_v4`] and [`Options::addr_v6`]. + pub(crate) async fn spawn(opts: Options) -> Result { + Handle::new(opts).await + } + + /// Returns the relay node we are connected to, that has the best latency. + /// + /// If `None`, then we are not connected to any relay nodes. + pub(crate) fn my_relay(&self) -> Option { + self.my_relay.get() + } + + /// Get the current proxy configuration. + pub(crate) fn proxy_url(&self) -> Option<&Url> { + self.proxy_url.as_ref() + } + + /// Sets the relay node with the best latency. + /// + /// If we are not connected to any relay nodes, set this to `None`. + fn set_my_relay(&self, my_relay: Option) -> Option { + self.my_relay.replace(my_relay) + } + + /// Sets the internal `quinn::Endpoint` that is used for QUIC address + /// discovery. + pub(crate) fn set_quic_endpoint(&self, endpoint: Option) { + let mut ep = self + .quic_endpoint + .write() + .expect("MagicSock::endpoint RwLock is poisoned"); + *ep = endpoint; + } + + fn is_closing(&self) -> bool { + self.closing.load(Ordering::Relaxed) + } + + fn is_closed(&self) -> bool { + self.closed.load(Ordering::SeqCst) + } + + fn public_key(&self) -> PublicKey { + self.secret_key.public() + } + + /// Get the cached version of the Ipv4 and Ipv6 addrs of the current connection. + pub(crate) fn local_addr(&self) -> (SocketAddr, Option) { + *self.local_addrs.read().expect("not poisoned") + } + + /// Returns `true` if we have at least one candidate address where we can send packets to. + pub(crate) fn has_send_address(&self, node_key: PublicKey) -> bool { + self.remote_info(node_key) + .map(|info| info.has_send_address()) + .unwrap_or(false) + } + + /// Return the [`RemoteInfo`]s of all nodes in the node map. + pub(crate) fn list_remote_infos(&self) -> Vec { + self.node_map.list_remote_infos(Instant::now()) + } + + /// Return the [`RemoteInfo`] for a single node in the node map. + pub(crate) fn remote_info(&self, node_id: NodeId) -> Option { + self.node_map.remote_info(node_id) + } + + /// Returns the direct addresses as a stream. + /// + /// The [`MagicSock`] continuously monitors the direct addresses, the network addresses + /// it might be able to be contacted on, for changes. Whenever changes are detected + /// this stream will yield a new list of addresses. + /// + /// Upon the first creation on the [`MagicSock`] it may not yet have completed a first + /// direct addresses discovery, in this case the first item of the stream will not be + /// immediately available. Once this first set of direct addresses are discovered the + /// stream will always return the first set of addresses immediately, which are the most + /// recently discovered addresses. + /// + /// To get the current direct addresses, drop the stream after the first item was + /// received. + pub(crate) fn direct_addresses(&self) -> DirectAddrsStream { + self.direct_addrs.updates_stream() + } + + /// Watch for changes to the home relay. + /// + /// Note that this can be used to wait for the initial home relay to be known. If the home + /// relay is known at this point, it will be the first item in the stream. + pub(crate) fn watch_home_relay(&self) -> impl Stream { + let current = futures_lite::stream::iter(self.my_relay()); + let changes = self + .my_relay + .watch() + .into_stream() + .filter_map(|maybe_relay| maybe_relay); + current.chain(changes) + } + + /// Returns a stream that reports the [`ConnectionType`] we have to the + /// given `node_id`. + /// + /// The `NodeMap` continuously monitors the `node_id`'s endpoint for + /// [`ConnectionType`] changes, and sends the latest [`ConnectionType`] + /// on the stream. + /// + /// The current [`ConnectionType`] will the the initial entry on the stream. + /// + /// # Errors + /// + /// Will return an error if there is no address information known about the + /// given `node_id`. + pub(crate) fn conn_type_stream(&self, node_id: NodeId) -> Result { + self.node_map.conn_type_stream(node_id) + } + + /// Returns the socket address which can be used by the QUIC layer to dial this node. + pub(crate) fn get_mapping_addr(&self, node_id: NodeId) -> Option { + self.node_map.get_quic_mapped_addr_for_node_key(node_id) + } + + /// Add addresses for a node to the magic socket's addresbook. + #[instrument(skip_all, fields(me = %self.me))] + pub fn add_node_addr(&self, mut addr: NodeAddr, source: node_map::Source) -> Result<()> { + let mut pruned = 0; + for my_addr in self.direct_addrs.sockaddrs() { + if addr.info.direct_addresses.remove(&my_addr) { + warn!( node_id=addr.node_id.fmt_short(), %my_addr, %source, "not adding our addr for node"); + pruned += 1; + } + } + if !addr.info.is_empty() { + self.node_map.add_node_addr(addr, source); + Ok(()) + } else if pruned != 0 { + Err(anyhow::anyhow!( + "empty addressing info, {pruned} direct addresses have been pruned" + )) + } else { + Err(anyhow::anyhow!("empty addressing info")) + } + } + + /// Stores a new set of direct addresses. + /// + /// If the direct addresses have changed from the previous set, they are published to + /// discovery. + pub(super) fn store_direct_addresses(&self, addrs: BTreeSet) { + let updated = self.direct_addrs.update(addrs); + if updated { + self.node_map + .on_direct_addr_discovered(self.direct_addrs.sockaddrs()); + self.publish_my_addr(); + } + } + + /// Get a reference to the DNS resolver used in this [`MagicSock`]. + pub(crate) fn dns_resolver(&self) -> &DnsResolver { + &self.dns_resolver + } + + /// Reference to optional discovery service + pub(crate) fn discovery(&self) -> Option<&dyn Discovery> { + self.discovery.as_ref().map(Box::as_ref) + } + + /// Call to notify the system of potential network changes. + pub(crate) async fn network_change(&self) { + self.actor_sender + .send(ActorMessage::NetworkChange) + .await + .ok(); + } + + #[cfg(test)] + async fn force_network_change(&self, is_major: bool) { + self.actor_sender + .send(ActorMessage::ForceNetworkChange(is_major)) + .await + .ok(); + } + + #[cfg_attr(windows, allow(dead_code))] + fn normalized_local_addr(&self) -> io::Result { + let (v4, v6) = self.local_addr(); + let addr = if let Some(v6) = v6 { v6 } else { v4 }; + Ok(addr) + } + + fn create_io_poller(&self) -> Pin> { + // To do this properly the MagicSock would need a registry of pollers. For each + // node we would look up the poller or create one. Then on each try_send we can + // look up the correct poller and configure it to poll the paths it needs. + // + // Note however that the current quinn impl calls UdpPoller::poll_writable() + // **before** it calls try_send(), as opposed to how it is documented. That is a + // problem as we would not yet know the path that needs to be polled. To avoid such + // ambiguity the API could be changed to a .poll_send(&self, cx: &mut Context, + // io_poller: Pin<&mut dyn UdpPoller>, transmit: &Transmit) -> Poll> + // instead of the existing .try_send() because then we would have control over this. + // + // Right now however we have one single poller behaving the same for each + // connection. It checks all paths and returns Poll::Ready as soon as any path is + // ready. + let ipv4_poller = Arc::new(self.pconn4.clone()).create_io_poller(); + let ipv6_poller = self + .pconn6 + .as_ref() + .map(|sock| Arc::new(sock.clone()).create_io_poller()); + let relay_sender = self.relay_actor_sender.clone(); + Box::pin(IoPoller { + ipv4_poller, + ipv6_poller, + relay_sender, + relay_send_waker: self.network_send_wakers.clone(), + }) + } + + /// Implementation for AsyncUdpSocket::try_send + #[instrument(skip_all)] + fn try_send(&self, transmit: &quinn_udp::Transmit) -> io::Result<()> { + inc_by!(MagicsockMetrics, send_data, transmit.contents.len() as _); + + if self.is_closed() { + inc_by!( + MagicsockMetrics, + send_data_network_down, + transmit.contents.len() as _ + ); + return Err(io::Error::new( + io::ErrorKind::NotConnected, + "connection closed", + )); + } + + let dest = QuicMappedAddr(transmit.destination); + trace!( + dst = %dest, + src = ?transmit.src_ip, + len = %transmit.contents.len(), + "sending", + ); + let mut transmit = transmit.clone(); + match self + .node_map + .get_send_addrs(dest, self.ipv6_reported.load(Ordering::Relaxed)) + { + Some((node_id, udp_addr, relay_url, msgs)) => { + let mut pings_sent = false; + // If we have pings to send, we *have* to send them out first. + if !msgs.is_empty() { + if let Err(err) = self.try_send_ping_actions(msgs) { + warn!( + node = %node_id.fmt_short(), + "failed to handle ping actions: {err:#}", + ); + } + pings_sent = true; + } + + let mut udp_sent = false; + let mut udp_error = None; + let mut relay_sent = false; + let mut relay_error = None; + + // send udp + if let Some(addr) = udp_addr { + // rewrite target address + transmit.destination = addr; + match self.try_send_udp(addr, &transmit) { + Ok(()) => { + trace!(node = %node_id.fmt_short(), dst = %addr, + "sent transmit over UDP"); + udp_sent = true; + } + Err(err) => { + error!(node = %node_id.fmt_short(), dst = %addr, + "failed to send udp: {err:#}"); + udp_error = Some(err); + } + } + } + + // send relay + if let Some(ref relay_url) = relay_url { + match self.try_send_relay(relay_url, node_id, split_packets(&transmit)) { + Ok(()) => { + relay_sent = true; + } + Err(err) => { + relay_error = Some(err); + } + } + } + + let udp_pending = udp_error + .as_ref() + .map(|err| err.kind() == io::ErrorKind::WouldBlock) + .unwrap_or_default(); + let relay_pending = relay_error + .as_ref() + .map(|err| err.kind() == io::ErrorKind::WouldBlock) + .unwrap_or_default(); + if udp_pending && relay_pending { + // Handle backpressure. + Err(io::Error::new(io::ErrorKind::WouldBlock, "pending")) + } else { + if relay_sent || udp_sent { + trace!( + node = %node_id.fmt_short(), + send_udp = ?udp_addr, + send_relay = ?relay_url, + "sent transmit", + ); + } else if !pings_sent { + // Returning Ok here means we let QUIC handle a timeout for a lost + // packet, same would happen if we returned any errors. The + // philosophy of quinn-udp is that a UDP connection could come back + // at any time so these errors should be treated as transient and + // are just timeouts. Hence we opt for returning Ok. See + // test_try_send_no_udp_addr_or_relay_url to explore this further. + error!( + node = %node_id.fmt_short(), + "no UDP or relay paths available for node", + ); + } + Ok(()) + } + } + None => { + error!(%dest, "no NodeState for mapped address"); + // Returning Ok here means we let QUIC timeout. Returning WouldBlock + // triggers a hot loop. Returning an error would immediately fail a + // connection. The philosophy of quinn-udp is that a UDP connection could + // come back at any time or missing should be transient so chooses to let + // these kind of errors time out. See test_try_send_no_send_addr to try + // this out. + Ok(()) + } + } + } + + fn try_send_relay( + &self, + url: &RelayUrl, + node: NodeId, + contents: RelayContents, + ) -> io::Result<()> { + trace!( + node = %node.fmt_short(), + relay_url = %url, + count = contents.len(), + len = contents.iter().map(|c| c.len()).sum::(), + "send relay", + ); + let msg = RelayActorMessage::Send { + url: url.clone(), + contents, + remote_node: node, + }; + match self.relay_actor_sender.try_send(msg) { + Ok(_) => { + trace!(node = %node.fmt_short(), relay_url = %url, + "send relay: message queued"); + Ok(()) + } + Err(mpsc::error::TrySendError::Closed(_)) => { + warn!(node = %node.fmt_short(), relay_url = %url, + "send relay: message dropped, channel to actor is closed"); + Err(io::Error::new( + io::ErrorKind::ConnectionReset, + "channel to actor is closed", + )) + } + Err(mpsc::error::TrySendError::Full(_)) => { + warn!(node = %node.fmt_short(), relay_url = %url, + "send relay: message dropped, channel to actor is full"); + Err(io::Error::new( + io::ErrorKind::WouldBlock, + "channel to actor is full", + )) + } + } + } + + fn try_send_udp(&self, addr: SocketAddr, transmit: &quinn_udp::Transmit) -> io::Result<()> { + let conn = self.conn_for_addr(addr)?; + conn.try_send(transmit)?; + let total_bytes: u64 = transmit.contents.len() as u64; + if addr.is_ipv6() { + inc_by!(MagicsockMetrics, send_ipv6, total_bytes); + } else { + inc_by!(MagicsockMetrics, send_ipv4, total_bytes); + } + Ok(()) + } + + fn conn_for_addr(&self, addr: SocketAddr) -> io::Result<&UdpConn> { + let sock = match addr { + SocketAddr::V4(_) => &self.pconn4, + SocketAddr::V6(_) => self + .pconn6 + .as_ref() + .ok_or(io::Error::new(io::ErrorKind::Other, "no IPv6 connection"))?, + }; + Ok(sock) + } + + /// NOTE: Receiving on a [`Self::closed`] socket will return [`Poll::Pending`] indefinitely. + #[instrument(skip_all)] + fn poll_recv( + &self, + cx: &mut Context, + bufs: &mut [io::IoSliceMut<'_>], + metas: &mut [quinn_udp::RecvMeta], + ) -> Poll> { + // FIXME: currently ipv4 load results in ipv6 traffic being ignored + debug_assert_eq!(bufs.len(), metas.len(), "non matching bufs & metas"); + if self.is_closed() { + return Poll::Pending; + } + + // order of polling is: UDPv4, UDPv6, relay + let (msgs, from_ipv4) = match self.pconn4.poll_recv(cx, bufs, metas)? { + Poll::Pending | Poll::Ready(0) => match &self.pconn6 { + Some(conn) => match conn.poll_recv(cx, bufs, metas)? { + Poll::Pending | Poll::Ready(0) => { + return self.poll_recv_relay(cx, bufs, metas); + } + Poll::Ready(n) => (n, false), + }, + None => { + return self.poll_recv_relay(cx, bufs, metas); + } + }, + Poll::Ready(n) => (n, true), + }; + + // Adding the IP address we received something on results in Quinn using this + // address on the send path to send from. However we let Quinn use a + // QuicMappedAddress, not a real address. So we used to substitute our bind address + // here so that Quinn would send on the right address. But that would sometimes + // result in the wrong address family and Windows trips up on that. + // + // What should be done is that this dst_ip from the RecvMeta is stored in the + // NodeState/PathState. Then on the send path it should be retrieved from the + // NodeState/PathSate together with the send address and substituted at send time. + // This is relevant for IPv6 link-local addresses where the OS otherwise does not + // know which intervace to send from. + #[cfg(not(windows))] + let dst_ip = self.normalized_local_addr().ok().map(|addr| addr.ip()); + // Reasoning for this here: + // https://github.com/n0-computer/iroh/pull/2595#issuecomment-2290947319 + #[cfg(windows)] + let dst_ip = None; + + let mut quic_packets_total = 0; + + for (meta, buf) in metas.iter_mut().zip(bufs.iter_mut()).take(msgs) { + let mut is_quic = false; + let mut quic_packets_count = 0; + if meta.len > meta.stride { + trace!(%meta.len, %meta.stride, "GRO datagram received"); + inc!(MagicsockMetrics, recv_gro_datagrams); + } + + // find disco and stun packets and forward them to the actor + for packet in buf[..meta.len].chunks_mut(meta.stride) { + if packet.len() < meta.stride { + trace!( + len = %packet.len(), + %meta.stride, + "Last GRO datagram smaller than stride", + ); + } + + let packet_is_quic = if stun::is(packet) { + trace!(src = %meta.addr, len = %meta.stride, "UDP recv: stun packet"); + let packet2 = Bytes::copy_from_slice(packet); + self.net_reporter.receive_stun_packet(packet2, meta.addr); + false + } else if let Some((sender, sealed_box)) = disco::source_and_box(packet) { + // Disco? + trace!(src = %meta.addr, len = %meta.stride, "UDP recv: disco packet"); + self.handle_disco_message( + sender, + sealed_box, + DiscoMessageSource::Udp(meta.addr), + ); + false + } else { + trace!(src = %meta.addr, len = %meta.stride, "UDP recv: quic packet"); + if from_ipv4 { + inc_by!(MagicsockMetrics, recv_data_ipv4, packet.len() as _); + } else { + inc_by!(MagicsockMetrics, recv_data_ipv6, packet.len() as _); + } + true + }; + + if packet_is_quic { + quic_packets_count += 1; + is_quic = true; + } else { + // overwrite the first byte of the packets with zero. + // this makes quinn reliably and quickly ignore the packet as long as + // [`quinn::EndpointConfig::grease_quic_bit`] is set to `false` + // (which we always do in Endpoint::bind). + packet[0] = 0u8; + } + } + + if is_quic { + // remap addr + match self.node_map.receive_udp(meta.addr) { + None => { + warn!(src = ?meta.addr, count = %quic_packets_count, len = meta.len, "UDP recv quic packets: no node state found, skipping"); + // if we have no node state for the from addr, set len to 0 to make quinn skip the buf completely. + meta.len = 0; + } + Some((node_id, quic_mapped_addr)) => { + trace!(src = ?meta.addr, node = %node_id.fmt_short(), count = %quic_packets_count, len = meta.len, "UDP recv quic packets"); + quic_packets_total += quic_packets_count; + meta.addr = quic_mapped_addr.0; + } + } + } else { + // if there is no non-stun,non-disco packet in the chunk, set len to zero to make + // quinn skip the buf completely. + meta.len = 0; + } + // Normalize local_ip + meta.dst_ip = dst_ip; + } + + if quic_packets_total > 0 { + inc_by!(MagicsockMetrics, recv_datagrams, quic_packets_total as _); + trace!("UDP recv: {} packets", quic_packets_total); + } + + Poll::Ready(Ok(msgs)) + } + + #[instrument(skip_all)] + fn poll_recv_relay( + &self, + cx: &mut Context, + bufs: &mut [io::IoSliceMut<'_>], + metas: &mut [quinn_udp::RecvMeta], + ) -> Poll> { + let mut num_msgs = 0; + for (buf_out, meta_out) in bufs.iter_mut().zip(metas.iter_mut()) { + if self.is_closed() { + break; + } + let mut relay_recv_receiver = self.relay_recv_receiver.lock(); + match relay_recv_receiver.try_recv() { + Err(mpsc::error::TryRecvError::Empty) => { + self.network_recv_wakers.lock().replace(cx.waker().clone()); + break; + } + Err(mpsc::error::TryRecvError::Disconnected) => { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::NotConnected, + "connection closed", + ))); + } + Ok(Err(err)) => return Poll::Ready(Err(err)), + Ok(Ok((node_id, meta, bytes))) => { + inc_by!(MagicsockMetrics, recv_data_relay, bytes.len() as _); + trace!(src = %meta.addr, node = %node_id.fmt_short(), count = meta.len / meta.stride, len = meta.len, "recv quic packets from relay"); + buf_out[..bytes.len()].copy_from_slice(&bytes); + *meta_out = meta; + num_msgs += 1; + } + } + } + + // If we have any msgs to report, they are in the first `num_msgs_total` slots + if num_msgs > 0 { + inc_by!(MagicsockMetrics, recv_datagrams, num_msgs as _); + Poll::Ready(Ok(num_msgs)) + } else { + Poll::Pending + } + } + + /// Handles a discovery message. + #[instrument("disco_in", skip_all, fields(node = %sender.fmt_short(), %src))] + fn handle_disco_message(&self, sender: PublicKey, sealed_box: &[u8], src: DiscoMessageSource) { + trace!("handle_disco_message start"); + if self.is_closed() { + return; + } + + // We're now reasonably sure we're expecting communication from + // this node, do the heavy crypto lifting to see what they want. + let dm = match self.disco_secrets.unseal_and_decode( + &self.secret_key, + sender, + sealed_box.to_vec(), + ) { + Ok(dm) => dm, + Err(DiscoBoxError::Open(err)) => { + warn!(?err, "failed to open disco box"); + inc!(MagicsockMetrics, recv_disco_bad_key); + return; + } + Err(DiscoBoxError::Parse(err)) => { + // Couldn't parse it, but it was inside a correctly + // signed box, so just ignore it, assuming it's from a + // newer version of Tailscale that we don't + // understand. Not even worth logging about, lest it + // be too spammy for old clients. + + inc!(MagicsockMetrics, recv_disco_bad_parse); + debug!(?err, "failed to parse disco message"); + return; + } + }; + + if src.is_relay() { + inc!(MagicsockMetrics, recv_disco_relay); + } else { + inc!(MagicsockMetrics, recv_disco_udp); + } + + let span = trace_span!("handle_disco", ?dm); + let _guard = span.enter(); + trace!("receive disco message"); + match dm { + disco::Message::Ping(ping) => { + inc!(MagicsockMetrics, recv_disco_ping); + self.handle_ping(ping, sender, src); + } + disco::Message::Pong(pong) => { + inc!(MagicsockMetrics, recv_disco_pong); + self.node_map.handle_pong(sender, &src, pong); + } + disco::Message::CallMeMaybe(cm) => { + inc!(MagicsockMetrics, recv_disco_call_me_maybe); + match src { + DiscoMessageSource::Relay { url, .. } => { + event!( + target: "events.net.call-me-maybe.recv", + Level::DEBUG, + remote_node = sender.fmt_short(), + via = ?url, + their_addrs = ?cm.my_numbers, + ); + } + _ => { + warn!("call-me-maybe packets should only come via relay"); + return; + } + } + let ping_actions = self.node_map.handle_call_me_maybe(sender, cm); + for action in ping_actions { + match action { + PingAction::SendCallMeMaybe { .. } => { + warn!("Unexpected CallMeMaybe as response of handling a CallMeMaybe"); + } + PingAction::SendPing(ping) => { + self.send_ping_queued(ping); + } + } + } + } + } + trace!("disco message handled"); + } + + /// Handle a ping message. + fn handle_ping(&self, dm: disco::Ping, sender: NodeId, src: DiscoMessageSource) { + // Insert the ping into the node map, and return whether a ping with this tx_id was already + // received. + let addr: SendAddr = src.clone().into(); + let handled = self.node_map.handle_ping(sender, addr.clone(), dm.tx_id); + match handled.role { + PingRole::Duplicate => { + debug!(%src, tx = %hex::encode(dm.tx_id), "received ping: path already confirmed, skip"); + return; + } + PingRole::LikelyHeartbeat => {} + PingRole::NewPath => { + debug!(%src, tx = %hex::encode(dm.tx_id), "received ping: new path"); + } + PingRole::Activate => { + debug!(%src, tx = %hex::encode(dm.tx_id), "received ping: path active"); + } + } + + // Send a pong. + debug!(tx = %hex::encode(dm.tx_id), %addr, dstkey = %sender.fmt_short(), + "sending pong"); + let pong = disco::Message::Pong(disco::Pong { + tx_id: dm.tx_id, + ping_observed_addr: addr.clone(), + }); + event!( + target: "events.net.pong.sent", + Level::DEBUG, + remote_node = %sender.fmt_short(), + dst = ?addr, + txn = ?dm.tx_id, + ); + + if !self.send_disco_message_queued(addr.clone(), sender, pong) { + warn!(%addr, "failed to queue pong"); + } + + if let Some(ping) = handled.needs_ping_back { + debug!( + %addr, + dstkey = %sender.fmt_short(), + "sending direct ping back", + ); + self.send_ping_queued(ping); + } + } + + fn encode_disco_message(&self, dst_key: PublicKey, msg: &disco::Message) -> Bytes { + self.disco_secrets + .encode_and_seal(&self.secret_key, dst_key, msg) + } + + fn send_ping_queued(&self, ping: SendPing) { + let SendPing { + id, + dst, + dst_node, + tx_id, + purpose, + } = ping; + let msg = disco::Message::Ping(disco::Ping { + tx_id, + node_key: self.public_key(), + }); + let sent = match dst { + SendAddr::Udp(addr) => self + .udp_disco_sender + .try_send((addr, dst_node, msg)) + .is_ok(), + SendAddr::Relay(ref url) => self.send_disco_message_relay(url, dst_node, msg), + }; + if sent { + let msg_sender = self.actor_sender.clone(); + trace!(%dst, tx = %hex::encode(tx_id), ?purpose, "ping sent (queued)"); + self.node_map + .notify_ping_sent(id, dst, tx_id, purpose, msg_sender); + } else { + warn!(dst = ?dst, tx = %hex::encode(tx_id), ?purpose, "failed to send ping: queues full"); + } + } + + /// Tries to send the ping actions. + /// + /// Note that on failure the (remaining) ping actions are simply dropped. That's bad! + /// The Endpoint will think a full ping was done and not request a new full-ping for a + /// while. We should probably be buffering the pings. + fn try_send_ping_actions(&self, msgs: Vec) -> io::Result<()> { + for msg in msgs { + // Abort sending as soon as we know we are shutting down. + if self.is_closing() || self.is_closed() { + return Ok(()); + } + match msg { + PingAction::SendCallMeMaybe { + ref relay_url, + dst_node, + } => { + self.send_or_queue_call_me_maybe(relay_url, dst_node); + } + PingAction::SendPing(ping) => { + self.try_send_ping(ping)?; + } + } + } + Ok(()) + } + + /// Send a disco message. UDP messages will be queued. + /// + /// If `dst` is [`SendAddr::Relay`], the message will be pushed into the relay client channel. + /// If `dst` is [`SendAddr::Udp`], the message will be pushed into the udp disco send channel. + /// + /// Returns true if the channel had capacity for the message, and false if the message was + /// dropped. + fn send_disco_message_queued( + &self, + dst: SendAddr, + dst_key: PublicKey, + msg: disco::Message, + ) -> bool { + match dst { + SendAddr::Udp(addr) => self.udp_disco_sender.try_send((addr, dst_key, msg)).is_ok(), + SendAddr::Relay(ref url) => self.send_disco_message_relay(url, dst_key, msg), + } + } + + /// Send a disco message. UDP messages will be polled to send directly on the UDP socket. + fn try_send_disco_message( + &self, + dst: SendAddr, + dst_key: PublicKey, + msg: disco::Message, + ) -> io::Result<()> { + match dst { + SendAddr::Udp(addr) => { + self.try_send_disco_message_udp(addr, dst_key, &msg)?; + } + SendAddr::Relay(ref url) => { + self.send_disco_message_relay(url, dst_key, msg); + } + } + Ok(()) + } + + fn send_disco_message_relay(&self, url: &RelayUrl, dst: NodeId, msg: disco::Message) -> bool { + debug!(node = %dst.fmt_short(), %url, %msg, "send disco message (relay)"); + let pkt = self.encode_disco_message(dst, &msg); + inc!(MagicsockMetrics, send_disco_relay); + match self.try_send_relay(url, dst, smallvec![pkt]) { + Ok(()) => { + if let disco::Message::CallMeMaybe(CallMeMaybe { ref my_numbers }) = msg { + event!( + target: "events.net.call-me-maybe.sent", + Level::DEBUG, + remote_node = %dst.fmt_short(), + via = ?url, + addrs = ?my_numbers, + ); + } + inc!(MagicsockMetrics, sent_disco_relay); + disco_message_sent(&msg); + true + } + Err(_) => false, + } + } + + async fn send_disco_message_udp( + &self, + dst: SocketAddr, + dst_node: NodeId, + msg: &disco::Message, + ) -> io::Result<()> { + futures_lite::future::poll_fn(move |cx| { + loop { + match self.try_send_disco_message_udp(dst, dst_node, msg) { + Ok(()) => return Poll::Ready(Ok(())), + Err(err) if err.kind() == io::ErrorKind::WouldBlock => { + // This is the socket .try_send_disco_message_udp used. + let sock = self.conn_for_addr(dst)?; + let sock = Arc::new(sock.clone()); + let mut poller = sock.create_io_poller(); + match poller.as_mut().poll_writable(cx)? { + Poll::Ready(()) => continue, + Poll::Pending => return Poll::Pending, + } + } + Err(err) => return Poll::Ready(Err(err)), + } + } + }) + .await + } + + fn try_send_disco_message_udp( + &self, + dst: SocketAddr, + dst_node: NodeId, + msg: &disco::Message, + ) -> std::io::Result<()> { + trace!(%dst, %msg, "send disco message (UDP)"); + if self.is_closed() { + return Err(io::Error::new( + io::ErrorKind::NotConnected, + "connection closed", + )); + } + let pkt = self.encode_disco_message(dst_node, msg); + // TODO: These metrics will be wrong with the poll impl + // Also - do we need it? I'd say the `sent_disco_udp` below is enough. + inc!(MagicsockMetrics, send_disco_udp); + let transmit = quinn_udp::Transmit { + destination: dst, + contents: &pkt, + ecn: None, + segment_size: None, + src_ip: None, // TODO + }; + let sent = self.try_send_udp(dst, &transmit); + match sent { + Ok(()) => { + trace!(%dst, node = %dst_node.fmt_short(), %msg, "sent disco message"); + inc!(MagicsockMetrics, sent_disco_udp); + disco_message_sent(msg); + Ok(()) + } + Err(err) => { + warn!(%dst, node = %dst_node.fmt_short(), ?msg, ?err, + "failed to send disco message"); + Err(err) + } + } + } + + #[instrument(skip_all)] + async fn handle_ping_actions(&mut self, msgs: Vec) { + // TODO: This used to make sure that all ping actions are sent. Though on the + // poll_send/try_send path we also do fire-and-forget. try_send_ping_actions() + // really should store any unsent pings on the Inner and send them at the next + // possible time. + if let Err(err) = self.try_send_ping_actions(msgs) { + warn!("Not all ping actions were sent: {err:#}"); + } + } + + fn try_send_ping(&self, ping: SendPing) -> io::Result<()> { + let SendPing { + id, + dst, + dst_node, + tx_id, + purpose, + } = ping; + let msg = disco::Message::Ping(disco::Ping { + tx_id, + node_key: self.public_key(), + }); + self.try_send_disco_message(dst.clone(), dst_node, msg)?; + debug!(%dst, tx = %hex::encode(tx_id), ?purpose, "ping sent (polled)"); + let msg_sender = self.actor_sender.clone(); + self.node_map + .notify_ping_sent(id, dst.clone(), tx_id, purpose, msg_sender); + Ok(()) + } + + fn poll_send_relay( + &self, + url: &RelayUrl, + node: PublicKey, + contents: RelayContents, + ) -> Poll { + trace!(node = %node.fmt_short(), relay_url = %url, count = contents.len(), len = contents.iter().map(|c| c.len()).sum::(), "send relay"); + let msg = RelayActorMessage::Send { + url: url.clone(), + contents, + remote_node: node, + }; + match self.relay_actor_sender.try_send(msg) { + Ok(_) => { + trace!(node = %node.fmt_short(), relay_url = %url, "send relay: message queued"); + Poll::Ready(true) + } + Err(mpsc::error::TrySendError::Closed(_)) => { + warn!(node = %node.fmt_short(), relay_url = %url, "send relay: message dropped, channel to actor is closed"); + Poll::Ready(false) + } + Err(mpsc::error::TrySendError::Full(_)) => { + warn!(node = %node.fmt_short(), relay_url = %url, "send relay: message dropped, channel to actor is full"); + Poll::Pending + } + } + } + + fn send_queued_call_me_maybes(&self) { + let msg = self.direct_addrs.to_call_me_maybe_message(); + let msg = disco::Message::CallMeMaybe(msg); + for (public_key, url) in self.pending_call_me_maybes.lock().drain() { + if !self.send_disco_message_relay(&url, public_key, msg.clone()) { + warn!(node = %public_key.fmt_short(), "relay channel full, dropping call-me-maybe"); + } + } + } + + /// Sends the call-me-maybe DISCO message, queuing if addresses are too stale. + /// + /// To send the call-me-maybe message, we need to know our current direct addresses. If + /// this information is too stale, the call-me-maybe is queued while a net_report run is + /// scheduled. Once this run finishes, the call-me-maybe will be sent. + fn send_or_queue_call_me_maybe(&self, url: &RelayUrl, dst_node: NodeId) { + match self.direct_addrs.fresh_enough() { + Ok(()) => { + let msg = self.direct_addrs.to_call_me_maybe_message(); + let msg = disco::Message::CallMeMaybe(msg); + if !self.send_disco_message_relay(url, dst_node, msg) { + warn!(dstkey = %dst_node.fmt_short(), relayurl = %url, + "relay channel full, dropping call-me-maybe"); + } else { + debug!(dstkey = %dst_node.fmt_short(), relayurl = %url, "call-me-maybe sent"); + } + } + Err(last_refresh_ago) => { + self.pending_call_me_maybes + .lock() + .insert(dst_node, url.clone()); + debug!( + ?last_refresh_ago, + "want call-me-maybe but direct addrs stale; queuing after restun", + ); + self.re_stun("refresh-for-peering"); + } + } + } + + /// Triggers an address discovery. The provided why string is for debug logging only. + #[instrument(skip_all)] + fn re_stun(&self, why: &'static str) { + debug!("re_stun: {}", why); + inc!(MagicsockMetrics, re_stun_calls); + self.direct_addr_update_state.schedule_run(why); + } + + /// Publishes our address to a discovery service, if configured. + /// + /// Called whenever our addresses or home relay node changes. + fn publish_my_addr(&self) { + if let Some(ref discovery) = self.discovery { + let info = AddrInfo { + relay_url: self.my_relay(), + direct_addresses: self.direct_addrs.sockaddrs(), + }; + discovery.publish(&info); + } + } +} + +#[derive(Clone, Debug)] +enum DiscoMessageSource { + Udp(SocketAddr), + Relay { url: RelayUrl, key: PublicKey }, +} + +impl Display for DiscoMessageSource { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Udp(addr) => write!(f, "Udp({addr})"), + Self::Relay { ref url, key } => write!(f, "Relay({url}, {})", key.fmt_short()), + } + } +} + +impl From for SendAddr { + fn from(value: DiscoMessageSource) -> Self { + match value { + DiscoMessageSource::Udp(addr) => SendAddr::Udp(addr), + DiscoMessageSource::Relay { url, .. } => SendAddr::Relay(url), + } + } +} + +impl From<&DiscoMessageSource> for SendAddr { + fn from(value: &DiscoMessageSource) -> Self { + match value { + DiscoMessageSource::Udp(addr) => SendAddr::Udp(*addr), + DiscoMessageSource::Relay { url, .. } => SendAddr::Relay(url.clone()), + } + } +} + +impl DiscoMessageSource { + fn is_relay(&self) -> bool { + matches!(self, DiscoMessageSource::Relay { .. }) + } +} + +/// Manages currently running direct addr discovery, aka net_report runs. +/// +/// Invariants: +/// - only one direct addr update must be running at a time +/// - if an update is scheduled while another one is running, remember that +/// and start a new one when the current one has finished +#[derive(Debug)] +struct DirectAddrUpdateState { + /// If running, set to the reason for the currently the update. + running: sync::watch::Sender>, + /// If set, start a new update as soon as the current one is finished. + want_update: parking_lot::Mutex>, +} + +impl DirectAddrUpdateState { + fn new() -> Self { + let (running, _) = sync::watch::channel(None); + DirectAddrUpdateState { + running, + want_update: Default::default(), + } + } + + /// Schedules a new run, either starting it immediately if none is running or + /// scheduling it for later. + fn schedule_run(&self, why: &'static str) { + if self.is_running() { + let _ = self.want_update.lock().insert(why); + } else { + self.run(why); + } + } + + /// Returns `true` if an update is currently in progress. + fn is_running(&self) -> bool { + self.running.borrow().is_some() + } + + /// Trigger a new run. + fn run(&self, why: &'static str) { + self.running.send(Some(why)).ok(); + } + + /// Clears the current running state. + fn finish_run(&self) { + self.running.send(None).ok(); + } + + /// Returns the next update, if one is set. + fn next_update(&self) -> Option<&'static str> { + self.want_update.lock().take() + } +} + +impl Handle { + /// Creates a magic [`MagicSock`] listening on [`Options::addr_v4`] and [`Options::addr_v6`]. + async fn new(opts: Options) -> Result { + let me = opts.secret_key.public().fmt_short(); + if crate::util::relay_only_mode() { + warn!( + "creating a MagicSock that will only send packets over a relay relay connection." + ); + } + + Self::with_name(me, opts) + .instrument(error_span!("magicsock")) + .await + } + + async fn with_name(me: String, opts: Options) -> Result { + let port_mapper = portmapper::Client::default(); + + let Options { + addr_v4, + addr_v6, + secret_key, + relay_map, + node_map, + discovery, + dns_resolver, + proxy_url, + #[cfg(any(test, feature = "test-utils"))] + insecure_skip_relay_cert_verify, + } = opts; + + let (relay_recv_sender, relay_recv_receiver) = mpsc::channel(128); + + let (pconn4, pconn6) = bind(addr_v4, addr_v6)?; + let port = pconn4.port(); + + // NOTE: we can end up with a zero port if `std::net::UdpSocket::socket_addr` fails + match port.try_into() { + Ok(non_zero_port) => { + port_mapper.update_local_port(non_zero_port); + } + Err(_zero_port) => debug!("Skipping port mapping with zero local port"), + } + let ipv4_addr = pconn4.local_addr()?; + let ipv6_addr = pconn6.as_ref().and_then(|c| c.local_addr().ok()); + + let net_reporter = + net_report::Client::new(Some(port_mapper.clone()), dns_resolver.clone())?; + + let (actor_sender, actor_receiver) = mpsc::channel(256); + let (relay_actor_sender, relay_actor_receiver) = mpsc::channel(256); + let (udp_disco_sender, mut udp_disco_receiver) = mpsc::channel(256); + + // load the node data + let node_map = node_map.unwrap_or_default(); + let node_map = NodeMap::load_from_vec(node_map); + + let inner = Arc::new(MagicSock { + me, + port: AtomicU16::new(port), + secret_key, + proxy_url, + local_addrs: std::sync::RwLock::new((ipv4_addr, ipv6_addr)), + closing: AtomicBool::new(false), + closed: AtomicBool::new(false), + relay_recv_receiver: parking_lot::Mutex::new(relay_recv_receiver), + network_recv_wakers: parking_lot::Mutex::new(None), + network_send_wakers: Arc::new(parking_lot::Mutex::new(None)), + actor_sender: actor_sender.clone(), + ipv6_reported: Arc::new(AtomicBool::new(false)), + relay_map, + my_relay: Default::default(), + pconn4: pconn4.clone(), + pconn6: pconn6.clone(), + net_reporter: net_reporter.addr(), + disco_secrets: DiscoSecrets::default(), + node_map, + relay_actor_sender: relay_actor_sender.clone(), + udp_disco_sender, + discovery, + direct_addrs: Default::default(), + pending_call_me_maybes: Default::default(), + direct_addr_update_state: DirectAddrUpdateState::new(), + dns_resolver, + #[cfg(any(test, feature = "test-utils"))] + insecure_skip_relay_cert_verify, + quic_endpoint: Arc::new(RwLock::new(None)), + }); + + let mut actor_tasks = JoinSet::default(); + + let relay_actor = RelayActor::new(inner.clone(), actor_sender.clone()); + let relay_actor_cancel_token = relay_actor.cancel_token(); + actor_tasks.spawn( + async move { + relay_actor.run(relay_actor_receiver).await; + } + .instrument(info_span!("relay-actor")), + ); + + let inner2 = inner.clone(); + actor_tasks.spawn(async move { + while let Some((dst, dst_key, msg)) = udp_disco_receiver.recv().await { + if let Err(err) = inner2.send_disco_message_udp(dst, dst_key, &msg).await { + warn!(%dst, node = %dst_key.fmt_short(), ?err, "failed to send disco message (UDP)"); + } + } + }); + + let inner2 = inner.clone(); + let network_monitor = netmon::Monitor::new().await?; + actor_tasks.spawn( + async move { + let actor = Actor { + msg_receiver: actor_receiver, + msg_sender: actor_sender, + relay_actor_sender, + relay_actor_cancel_token, + msock: inner2, + relay_recv_sender, + periodic_re_stun_timer: new_re_stun_timer(false), + net_info_last: None, + port_mapper, + pconn4, + pconn6, + no_v4_send: false, + net_reporter, + network_monitor, + }; + + if let Err(err) = actor.run().await { + warn!("relay handler errored: {:?}", err); + } + } + .instrument(info_span!("actor")), + ); + + let c = Handle { + msock: inner, + actor_tasks: Arc::new(Mutex::new(actor_tasks)), + }; + + Ok(c) + } + + /// Closes the connection. + /// + /// Only the first close does anything. Any later closes return nil. + /// Polling the socket ([`AsyncUdpSocket::poll_recv`]) will return [`Poll::Pending`] + /// indefinitely after this call. + #[instrument(skip_all, fields(me = %self.msock.me))] + pub(crate) async fn close(&self) -> Result<()> { + if self.msock.is_closed() { + return Ok(()); + } + self.msock.closing.store(true, Ordering::Relaxed); + self.msock.actor_sender.send(ActorMessage::Shutdown).await?; + self.msock.closed.store(true, Ordering::SeqCst); + self.msock.direct_addrs.addrs.shutdown(); + + let mut tasks = self.actor_tasks.lock().await; + + // give the tasks a moment to shutdown cleanly + let tasks_ref = &mut tasks; + let shutdown_done = time::timeout(Duration::from_millis(100), async move { + while let Some(task) = tasks_ref.join_next().await { + if let Err(err) = task { + warn!("unexpected error in task shutdown: {:?}", err); + } + } + }) + .await; + if shutdown_done.is_ok() { + debug!("tasks shutdown complete"); + } else { + // shutdown all tasks + debug!("aborting remaining {}/3 tasks", tasks.len()); + tasks.shutdown().await; + } + + Ok(()) + } +} + +#[derive(Debug, Default)] +struct DiscoSecrets(parking_lot::Mutex>); + +impl DiscoSecrets { + fn get( + &self, + secret: &SecretKey, + node_id: PublicKey, + ) -> parking_lot::MappedMutexGuard { + parking_lot::MutexGuard::map(self.0.lock(), |inner| { + inner + .entry(node_id) + .or_insert_with(|| secret.shared(&node_id)) + }) + } + + pub fn encode_and_seal( + &self, + secret_key: &SecretKey, + node_id: PublicKey, + msg: &disco::Message, + ) -> Bytes { + let mut seal = msg.as_bytes(); + self.get(secret_key, node_id).seal(&mut seal); + disco::encode_message(&secret_key.public(), seal).into() + } + + pub fn unseal_and_decode( + &self, + secret: &SecretKey, + node_id: PublicKey, + mut sealed_box: Vec, + ) -> Result { + self.get(secret, node_id) + .open(&mut sealed_box) + .map_err(DiscoBoxError::Open)?; + disco::Message::from_bytes(&sealed_box).map_err(DiscoBoxError::Parse) + } +} + +#[derive(Debug, thiserror::Error)] +enum DiscoBoxError { + #[error("Failed to open crypto box")] + Open(anyhow::Error), + #[error("Failed to parse disco message")] + Parse(anyhow::Error), +} + +type RelayRecvResult = Result<(PublicKey, quinn_udp::RecvMeta, Bytes), io::Error>; + +impl AsyncUdpSocket for Handle { + fn create_io_poller(self: Arc) -> Pin> { + self.msock.create_io_poller() + } + + fn try_send(&self, transmit: &quinn_udp::Transmit) -> io::Result<()> { + self.msock.try_send(transmit) + } + + /// NOTE: Receiving on a [`Self::close`]d socket will return [`Poll::Pending`] indefinitely. + fn poll_recv( + &self, + cx: &mut Context, + bufs: &mut [io::IoSliceMut<'_>], + metas: &mut [quinn_udp::RecvMeta], + ) -> Poll> { + self.msock.poll_recv(cx, bufs, metas) + } + + fn local_addr(&self) -> io::Result { + match &*self.msock.local_addrs.read().expect("not poisoned") { + (ipv4, None) => { + // Pretend to be IPv6, because our QuinnMappedAddrs + // need to be IPv6. + let ip: IpAddr = match ipv4.ip() { + IpAddr::V4(ip) => ip.to_ipv6_mapped().into(), + IpAddr::V6(ip) => ip.into(), + }; + Ok(SocketAddr::new(ip, ipv4.port())) + } + (_, Some(ipv6)) => Ok(*ipv6), + } + } + + fn max_transmit_segments(&self) -> usize { + if let Some(pconn6) = self.pconn6.as_ref() { + std::cmp::min( + pconn6.max_transmit_segments(), + self.pconn4.max_transmit_segments(), + ) + } else { + self.pconn4.max_transmit_segments() + } + } + + fn max_receive_segments(&self) -> usize { + if let Some(pconn6) = self.pconn6.as_ref() { + // `max_receive_segments` controls the size of the `RecvMeta` buffer + // that quinn creates. Having buffers slightly bigger than necessary + // isn't terrible, and makes sure a single socket can read the maximum + // amount with a single poll. We considered adding these numbers instead, + // but we never get data from both sockets at the same time in `poll_recv` + // and it's impossible and unnecessary to be refactored that way. + std::cmp::max( + pconn6.max_receive_segments(), + self.pconn4.max_receive_segments(), + ) + } else { + self.pconn4.max_receive_segments() + } + } + + fn may_fragment(&self) -> bool { + if let Some(pconn6) = self.pconn6.as_ref() { + pconn6.may_fragment() || self.pconn4.may_fragment() + } else { + self.pconn4.may_fragment() + } + } +} + +#[derive(Debug)] +struct IoPoller { + ipv4_poller: Pin>, + ipv6_poller: Option>>, + relay_sender: mpsc::Sender, + relay_send_waker: Arc>>, +} + +impl quinn::UdpPoller for IoPoller { + fn poll_writable(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + // This version returns Ready as soon as any of them are ready. + let this = &mut *self; + match this.ipv4_poller.as_mut().poll_writable(cx) { + Poll::Ready(_) => return Poll::Ready(Ok(())), + Poll::Pending => (), + } + if let Some(ref mut ipv6_poller) = this.ipv6_poller { + match ipv6_poller.as_mut().poll_writable(cx) { + Poll::Ready(_) => return Poll::Ready(Ok(())), + Poll::Pending => (), + } + } + match this.relay_sender.capacity() { + 0 => { + self.relay_send_waker.lock().replace(cx.waker().clone()); + Poll::Pending + } + _ => Poll::Ready(Ok(())), + } + } +} + +#[derive(Debug)] +enum ActorMessage { + Shutdown, + ReceiveRelay(RelayReadResult), + EndpointPingExpired(usize, stun_rs::TransactionId), + NetReport(Result>>, &'static str), + NetworkChange, + #[cfg(test)] + ForceNetworkChange(bool), +} + +struct Actor { + msock: Arc, + msg_receiver: mpsc::Receiver, + msg_sender: mpsc::Sender, + relay_actor_sender: mpsc::Sender, + relay_actor_cancel_token: CancellationToken, + /// Channel to send received relay messages on, for processing. + relay_recv_sender: mpsc::Sender, + /// When set, is an AfterFunc timer that will call MagicSock::do_periodic_stun. + periodic_re_stun_timer: time::Interval, + /// The `NetInfo` provided in the last call to `net_info_func`. It's used to deduplicate calls to netInfoFunc. + net_info_last: Option, + + // The underlying UDP sockets used to send/rcv packets. + pconn4: UdpConn, + pconn6: Option, + + /// The NAT-PMP/PCP/UPnP prober/client, for requesting port mappings from NAT devices. + port_mapper: portmapper::Client, + + /// Whether IPv4 UDP is known to be unable to transmit + /// at all. This could happen if the socket is in an invalid state + /// (as can happen on darwin after a network link status change). + no_v4_send: bool, + + /// The prober that discovers local network conditions, including the closest relay relay and NAT mappings. + net_reporter: net_report::Client, + + network_monitor: netmon::Monitor, +} + +impl Actor { + async fn run(mut self) -> Result<()> { + // Setup network monitoring + let (link_change_s, mut link_change_r) = mpsc::channel(8); + let _token = self + .network_monitor + .subscribe(move |is_major| { + let link_change_s = link_change_s.clone(); + async move { + link_change_s.send(is_major).await.ok(); + } + .boxed() + }) + .await?; + + // Let the the heartbeat only start a couple seconds later + let mut direct_addr_heartbeat_timer = time::interval_at( + time::Instant::now() + HEARTBEAT_INTERVAL, + HEARTBEAT_INTERVAL, + ); + let mut direct_addr_update_receiver = + self.msock.direct_addr_update_state.running.subscribe(); + let mut portmap_watcher = self.port_mapper.watch_external_address(); + + let mut discovery_events: BoxStream = + Box::pin(futures_lite::stream::empty()); + if let Some(d) = self.msock.discovery() { + if let Some(events) = d.subscribe() { + discovery_events = events; + } + } + + let mut receiver_closed = false; + let mut portmap_watcher_closed = false; + let mut link_change_closed = false; + loop { + inc!(Metrics, actor_tick_main); + tokio::select! { + msg = self.msg_receiver.recv(), if !receiver_closed => { + let Some(msg) = msg else { + trace!("tick: magicsock receiver closed"); + inc!(Metrics, actor_tick_other); + + receiver_closed = true; + continue; + }; + + trace!(?msg, "tick: msg"); + inc!(Metrics, actor_tick_msg); + if self.handle_actor_message(msg).await { + return Ok(()); + } + } + tick = self.periodic_re_stun_timer.tick() => { + trace!("tick: re_stun {:?}", tick); + inc!(Metrics, actor_tick_re_stun); + self.msock.re_stun("periodic"); + } + change = portmap_watcher.changed(), if !portmap_watcher_closed => { + if change.is_err() { + trace!("tick: portmap watcher closed"); + inc!(Metrics, actor_tick_other); + + portmap_watcher_closed = true; + continue; + } + + trace!("tick: portmap changed"); + inc!(Metrics, actor_tick_portmap_changed); + let new_external_address = *portmap_watcher.borrow(); + debug!("external address updated: {new_external_address:?}"); + self.msock.re_stun("portmap_updated"); + }, + _ = direct_addr_heartbeat_timer.tick() => { + trace!( + "tick: direct addr heartbeat {} direct addrs", + self.msock.node_map.node_count(), + ); + inc!(Metrics, actor_tick_direct_addr_heartbeat); + // TODO: this might trigger too many packets at once, pace this + + self.msock.node_map.prune_inactive(); + let msgs = self.msock.node_map.nodes_stayin_alive(); + self.handle_ping_actions(msgs).await; + } + _ = direct_addr_update_receiver.changed() => { + let reason = *direct_addr_update_receiver.borrow(); + trace!("tick: direct addr update receiver {:?}", reason); + inc!(Metrics, actor_tick_direct_addr_update_receiver); + if let Some(reason) = reason { + self.refresh_direct_addrs(reason).await; + } + } + is_major = link_change_r.recv(), if !link_change_closed => { + let Some(is_major) = is_major else { + trace!("tick: link change receiver closed"); + inc!(Metrics, actor_tick_other); + + link_change_closed = true; + continue; + }; + + trace!("tick: link change {}", is_major); + inc!(Metrics, actor_link_change); + self.handle_network_change(is_major).await; + } + // Even if `discovery_events` yields `None`, it could begin to yield + // `Some` again in the future, so we don't want to disable this branch + // forever like we do with the other branches that yield `Option`s + Some(discovery_item) = discovery_events.next() => { + trace!("tick: discovery event, address discovered: {discovery_item:?}"); + let node_addr = NodeAddr {node_id: discovery_item.node_id, info: discovery_item.addr_info}; + if let Err(e) = self.msock.add_node_addr(node_addr.clone(), Source::Discovery { name: discovery_item.provenance.into() }) { + warn!(?node_addr, "unable to add discovered node address to the node map: {e:?}"); + } + } + } + } + } + + async fn handle_network_change(&mut self, is_major: bool) { + debug!("link change detected: major? {}", is_major); + + if is_major { + self.msock.dns_resolver.clear_cache(); + self.msock.re_stun("link-change-major"); + self.close_stale_relay_connections().await; + self.reset_endpoint_states(); + } else { + self.msock.re_stun("link-change-minor"); + } + } + + #[instrument(skip_all)] + async fn handle_ping_actions(&mut self, msgs: Vec) { + // TODO: This used to make sure that all ping actions are sent. Though on the + // poll_send/try_send path we also do fire-and-forget. try_send_ping_actions() + // really should store any unsent pings on the Inner and send them at the next + // possible time. + if let Err(err) = self.msock.try_send_ping_actions(msgs) { + warn!("Not all ping actions were sent: {err:#}"); + } + } + + /// Processes an incoming actor message. + /// + /// Returns `true` if it was a shutdown. + async fn handle_actor_message(&mut self, msg: ActorMessage) -> bool { + match msg { + ActorMessage::Shutdown => { + debug!("shutting down"); + + self.msock.node_map.notify_shutdown(); + self.port_mapper.deactivate(); + self.relay_actor_cancel_token.cancel(); + + // Ignore errors from pconnN + // They will frequently have been closed already by a call to connBind.Close. + debug!("stopping connections"); + if let Some(ref conn) = self.pconn6 { + conn.close().await.ok(); + } + self.pconn4.close().await.ok(); + + debug!("shutdown complete"); + return true; + } + ActorMessage::ReceiveRelay(read_result) => { + let passthroughs = self.process_relay_read_result(read_result); + for passthrough in passthroughs { + self.relay_recv_sender + .send(passthrough) + .await + .expect("missing recv sender"); + let mut wakers = self.msock.network_recv_wakers.lock(); + if let Some(waker) = wakers.take() { + waker.wake(); + } + } + } + ActorMessage::EndpointPingExpired(id, txid) => { + self.msock.node_map.notify_ping_timeout(id, txid); + } + ActorMessage::NetReport(report, why) => { + match report { + Ok(report) => { + self.handle_net_report_report(report).await; + } + Err(err) => { + warn!( + "failed to generate net_report report for: {}: {:?}", + why, err + ); + } + } + self.finalize_direct_addrs_update(why); + } + ActorMessage::NetworkChange => { + self.network_monitor.network_change().await.ok(); + } + #[cfg(test)] + ActorMessage::ForceNetworkChange(is_major) => { + self.handle_network_change(is_major).await; + } + } + + false + } + + #[cfg_attr(windows, allow(dead_code))] + fn normalized_local_addr(&self) -> io::Result { + self.msock.normalized_local_addr() + } + + fn process_relay_read_result(&mut self, dm: RelayReadResult) -> Vec { + trace!("process_relay_read {} bytes", dm.buf.len()); + if dm.buf.is_empty() { + warn!("received empty relay packet"); + return Vec::new(); + } + let url = &dm.url; + + let quic_mapped_addr = self.msock.node_map.receive_relay(url, dm.src); + + // the relay packet is made up of multiple udp packets, prefixed by a u16 be length prefix + // + // split the packet into these parts + let parts = PacketSplitIter::new(dm.buf); + // Normalize local_ip + #[cfg(not(windows))] + let dst_ip = self.normalized_local_addr().ok().map(|addr| addr.ip()); + // Reasoning for this here: https://github.com/n0-computer/iroh/pull/2595#issuecomment-2290947319 + + let mut out = Vec::new(); + for part in parts { + match part { + Ok(part) => { + if self.handle_relay_disco_message(&part, url, dm.src) { + // Message was internal, do not bubble up. + continue; + } + + let meta = quinn_udp::RecvMeta { + len: part.len(), + stride: part.len(), + addr: quic_mapped_addr.0, + dst_ip, + ecn: None, + }; + out.push(Ok((dm.src, meta, part))); + } + Err(e) => { + out.push(Err(e)); + } + } + } + + out + } + + /// Refreshes knowledge about our direct addresses. + /// + /// In other words, this triggers a net_report run. + /// + /// Note that invoking this is managed by the [`DirectAddrUpdateState`] and this should + /// never be invoked directly. Some day this will be refactored to not allow this easy + /// mistake to be made. + #[instrument(level = "debug", skip_all)] + async fn refresh_direct_addrs(&mut self, why: &'static str) { + inc!(MagicsockMetrics, update_direct_addrs); + + debug!("starting direct addr update ({})", why); + self.port_mapper.procure_mapping(); + self.update_net_info(why).await; + } + + /// Updates the direct addresses of this magic socket. + /// + /// Updates the [`DiscoveredDirectAddrs`] of this [`MagicSock`] with the current set of + /// direct addresses from: + /// + /// - The portmapper. + /// - A net_report report. + /// - The local interfaces IP addresses. + fn update_direct_addresses(&mut self, net_report_report: Option>) { + let portmap_watcher = self.port_mapper.watch_external_address(); + + // We only want to have one DirectAddr for each SocketAddr we have. So we store + // this as a map of SocketAddr -> DirectAddrType. At the end we will construct a + // DirectAddr from each entry. + let mut addrs: BTreeMap = BTreeMap::new(); + + // First add PortMapper provided addresses. + let maybe_port_mapped = *portmap_watcher.borrow(); + if let Some(portmap_ext) = maybe_port_mapped.map(SocketAddr::V4) { + addrs + .entry(portmap_ext) + .or_insert(DirectAddrType::Portmapped); + self.set_net_info_have_port_map(); + } + + // Next add STUN addresses from the net_report report. + if let Some(net_report_report) = net_report_report { + if let Some(global_v4) = net_report_report.global_v4 { + addrs + .entry(global_v4.into()) + .or_insert(DirectAddrType::Stun); + + // If they're behind a hard NAT and are using a fixed + // port locally, assume they might've added a static + // port mapping on their router to the same explicit + // port that we are running with. Worst case it's an invalid candidate mapping. + let port = self.msock.port.load(Ordering::Relaxed); + if net_report_report + .mapping_varies_by_dest_ip + .unwrap_or_default() + && port != 0 + { + let mut addr = global_v4; + addr.set_port(port); + addrs + .entry(addr.into()) + .or_insert(DirectAddrType::Stun4LocalPort); + } + } + if let Some(global_v6) = net_report_report.global_v6 { + addrs + .entry(global_v6.into()) + .or_insert(DirectAddrType::Stun); + } + } + + let local_addr_v4 = self.pconn4.local_addr().ok(); + let local_addr_v6 = self.pconn6.as_ref().and_then(|c| c.local_addr().ok()); + + let is_unspecified_v4 = local_addr_v4 + .map(|a| a.ip().is_unspecified()) + .unwrap_or(false); + let is_unspecified_v6 = local_addr_v6 + .map(|a| a.ip().is_unspecified()) + .unwrap_or(false); + + let msock = self.msock.clone(); + + // The following code can be slow, we do not want to block the caller since it would + // block the actor loop. + tokio::spawn( + async move { + // If a socket is bound to the unspecified address, create SocketAddrs for + // each local IP address by pairing it with the port the socket is bound on. + if is_unspecified_v4 || is_unspecified_v6 { + // Depending on the OS and network interfaces attached and their state + // enumerating the local interfaces can take a long time. Especially + // Windows is very slow. + let LocalAddresses { + regular: mut ips, + loopback, + } = tokio::task::spawn_blocking(LocalAddresses::new) + .await + .unwrap(); + if ips.is_empty() && addrs.is_empty() { + // Include loopback addresses only if there are no other interfaces + // or public addresses, this allows testing offline. + ips = loopback; + } + for ip in ips { + let port_if_unspecified = match ip { + IpAddr::V4(_) if is_unspecified_v4 => { + local_addr_v4.map(|addr| addr.port()) + } + IpAddr::V6(_) if is_unspecified_v6 => { + local_addr_v6.map(|addr| addr.port()) + } + _ => None, + }; + if let Some(port) = port_if_unspecified { + let addr = SocketAddr::new(ip, port); + addrs.entry(addr).or_insert(DirectAddrType::Local); + } + } + } + + // If a socket is bound to a specific address, add it. + if !is_unspecified_v4 { + if let Some(addr) = local_addr_v4 { + addrs.entry(addr).or_insert(DirectAddrType::Local); + } + } + if !is_unspecified_v6 { + if let Some(addr) = local_addr_v6 { + addrs.entry(addr).or_insert(DirectAddrType::Local); + } + } + + // Finally create and store store all these direct addresses and send any + // queued call-me-maybe messages. + msock.store_direct_addresses( + addrs + .iter() + .map(|(addr, typ)| DirectAddr { + addr: *addr, + typ: *typ, + }) + .collect(), + ); + msock.send_queued_call_me_maybes(); + } + .instrument(Span::current()), + ); + } + + /// Called when a direct addr update is done, no matter if it was successful or not. + fn finalize_direct_addrs_update(&mut self, why: &'static str) { + let new_why = self.msock.direct_addr_update_state.next_update(); + if !self.msock.is_closed() { + if let Some(new_why) = new_why { + self.msock.direct_addr_update_state.run(new_why); + return; + } + self.periodic_re_stun_timer = new_re_stun_timer(true); + } + + self.msock.direct_addr_update_state.finish_run(); + debug!("direct addr update done ({})", why); + } + + /// Updates `NetInfo.HavePortMap` to true. + #[instrument(level = "debug", skip_all)] + fn set_net_info_have_port_map(&mut self) { + if let Some(ref mut net_info_last) = self.net_info_last { + if net_info_last.have_port_map { + // No change. + return; + } + net_info_last.have_port_map = true; + self.net_info_last = Some(net_info_last.clone()); + } + } + + #[instrument(level = "debug", skip_all)] + async fn call_net_info_callback(&mut self, ni: NetInfo) { + if let Some(ref net_info_last) = self.net_info_last { + if ni.basically_equal(net_info_last) { + return; + } + } + + self.net_info_last = Some(ni); + } + + /// Calls net_report. + /// + /// Note that invoking this is managed by [`DirectAddrUpdateState`] via + /// [`Actor::refresh_direct_addrs`] and this should never be invoked directly. Some day + /// this will be refactored to not allow this easy mistake to be made. + #[instrument(level = "debug", skip_all)] + async fn update_net_info(&mut self, why: &'static str) { + if self.msock.relay_map.is_empty() { + debug!("skipping net_report, empty RelayMap"); + self.msg_sender + .send(ActorMessage::NetReport(Ok(None), why)) + .await + .ok(); + return; + } + + let relay_map = self.msock.relay_map.clone(); + let pconn4 = Some(self.pconn4.as_socket()); + let pconn6 = self.pconn6.as_ref().map(|p| p.as_socket()); + let quic_endpoint = self.msock.quic_endpoint.read().expect("poisoned").clone(); + let quic_addr_disc = match quic_endpoint { + Some(ep) => { + match crate::tls::make_client_config(&self.msock.secret_key, None, vec![], false) { + Ok(client_config) => Some(net_report::QuicAddressDiscovery { + ep, + client_config: quinn::ClientConfig::new(Arc::new(client_config)), + }), + Err(err) => { + warn!("Unable to make a QuicClientConfig to run QUIC address discovery probes in net_report: {err:?}"); + None + } + } + } + None => None, + }; + + debug!("requesting net_report report"); + match self + .net_reporter + .get_report_channel(relay_map, pconn4, pconn6, quic_addr_disc) + .await + { + Ok(rx) => { + let msg_sender = self.msg_sender.clone(); + tokio::task::spawn(async move { + let report = time::timeout(NET_REPORT_TIMEOUT, rx).await; + let report: anyhow::Result<_> = match report { + Ok(Ok(Ok(report))) => Ok(Some(report)), + Ok(Ok(Err(err))) => Err(err), + Ok(Err(_)) => Err(anyhow!("net_report report not received")), + Err(err) => Err(anyhow!("net_report report timeout: {:?}", err)), + }; + msg_sender + .send(ActorMessage::NetReport(report, why)) + .await + .ok(); + // The receiver of the NetReport message will call + // .finalize_direct_addrs_update(). + }); + } + Err(err) => { + warn!("unable to start net_report generation: {:?}", err); + self.finalize_direct_addrs_update(why); + } + } + } + + async fn handle_net_report_report(&mut self, report: Option>) { + if let Some(ref report) = report { + self.msock + .ipv6_reported + .store(report.ipv6, Ordering::Relaxed); + let r = &report; + trace!( + "setting no_v4_send {} -> {}", + self.no_v4_send, + !r.ipv4_can_send + ); + self.no_v4_send = !r.ipv4_can_send; + + let have_port_map = self.port_mapper.watch_external_address().borrow().is_some(); + let mut ni = NetInfo { + relay_latency: Default::default(), + mapping_varies_by_dest_ip: r.mapping_varies_by_dest_ip, + hair_pinning: r.hair_pinning, + portmap_probe: r.portmap_probe.clone(), + have_port_map, + working_ipv6: Some(r.ipv6), + os_has_ipv6: Some(r.os_has_ipv6), + working_udp: Some(r.udp), + working_icmp_v4: r.icmpv4, + working_icmp_v6: r.icmpv6, + preferred_relay: r.preferred_relay.clone(), + }; + for (rid, d) in r.relay_v4_latency.iter() { + ni.relay_latency + .insert(format!("{rid}-v4"), d.as_secs_f64()); + } + for (rid, d) in r.relay_v6_latency.iter() { + ni.relay_latency + .insert(format!("{rid}-v6"), d.as_secs_f64()); + } + + if ni.preferred_relay.is_none() { + // Perhaps UDP is blocked. Pick a deterministic but arbitrary one. + ni.preferred_relay = self.pick_relay_fallback(); + } + + if !self.set_nearest_relay(ni.preferred_relay.clone()) { + ni.preferred_relay = None; + } + + // TODO: set link type + self.call_net_info_callback(ni).await; + } + self.update_direct_addresses(report); + } + + fn set_nearest_relay(&mut self, relay_url: Option) -> bool { + let my_relay = self.msock.my_relay(); + if relay_url == my_relay { + // No change. + return true; + } + let old_relay = self.msock.set_my_relay(relay_url.clone()); + + if let Some(ref relay_url) = relay_url { + inc!(MagicsockMetrics, relay_home_change); + + // On change, notify all currently connected relay servers and + // start connecting to our home relay if we are not already. + info!("home is now relay {}, was {:?}", relay_url, old_relay); + self.msock.publish_my_addr(); + + self.send_relay_actor(RelayActorMessage::SetHome { + url: relay_url.clone(), + }); + } + + true + } + + /// Returns a deterministic relay node to connect to. This is only used if net_report + /// couldn't find the nearest one, for instance, if UDP is blocked and thus STUN + /// latency checks aren't working. + /// + /// If no the [`RelayMap`] is empty, returns `0`. + fn pick_relay_fallback(&self) -> Option { + // TODO: figure out which relay node most of our nodes are using, + // and use that region as our fallback. + // + // If we already had selected something in the past and it has any + // nodes, we want to stay on it. If there are no nodes at all, + // stay on whatever relay we previously picked. If we need to pick + // one and have no node info, pick a node randomly. + // + // We used to do the above for legacy clients, but never updated it for disco. + + let my_relay = self.msock.my_relay(); + if my_relay.is_some() { + return my_relay; + } + + let ids = self.msock.relay_map.urls().collect::>(); + let mut rng = rand::rngs::StdRng::seed_from_u64(0); + ids.choose(&mut rng).map(|c| (*c).clone()) + } + + /// Resets the preferred address for all nodes. + /// This is called when connectivity changes enough that we no longer trust the old routes. + #[instrument(skip_all, fields(me = %self.msock.me))] + fn reset_endpoint_states(&mut self) { + self.msock.node_map.reset_node_states() + } + + /// Tells the relay actor to close stale relay connections. + /// + /// The relay connections who's local endpoints no longer exist after a network change + /// will error out soon enough. Closing them eagerly speeds this up however and allows + /// re-establishing a relay connection faster. + async fn close_stale_relay_connections(&self) { + let ifs = interfaces::State::new().await; + let local_ips = ifs + .interfaces + .values() + .flat_map(|netif| netif.addrs()) + .map(|ipnet| ipnet.addr()) + .collect(); + self.send_relay_actor(RelayActorMessage::MaybeCloseRelaysOnRebind(local_ips)); + } + + fn send_relay_actor(&self, msg: RelayActorMessage) { + match self.relay_actor_sender.try_send(msg) { + Ok(_) => {} + Err(mpsc::error::TrySendError::Closed(_)) => { + warn!("unable to send to relay actor, already closed"); + } + Err(mpsc::error::TrySendError::Full(_)) => { + warn!("dropping message for relay actor, channel is full"); + } + } + } + + fn handle_relay_disco_message( + &mut self, + msg: &[u8], + url: &RelayUrl, + relay_node_src: PublicKey, + ) -> bool { + match disco::source_and_box(msg) { + Some((source, sealed_box)) => { + if relay_node_src != source { + // TODO: return here? + warn!("Received relay disco message from connection for {}, but with message from {}", relay_node_src.fmt_short(), source.fmt_short()); + } + self.msock.handle_disco_message( + source, + sealed_box, + DiscoMessageSource::Relay { + url: url.clone(), + key: relay_node_src, + }, + ); + true + } + None => false, + } + } +} + +fn new_re_stun_timer(initial_delay: bool) -> time::Interval { + // Pick a random duration between 20 and 26 seconds (just under 30s, + // a common UDP NAT timeout on Linux,etc) + let mut rng = rand::thread_rng(); + let d: Duration = rng.gen_range(Duration::from_secs(20)..=Duration::from_secs(26)); + if initial_delay { + debug!("scheduling periodic_stun to run in {}s", d.as_secs()); + time::interval_at(time::Instant::now() + d, d) + } else { + debug!( + "scheduling periodic_stun to run immediately and in {}s", + d.as_secs() + ); + time::interval(d) + } +} + +/// Initial connection setup. +fn bind( + addr_v4: Option, + addr_v6: Option, +) -> Result<(UdpConn, Option)> { + let addr_v4 = addr_v4.unwrap_or_else(|| SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)); + let pconn4 = UdpConn::bind(SocketAddr::V4(addr_v4)).context("bind IPv4 failed")?; + + let ip4_port = pconn4.local_addr()?.port(); + let ip6_port = ip4_port.checked_add(1).unwrap_or(ip4_port - 1); + let addr_v6 = + addr_v6.unwrap_or_else(|| SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, ip6_port, 0, 0)); + let pconn6 = match UdpConn::bind(SocketAddr::V6(addr_v6)) { + Ok(conn) => Some(conn), + Err(err) => { + info!("bind ignoring IPv6 bind failure: {:?}", err); + None + } + }; + + Ok((pconn4, pconn6)) +} + +/// The discovered direct addresses of this [`MagicSock`]. +/// +/// These are all the [`DirectAddr`]s that this [`MagicSock`] is aware of for itself. +/// They include all locally bound ones as well as those discovered by other mechanisms like +/// STUN. +#[derive(derive_more::Debug, Default, Clone)] +struct DiscoveredDirectAddrs { + /// The last set of discovered direct addresses. + addrs: Watchable>, + + /// The last time the direct addresses were updated, even if there was no change. + /// + /// This is only ever None at startup. + updated_at: Arc>>, +} + +impl DiscoveredDirectAddrs { + /// Updates the direct addresses, returns `true` if they changed, `false` if not. + fn update(&self, addrs: BTreeSet) -> bool { + *self.updated_at.write().unwrap() = Some(Instant::now()); + let updated = self.addrs.update(addrs).is_ok(); + if updated { + event!( + target: "events.net.direct_addrs", + Level::DEBUG, + addrs = ?self.addrs.get(), + ); + } + updated + } + + fn sockaddrs(&self) -> BTreeSet { + self.addrs.read().iter().map(|da| da.addr).collect() + } + + /// Whether the direct addr information is considered "fresh". + /// + /// If not fresh you should probably update the direct addresses before using this info. + /// + /// Returns `Ok(())` if fresh enough and `Err(elapsed)` if not fresh enough. + /// `elapsed` is the time elapsed since the direct addresses were last updated. + /// + /// If there is no direct address information `Err(Duration::ZERO)` is returned. + fn fresh_enough(&self) -> Result<(), Duration> { + match *self.updated_at.read().expect("poisoned") { + None => Err(Duration::ZERO), + Some(time) => { + let elapsed = time.elapsed(); + if elapsed <= ENDPOINTS_FRESH_ENOUGH_DURATION { + Ok(()) + } else { + Err(elapsed) + } + } + } + } + + fn to_call_me_maybe_message(&self) -> disco::CallMeMaybe { + let my_numbers = self.addrs.read().iter().map(|da| da.addr).collect(); + disco::CallMeMaybe { my_numbers } + } + + fn updates_stream(&self) -> DirectAddrsStream { + DirectAddrsStream { + initial: Some(self.addrs.get()), + inner: self.addrs.watch().into_stream(), + } + } +} + +/// Stream returning local endpoints as they change. +#[derive(Debug)] +pub struct DirectAddrsStream { + initial: Option>, + inner: watchable::WatcherStream>, +} + +impl Stream for DirectAddrsStream { + type Item = BTreeSet; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = &mut *self; + if let Some(addrs) = this.initial.take() { + if !addrs.is_empty() { + return Poll::Ready(Some(addrs)); + } + } + loop { + match Pin::new(&mut this.inner).poll_next(cx) { + Poll::Pending => break Poll::Pending, + Poll::Ready(Some(addrs)) => { + if addrs.is_empty() { + // When we start up we might initially have empty direct addrs as + // the magic socket has not yet figured this out. Later on this set + // should never be empty. However even if it was the magicsock + // would be in a state not very usable so skipping those events is + // probably fine. + // To make sure we install the right waker we loop rather than + // returning Poll::Pending immediately here. + continue; + } else { + break Poll::Ready(Some(addrs)); + } + } + Poll::Ready(None) => break Poll::Ready(None), + } + } + } +} + +/// Split a transmit containing a GSO payload into individual packets. +/// +/// This allocates the data. +/// +/// If the transmit has a segment size it contains multiple GSO packets. It will be split +/// into multiple packets according to that segment size. If it does not have a segment +/// size, the contents will be sent as a single packet. +// TODO: If quinn stayed on bytes this would probably be much cheaper, probably. Need to +// figure out where they allocate the Vec. +fn split_packets(transmit: &quinn_udp::Transmit) -> RelayContents { + let mut res = SmallVec::with_capacity(1); + let contents = transmit.contents; + if let Some(segment_size) = transmit.segment_size { + for chunk in contents.chunks(segment_size) { + res.push(Bytes::from(chunk.to_vec())); + } + } else { + res.push(Bytes::from(contents.to_vec())); + } + res +} + +/// Splits a packet into its component items. +#[derive(Debug)] +struct PacketSplitIter { + bytes: Bytes, +} + +impl PacketSplitIter { + /// Create a new PacketSplitIter from a packet. + /// + /// Returns an error if the packet is too big. + fn new(bytes: Bytes) -> Self { + Self { bytes } + } + + fn fail(&mut self) -> Option> { + self.bytes.clear(); + Some(Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "", + ))) + } +} + +impl Iterator for PacketSplitIter { + type Item = std::io::Result; + + fn next(&mut self) -> Option { + use bytes::Buf; + if self.bytes.has_remaining() { + if self.bytes.remaining() < 2 { + return self.fail(); + } + let len = self.bytes.get_u16_le() as usize; + if self.bytes.remaining() < len { + return self.fail(); + } + let item = self.bytes.split_to(len); + Some(Ok(item)) + } else { + None + } + } +} + +/// The fake address used by the QUIC layer to address a node. +/// +/// You can consider this as nothing more than a lookup key for a node the [`MagicSock`] knows +/// about. +/// +/// [`MagicSock`] can reach a node by several real socket addresses, or maybe even via the relay +/// node. The QUIC layer however needs to address a node by a stable [`SocketAddr`] so +/// that normal socket APIs can function. Thus when a new node is introduced to a [`MagicSock`] +/// it is given a new fake address. This is the type of that address. +/// +/// It is but a newtype. And in our QUIC-facing socket APIs like [`AsyncUdpSocket`] it +/// comes in as the inner [`SocketAddr`], in those interfaces we have to be careful to do +/// the conversion to this type. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub(crate) struct QuicMappedAddr(pub(crate) SocketAddr); + +/// Counter to always generate unique addresses for [`QuicMappedAddr`]. +static ADDR_COUNTER: AtomicU64 = AtomicU64::new(1); + +impl QuicMappedAddr { + /// The Prefix/L of our Unique Local Addresses. + const ADDR_PREFIXL: u8 = 0xfd; + /// The Global ID used in our Unique Local Addresses. + const ADDR_GLOBAL_ID: [u8; 5] = [21, 7, 10, 81, 11]; + /// The Subnet ID used in our Unique Local Addresses. + const ADDR_SUBNET: [u8; 2] = [0; 2]; + + /// Generates a globally unique fake UDP address. + /// + /// This generates and IPv6 Unique Local Address according to RFC 4193. + pub(crate) fn generate() -> Self { + let mut addr = [0u8; 16]; + addr[0] = Self::ADDR_PREFIXL; + addr[1..6].copy_from_slice(&Self::ADDR_GLOBAL_ID); + addr[6..8].copy_from_slice(&Self::ADDR_SUBNET); + + let counter = ADDR_COUNTER.fetch_add(1, Ordering::Relaxed); + addr[8..16].copy_from_slice(&counter.to_be_bytes()); + + Self(SocketAddr::new(IpAddr::V6(Ipv6Addr::from(addr)), 12345)) + } +} + +impl std::fmt::Display for QuicMappedAddr { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "QuicMappedAddr({})", self.0) + } +} +fn disco_message_sent(msg: &disco::Message) { + match msg { + disco::Message::Ping(_) => { + inc!(MagicsockMetrics, sent_disco_ping); + } + disco::Message::Pong(_) => { + inc!(MagicsockMetrics, sent_disco_pong); + } + disco::Message::CallMeMaybe(_) => { + inc!(MagicsockMetrics, sent_disco_call_me_maybe); + } + } +} + +/// A *direct address* on which an iroh-node might be contactable. +/// +/// Direct addresses are UDP socket addresses on which an iroh-net node could potentially be +/// contacted. These can come from various sources depending on the network topology of the +/// iroh-net node, see [`DirectAddrType`] for the several kinds of sources. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DirectAddr { + /// The address. + pub addr: SocketAddr, + /// The origin of this direct address. + pub typ: DirectAddrType, +} + +/// The type of direct address. +/// +/// These are the various sources or origins from which an iroh-net node might have found a +/// possible [`DirectAddr`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum DirectAddrType { + /// Not yet determined.. + Unknown, + /// A locally bound socket address. + Local, + /// Public internet address discovered via STUN. + /// + /// When possible an iroh-net node will perform STUN to discover which is the address + /// from which it sends data on the public internet. This can be different from locally + /// bound addresses when the node is on a local network which performs NAT or similar. + Stun, + /// An address assigned by the router using port mapping. + /// + /// When possible an iroh-net node will request a port mapping from the local router to + /// get a publicly routable direct address. + Portmapped, + /// Hard NAT: STUN'ed IPv4 address + local fixed port. + /// + /// It is possible to configure iroh-net to bound to a specific port and independently + /// configure the router to forward this port to the iroh-net node. This indicates a + /// situation like this, which still uses STUN to discover the public address. + Stun4LocalPort, +} + +impl Display for DirectAddrType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DirectAddrType::Unknown => write!(f, "?"), + DirectAddrType::Local => write!(f, "local"), + DirectAddrType::Stun => write!(f, "stun"), + DirectAddrType::Portmapped => write!(f, "portmap"), + DirectAddrType::Stun4LocalPort => write!(f, "stun4localport"), + } + } +} + +/// Contains information about the host's network state. +#[derive(Debug, Clone, PartialEq)] +struct NetInfo { + /// Says whether the host's NAT mappings vary based on the destination IP. + mapping_varies_by_dest_ip: Option, + + /// If their router does hairpinning. It reports true even if there's no NAT involved. + hair_pinning: Option, + + /// Whether the host has IPv6 internet connectivity. + working_ipv6: Option, + + /// Whether the OS supports IPv6 at all, regardless of whether IPv6 internet connectivity is available. + os_has_ipv6: Option, + + /// Whether the host has UDP internet connectivity. + working_udp: Option, + + /// Whether ICMPv4 works, `None` means not checked. + working_icmp_v4: Option, + + /// Whether ICMPv6 works, `None` means not checked. + working_icmp_v6: Option, + + /// Whether we have an existing portmap open (UPnP, PMP, or PCP). + have_port_map: bool, + + /// Probe indicating the presence of port mapping protocols on the LAN. + portmap_probe: Option, + + /// This node's preferred relay server for incoming traffic. + /// + /// The node might be be temporarily connected to multiple relay servers (to send to + /// other nodes) but this is the relay on which you can always contact this node. Also + /// known as home relay. + preferred_relay: Option, + + /// The fastest recent time to reach various relay STUN servers, in seconds. + /// + /// This should only be updated rarely, or when there's a + /// material change, as any change here also gets uploaded to the control plane. + relay_latency: BTreeMap, +} + +impl NetInfo { + /// Checks if this is probably still the same network as *other*. + /// + /// This tries to compare the network situation, without taking into account things + /// expected to change a little like e.g. latency to the relay server. + fn basically_equal(&self, other: &Self) -> bool { + let eq_icmp_v4 = match (self.working_icmp_v4, other.working_icmp_v4) { + (Some(slf), Some(other)) => slf == other, + _ => true, // ignore for comparison if only one report had this info + }; + let eq_icmp_v6 = match (self.working_icmp_v6, other.working_icmp_v6) { + (Some(slf), Some(other)) => slf == other, + _ => true, // ignore for comparison if only one report had this info + }; + self.mapping_varies_by_dest_ip == other.mapping_varies_by_dest_ip + && self.hair_pinning == other.hair_pinning + && self.working_ipv6 == other.working_ipv6 + && self.os_has_ipv6 == other.os_has_ipv6 + && self.working_udp == other.working_udp + && eq_icmp_v4 + && eq_icmp_v6 + && self.have_port_map == other.have_port_map + && self.portmap_probe == other.portmap_probe + && self.preferred_relay == other.preferred_relay + } +} + +#[cfg(test)] +mod tests { + use anyhow::Context; + use iroh_test::CallOnDrop; + use rand::RngCore; + use tokio_util::task::AbortOnDropHandle; + + use super::*; + use crate::{defaults::staging::EU_RELAY_HOSTNAME, tls, Endpoint, RelayMode}; + + const ALPN: &[u8] = b"n0/test/1"; + + impl MagicSock { + #[track_caller] + pub fn add_test_addr(&self, node_addr: NodeAddr) { + self.add_node_addr( + node_addr, + Source::NamedApp { + name: "test".into(), + }, + ) + .unwrap() + } + } + + /// Magicsock plus wrappers for sending packets + #[derive(Clone)] + struct MagicStack { + secret_key: SecretKey, + endpoint: Endpoint, + } + + impl MagicStack { + async fn new(relay_mode: RelayMode) -> Result { + let secret_key = SecretKey::generate(); + + let mut transport_config = quinn::TransportConfig::default(); + transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); + + let endpoint = Endpoint::builder() + .secret_key(secret_key.clone()) + .transport_config(transport_config) + .relay_mode(relay_mode) + .alpns(vec![ALPN.to_vec()]) + .bind() + .await?; + + Ok(Self { + secret_key, + endpoint, + }) + } + + fn tracked_endpoints(&self) -> Vec { + self.endpoint + .magic_sock() + .list_remote_infos() + .into_iter() + .map(|ep| ep.node_id) + .collect() + } + + fn public(&self) -> PublicKey { + self.secret_key.public() + } + } + + /// Monitors endpoint changes and plumbs things together. + /// + /// This is a way of connecting endpoints without a relay server. Whenever the local + /// endpoints of a magic endpoint change this address is added to the other magic + /// sockets. This function will await until the endpoints are connected the first time + /// before returning. + /// + /// When the returned drop guard is dropped, the tasks doing this updating are stopped. + #[instrument(skip_all)] + async fn mesh_stacks(stacks: Vec) -> Result { + /// Registers endpoint addresses of a node to all other nodes. + fn update_direct_addrs( + stacks: &[MagicStack], + my_idx: usize, + new_addrs: BTreeSet, + ) { + let me = &stacks[my_idx]; + for (i, m) in stacks.iter().enumerate() { + if i == my_idx { + continue; + } + + let addr = NodeAddr { + node_id: me.public(), + info: crate::AddrInfo { + relay_url: None, + direct_addresses: new_addrs.iter().map(|ep| ep.addr).collect(), + }, + }; + m.endpoint.magic_sock().add_test_addr(addr); + } + } + + // For each node, start a task which monitors its local endpoints and registers them + // with the other nodes as local endpoints become known. + let mut tasks = JoinSet::new(); + for (my_idx, m) in stacks.iter().enumerate() { + let m = m.clone(); + let stacks = stacks.clone(); + tasks.spawn(async move { + let me = m.endpoint.node_id().fmt_short(); + let mut stream = m.endpoint.direct_addresses(); + while let Some(new_eps) = stream.next().await { + info!(%me, "conn{} endpoints update: {:?}", my_idx + 1, new_eps); + update_direct_addrs(&stacks, my_idx, new_eps); + } + }); + } + let guard = CallOnDrop::new(move || { + tasks.abort_all(); + }); + + // Wait for all nodes to be registered with each other. + time::timeout(Duration::from_secs(10), async move { + let all_node_ids: Vec<_> = stacks.iter().map(|ms| ms.endpoint.node_id()).collect(); + loop { + let mut ready = Vec::with_capacity(stacks.len()); + for ms in stacks.iter() { + let endpoints = ms.tracked_endpoints(); + let my_node_id = ms.endpoint.node_id(); + let all_nodes_meshed = all_node_ids + .iter() + .filter(|node_id| **node_id != my_node_id) + .all(|node_id| endpoints.contains(node_id)); + ready.push(all_nodes_meshed); + } + if ready.iter().all(|meshed| *meshed) { + break; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + }) + .await + .context("failed to connect nodes")?; + info!("all nodes meshed"); + Ok(guard) + } + + #[instrument(skip_all, fields(me = %ep.endpoint.node_id().fmt_short()))] + async fn echo_receiver(ep: MagicStack) -> Result<()> { + info!("accepting conn"); + let conn = ep.endpoint.accept().await.expect("no conn"); + + info!("connecting"); + let conn = conn.await.context("[receiver] connecting")?; + info!("accepting bi"); + let (mut send_bi, mut recv_bi) = + conn.accept_bi().await.context("[receiver] accepting bi")?; + + info!("reading"); + let val = recv_bi + .read_to_end(usize::MAX) + .await + .context("[receiver] reading to end")?; + + info!("replying"); + for chunk in val.chunks(12) { + send_bi + .write_all(chunk) + .await + .context("[receiver] sending chunk")?; + } + + info!("finishing"); + send_bi.finish().context("[receiver] finishing")?; + send_bi.stopped().await.context("[receiver] stopped")?; + + let stats = conn.stats(); + info!("stats: {:#?}", stats); + // TODO: ensure panics in this function are reported ok + assert!( + stats.path.lost_packets < 10, + "[receiver] should not loose many packets", + ); + + info!("close"); + conn.close(0u32.into(), b"done"); + info!("wait idle"); + ep.endpoint.endpoint().wait_idle().await; + + Ok(()) + } + + #[instrument(skip_all, fields(me = %ep.endpoint.node_id().fmt_short()))] + async fn echo_sender(ep: MagicStack, dest_id: PublicKey, msg: &[u8]) -> Result<()> { + info!("connecting to {}", dest_id.fmt_short()); + let dest = NodeAddr::new(dest_id); + let conn = ep + .endpoint + .connect(dest, ALPN) + .await + .context("[sender] connect")?; + + info!("opening bi"); + let (mut send_bi, mut recv_bi) = conn.open_bi().await.context("[sender] open bi")?; + + info!("writing message"); + send_bi.write_all(msg).await.context("[sender] write all")?; + + info!("finishing"); + send_bi.finish().context("[sender] finish")?; + send_bi.stopped().await.context("[sender] stopped")?; + + info!("reading_to_end"); + let val = recv_bi.read_to_end(usize::MAX).await.context("[sender]")?; + assert_eq!( + val, + msg, + "[sender] expected {}, got {}", + hex::encode(msg), + hex::encode(&val) + ); + + let stats = conn.stats(); + info!("stats: {:#?}", stats); + assert!( + stats.path.lost_packets < 10, + "[sender] should not loose many packets", + ); + + info!("close"); + conn.close(0u32.into(), b"done"); + info!("wait idle"); + ep.endpoint.endpoint().wait_idle().await; + Ok(()) + } + + /// Runs a roundtrip between the [`echo_sender`] and [`echo_receiver`]. + async fn run_roundtrip(sender: MagicStack, receiver: MagicStack, payload: &[u8]) { + let send_node_id = sender.endpoint.node_id(); + let recv_node_id = receiver.endpoint.node_id(); + info!("\nroundtrip: {send_node_id:#} -> {recv_node_id:#}"); + + let receiver_task = tokio::spawn(echo_receiver(receiver)); + let sender_res = echo_sender(sender, recv_node_id, payload).await; + let sender_is_err = match sender_res { + Ok(()) => false, + Err(err) => { + eprintln!("[sender] Error:\n{err:#?}"); + true + } + }; + let receiver_is_err = match receiver_task.await { + Ok(Ok(())) => false, + Ok(Err(err)) => { + eprintln!("[receiver] Error:\n{err:#?}"); + true + } + Err(joinerr) => { + if joinerr.is_panic() { + std::panic::resume_unwind(joinerr.into_panic()); + } else { + eprintln!("[receiver] Error:\n{joinerr:#?}"); + } + true + } + }; + if sender_is_err || receiver_is_err { + panic!("Sender or receiver errored"); + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_two_devices_roundtrip_quinn_magic() -> Result<()> { + iroh_test::logging::setup_multithreaded(); + + let m1 = MagicStack::new(RelayMode::Disabled).await?; + let m2 = MagicStack::new(RelayMode::Disabled).await?; + + let _guard = mesh_stacks(vec![m1.clone(), m2.clone()]).await?; + + for i in 0..5 { + info!("\n-- round {i}"); + run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await; + run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await; + + info!("\n-- larger data"); + let mut data = vec![0u8; 10 * 1024]; + rand::thread_rng().fill_bytes(&mut data); + run_roundtrip(m1.clone(), m2.clone(), &data).await; + run_roundtrip(m2.clone(), m1.clone(), &data).await; + } + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_two_devices_roundtrip_network_change() -> Result<()> { + time::timeout( + Duration::from_secs(90), + test_two_devices_roundtrip_network_change_impl(), + ) + .await? + } + + /// Same structure as `test_two_devices_roundtrip_quinn_magic`, but interrupts regularly + /// with (simulated) network changes. + async fn test_two_devices_roundtrip_network_change_impl() -> Result<()> { + iroh_test::logging::setup_multithreaded(); + + let m1 = MagicStack::new(RelayMode::Disabled).await?; + let m2 = MagicStack::new(RelayMode::Disabled).await?; + + let _guard = mesh_stacks(vec![m1.clone(), m2.clone()]).await?; + + let offset = || { + let delay = rand::thread_rng().gen_range(10..=500); + Duration::from_millis(delay) + }; + let rounds = 5; + + // Regular network changes to m1 only. + let m1_network_change_guard = { + let m1 = m1.clone(); + let task = tokio::spawn(async move { + loop { + println!("[m1] network change"); + m1.endpoint.magic_sock().force_network_change(true).await; + time::sleep(offset()).await; + } + }); + CallOnDrop::new(move || { + task.abort(); + }) + }; + + for i in 0..rounds { + println!("-- [m1 changes] round {}", i + 1); + run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await; + run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await; + + println!("-- [m1 changes] larger data"); + let mut data = vec![0u8; 10 * 1024]; + rand::thread_rng().fill_bytes(&mut data); + run_roundtrip(m1.clone(), m2.clone(), &data).await; + run_roundtrip(m2.clone(), m1.clone(), &data).await; + } + + std::mem::drop(m1_network_change_guard); + + // Regular network changes to m2 only. + let m2_network_change_guard = { + let m2 = m2.clone(); + let task = tokio::spawn(async move { + loop { + println!("[m2] network change"); + m2.endpoint.magic_sock().force_network_change(true).await; + time::sleep(offset()).await; + } + }); + CallOnDrop::new(move || { + task.abort(); + }) + }; + + for i in 0..rounds { + println!("-- [m2 changes] round {}", i + 1); + run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await; + run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await; + + println!("-- [m2 changes] larger data"); + let mut data = vec![0u8; 10 * 1024]; + rand::thread_rng().fill_bytes(&mut data); + run_roundtrip(m1.clone(), m2.clone(), &data).await; + run_roundtrip(m2.clone(), m1.clone(), &data).await; + } + + std::mem::drop(m2_network_change_guard); + + // Regular network changes to both m1 and m2 only. + let m1_m2_network_change_guard = { + let m1 = m1.clone(); + let m2 = m2.clone(); + let task = tokio::spawn(async move { + println!("-- [m1] network change"); + m1.endpoint.magic_sock().force_network_change(true).await; + println!("-- [m2] network change"); + m2.endpoint.magic_sock().force_network_change(true).await; + time::sleep(offset()).await; + }); + CallOnDrop::new(move || { + task.abort(); + }) + }; + + for i in 0..rounds { + println!("-- [m1 & m2 changes] round {}", i + 1); + run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await; + run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await; + + println!("-- [m1 & m2 changes] larger data"); + let mut data = vec![0u8; 10 * 1024]; + rand::thread_rng().fill_bytes(&mut data); + run_roundtrip(m1.clone(), m2.clone(), &data).await; + run_roundtrip(m2.clone(), m1.clone(), &data).await; + } + + std::mem::drop(m1_m2_network_change_guard); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_two_devices_setup_teardown() -> Result<()> { + iroh_test::logging::setup_multithreaded(); + for i in 0..10 { + println!("-- round {i}"); + println!("setting up magic stack"); + let m1 = MagicStack::new(RelayMode::Disabled).await?; + let m2 = MagicStack::new(RelayMode::Disabled).await?; + + let _guard = mesh_stacks(vec![m1.clone(), m2.clone()]).await?; + + println!("closing endpoints"); + let msock1 = m1.endpoint.magic_sock(); + let msock2 = m2.endpoint.magic_sock(); + m1.endpoint.close(0u32.into(), b"done").await?; + m2.endpoint.close(0u32.into(), b"done").await?; + + assert!(msock1.msock.is_closed()); + assert!(msock2.msock.is_closed()); + } + Ok(()) + } + + #[tokio::test] + async fn test_two_devices_roundtrip_quinn_raw() -> Result<()> { + let _guard = iroh_test::logging::setup(); + + let make_conn = |addr: SocketAddr| -> anyhow::Result { + let key = SecretKey::generate(); + let conn = std::net::UdpSocket::bind(addr)?; + + let quic_server_config = tls::make_server_config(&key, vec![ALPN.to_vec()], false)?; + let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_server_config)); + let mut transport_config = quinn::TransportConfig::default(); + transport_config.keep_alive_interval(Some(Duration::from_secs(5))); + transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); + server_config.transport_config(Arc::new(transport_config)); + let mut quic_ep = quinn::Endpoint::new( + quinn::EndpointConfig::default(), + Some(server_config), + conn, + Arc::new(quinn::TokioRuntime), + )?; + + let quic_client_config = + tls::make_client_config(&key, None, vec![ALPN.to_vec()], false)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(quic_client_config)); + let mut transport_config = quinn::TransportConfig::default(); + transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); + client_config.transport_config(Arc::new(transport_config)); + quic_ep.set_default_client_config(client_config); + + Ok(quic_ep) + }; + + let m1 = make_conn("127.0.0.1:0".parse().unwrap())?; + let m2 = make_conn("127.0.0.1:0".parse().unwrap())?; + + // msg from a -> b + macro_rules! roundtrip { + ($a:expr, $b:expr, $msg:expr) => { + let a = $a.clone(); + let b = $b.clone(); + let a_name = stringify!($a); + let b_name = stringify!($b); + println!("{} -> {} ({} bytes)", a_name, b_name, $msg.len()); + + let a_addr = a.local_addr()?; + let b_addr = b.local_addr()?; + + println!("{}: {}, {}: {}", a_name, a_addr, b_name, b_addr); + + let b_task = tokio::task::spawn(async move { + println!("[{b_name}] accepting conn"); + let conn = b.accept().await.expect("no conn"); + println!("[{}] connecting", b_name); + let conn = conn + .await + .with_context(|| format!("[{b_name}] connecting"))?; + println!("[{}] accepting bi", b_name); + let (mut send_bi, mut recv_bi) = conn + .accept_bi() + .await + .with_context(|| format!("[{b_name}] accepting bi"))?; + + println!("[{b_name}] reading"); + let val = recv_bi + .read_to_end(usize::MAX) + .await + .with_context(|| format!("[{b_name}] reading to end"))?; + println!("[{b_name}] finishing"); + send_bi + .finish() + .with_context(|| format!("[{b_name}] finishing"))?; + send_bi + .stopped() + .await + .with_context(|| format!("[b_name] stopped"))?; + + println!("[{b_name}] close"); + conn.close(0u32.into(), b"done"); + println!("[{b_name}] closed"); + + Ok::<_, anyhow::Error>(val) + }); + + println!("[{a_name}] connecting to {b_addr}"); + let conn = a + .connect(b_addr, "localhost")? + .await + .with_context(|| format!("[{a_name}] connect"))?; + + println!("[{a_name}] opening bi"); + let (mut send_bi, mut recv_bi) = conn + .open_bi() + .await + .with_context(|| format!("[{a_name}] open bi"))?; + println!("[{a_name}] writing message"); + send_bi + .write_all(&$msg[..]) + .await + .with_context(|| format!("[{a_name}] write all"))?; + + println!("[{a_name}] finishing"); + send_bi + .finish() + .with_context(|| format!("[{a_name}] finish"))?; + send_bi + .stopped() + .await + .with_context(|| format!("[{a_name}] stopped"))?; + + println!("[{a_name}] reading_to_end"); + let _ = recv_bi + .read_to_end(usize::MAX) + .await + .with_context(|| format!("[{a_name}] reading_to_end"))?; + println!("[{a_name}] close"); + conn.close(0u32.into(), b"done"); + println!("[{a_name}] wait idle"); + a.wait_idle().await; + + drop(send_bi); + + // make sure the right values arrived + println!("[{a_name}] waiting for channel"); + let val = b_task.await??; + anyhow::ensure!( + val == $msg, + "expected {}, got {}", + hex::encode($msg), + hex::encode(val) + ); + }; + } + + for i in 0..10 { + println!("-- round {}", i + 1); + roundtrip!(m1, m2, b"hello m1"); + roundtrip!(m2, m1, b"hello m2"); + + println!("-- larger data"); + + let mut data = vec![0u8; 10 * 1024]; + rand::thread_rng().fill_bytes(&mut data); + roundtrip!(m1, m2, data); + roundtrip!(m2, m1, data); + } + + Ok(()) + } + + #[tokio::test] + async fn test_two_devices_roundtrip_quinn_rebinding_conn() -> Result<()> { + let _guard = iroh_test::logging::setup(); + + fn make_conn(addr: SocketAddr) -> anyhow::Result { + let key = SecretKey::generate(); + let conn = UdpConn::bind(addr)?; + + let quic_server_config = tls::make_server_config(&key, vec![ALPN.to_vec()], false)?; + let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_server_config)); + let mut transport_config = quinn::TransportConfig::default(); + transport_config.keep_alive_interval(Some(Duration::from_secs(5))); + transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); + server_config.transport_config(Arc::new(transport_config)); + let mut quic_ep = quinn::Endpoint::new_with_abstract_socket( + quinn::EndpointConfig::default(), + Some(server_config), + Arc::new(conn), + Arc::new(quinn::TokioRuntime), + )?; + + let quic_client_config = + tls::make_client_config(&key, None, vec![ALPN.to_vec()], false)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(quic_client_config)); + let mut transport_config = quinn::TransportConfig::default(); + transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); + client_config.transport_config(Arc::new(transport_config)); + quic_ep.set_default_client_config(client_config); + + Ok(quic_ep) + } + + let m1 = make_conn("127.0.0.1:7770".parse().unwrap())?; + let m2 = make_conn("127.0.0.1:7771".parse().unwrap())?; + + // msg from a -> b + macro_rules! roundtrip { + ($a:expr, $b:expr, $msg:expr) => { + let a = $a.clone(); + let b = $b.clone(); + let a_name = stringify!($a); + let b_name = stringify!($b); + println!("{} -> {} ({} bytes)", a_name, b_name, $msg.len()); + + let a_addr: SocketAddr = format!("127.0.0.1:{}", a.local_addr()?.port()) + .parse() + .unwrap(); + let b_addr: SocketAddr = format!("127.0.0.1:{}", b.local_addr()?.port()) + .parse() + .unwrap(); + + println!("{}: {}, {}: {}", a_name, a_addr, b_name, b_addr); + + let b_task = tokio::task::spawn(async move { + println!("[{}] accepting conn", b_name); + let conn = b.accept().await.expect("no conn"); + println!("[{}] connecting", b_name); + let conn = conn + .await + .with_context(|| format!("[{}] connecting", b_name))?; + println!("[{}] accepting bi", b_name); + let (mut send_bi, mut recv_bi) = conn + .accept_bi() + .await + .with_context(|| format!("[{}] accepting bi", b_name))?; + + println!("[{}] reading", b_name); + let val = recv_bi + .read_to_end(usize::MAX) + .await + .with_context(|| format!("[{}] reading to end", b_name))?; + println!("[{}] finishing", b_name); + send_bi + .finish() + .with_context(|| format!("[{}] finishing", b_name))?; + send_bi + .stopped() + .await + .with_context(|| format!("[{b_name}] stopped"))?; + + println!("[{}] close", b_name); + conn.close(0u32.into(), b"done"); + println!("[{}] closed", b_name); + + Ok::<_, anyhow::Error>(val) + }); + + println!("[{}] connecting to {}", a_name, b_addr); + let conn = a + .connect(b_addr, "localhost")? + .await + .with_context(|| format!("[{}] connect", a_name))?; + + println!("[{}] opening bi", a_name); + let (mut send_bi, mut recv_bi) = conn + .open_bi() + .await + .with_context(|| format!("[{}] open bi", a_name))?; + println!("[{}] writing message", a_name); + send_bi + .write_all(&$msg[..]) + .await + .with_context(|| format!("[{}] write all", a_name))?; + + println!("[{}] finishing", a_name); + send_bi + .finish() + .with_context(|| format!("[{}] finish", a_name))?; + send_bi + .stopped() + .await + .with_context(|| format!("[{a_name}] stopped"))?; + + println!("[{}] reading_to_end", a_name); + let _ = recv_bi + .read_to_end(usize::MAX) + .await + .with_context(|| format!("[{}]", a_name))?; + println!("[{}] close", a_name); + conn.close(0u32.into(), b"done"); + println!("[{}] wait idle", a_name); + a.wait_idle().await; + + drop(send_bi); + + // make sure the right values arrived + println!("[{}] waiting for channel", a_name); + let val = b_task.await??; + anyhow::ensure!( + val == $msg, + "expected {}, got {}", + hex::encode($msg), + hex::encode(val) + ); + }; + } + + for i in 0..10 { + println!("-- round {}", i + 1); + roundtrip!(m1, m2, b"hello m1"); + roundtrip!(m2, m1, b"hello m2"); + + println!("-- larger data"); + + let mut data = vec![0u8; 10 * 1024]; + rand::thread_rng().fill_bytes(&mut data); + roundtrip!(m1, m2, data); + roundtrip!(m2, m1, data); + } + + Ok(()) + } + + #[test] + fn test_split_packets() { + fn mk_transmit(contents: &[u8], segment_size: Option) -> quinn_udp::Transmit<'_> { + let destination = "127.0.0.1:0".parse().unwrap(); + quinn_udp::Transmit { + destination, + ecn: None, + contents, + segment_size, + src_ip: None, + } + } + fn mk_expected(parts: impl IntoIterator) -> RelayContents { + parts + .into_iter() + .map(|p| p.as_bytes().to_vec().into()) + .collect() + } + // no split + assert_eq!( + split_packets(&mk_transmit(b"hello", None)), + mk_expected(["hello"]) + ); + // split without rest + assert_eq!( + split_packets(&mk_transmit(b"helloworld", Some(5))), + mk_expected(["hello", "world"]) + ); + // split with rest and second transmit + assert_eq!( + split_packets(&mk_transmit(b"hello world", Some(5))), + mk_expected(["hello", " worl", "d"]) // spellchecker:disable-line + ); + // split that results in 1 packet + assert_eq!( + split_packets(&mk_transmit(b"hello world", Some(1000))), + mk_expected(["hello world"]) + ); + } + + #[tokio::test] + async fn test_local_endpoints() { + let _guard = iroh_test::logging::setup(); + let ms = Handle::new(Default::default()).await.unwrap(); + + // See if we can get endpoints. + let eps0 = ms.direct_addresses().next().await.unwrap(); + println!("{eps0:?}"); + assert!(!eps0.is_empty()); + + // Getting the endpoints again immediately should give the same results. + let eps1 = ms.direct_addresses().next().await.unwrap(); + println!("{eps1:?}"); + assert_eq!(eps0, eps1); + } + + #[tokio::test] + async fn test_watch_home_relay() { + // use an empty relay map to get full control of the changes during the test + let ops = Options { + relay_map: RelayMap::empty(), + ..Default::default() + }; + let msock = MagicSock::spawn(ops).await.unwrap(); + let mut relay_stream = msock.watch_home_relay(); + + // no relay, nothing to report + assert_eq!( + futures_lite::future::poll_once(relay_stream.next()).await, + None + ); + + let url: RelayUrl = format!("https://{}", EU_RELAY_HOSTNAME).parse().unwrap(); + msock.set_my_relay(Some(url.clone())); + + assert_eq!(relay_stream.next().await, Some(url.clone())); + + // drop the stream and query it again, the result should be immediately available + + let mut relay_stream = msock.watch_home_relay(); + assert_eq!( + futures_lite::future::poll_once(relay_stream.next()).await, + Some(Some(url)) + ); + } + + /// Creates a new [`quinn::Endpoint`] hooked up to a [`MagicSock`]. + /// + /// This is without involving [`crate::endpoint::Endpoint`]. The socket will accept + /// connections using [`ALPN`]. + /// + /// Use [`magicsock_connect`] to establish connections. + #[instrument(name = "ep", skip_all, fields(me = secret_key.public().fmt_short()))] + async fn magicsock_ep(secret_key: SecretKey) -> anyhow::Result<(quinn::Endpoint, Handle)> { + let opts = Options { + addr_v4: None, + addr_v6: None, + secret_key: secret_key.clone(), + relay_map: RelayMap::empty(), + node_map: None, + discovery: None, + dns_resolver: crate::dns::default_resolver().clone(), + proxy_url: None, + insecure_skip_relay_cert_verify: true, + }; + let msock = MagicSock::spawn(opts).await?; + let server_config = crate::endpoint::make_server_config( + &secret_key, + vec![ALPN.to_vec()], + Arc::new(quinn::TransportConfig::default()), + true, + )?; + let mut endpoint_config = quinn::EndpointConfig::default(); + endpoint_config.grease_quic_bit(false); + let endpoint = quinn::Endpoint::new_with_abstract_socket( + endpoint_config, + Some(server_config), + Arc::new(msock.clone()), + Arc::new(quinn::TokioRuntime), + )?; + Ok((endpoint, msock)) + } + + /// Connects from `ep` returned by [`magicsock_ep`] to the `node_id`. + /// + /// Uses [`ALPN`], `node_id`, must match `addr`. + #[instrument(name = "connect", skip_all, fields(me = ep_secret_key.public().fmt_short()))] + async fn magicsock_connect( + ep: &quinn::Endpoint, + ep_secret_key: SecretKey, + addr: QuicMappedAddr, + node_id: NodeId, + ) -> Result { + // Endpoint::connect sets this, do the same to have similar behaviour. + let mut transport_config = quinn::TransportConfig::default(); + transport_config.keep_alive_interval(Some(Duration::from_secs(1))); + + magicsock_connet_with_transport_config( + ep, + ep_secret_key, + addr, + node_id, + Arc::new(transport_config), + ) + .await + } + + /// Connects from `ep` returned by [`magicsock_ep`] to the `node_id`. + /// + /// This version allows customising the transport config. + /// + /// Uses [`ALPN`], `node_id`, must match `addr`. + #[instrument(name = "connect", skip_all, fields(me = ep_secret_key.public().fmt_short()))] + async fn magicsock_connet_with_transport_config( + ep: &quinn::Endpoint, + ep_secret_key: SecretKey, + addr: QuicMappedAddr, + node_id: NodeId, + transport_config: Arc, + ) -> Result { + let alpns = vec![ALPN.to_vec()]; + let quic_client_config = + tls::make_client_config(&ep_secret_key, Some(node_id), alpns, true)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(quic_client_config)); + client_config.transport_config(transport_config); + let connect = ep.connect_with(client_config, addr.0, "localhost")?; + let connection = connect.await?; + Ok(connection) + } + + #[tokio::test] + async fn test_try_send_no_send_addr() { + // Regression test: if there is no send_addr we should keep being able to use the + // Endpoint. + let _guard = iroh_test::logging::setup(); + + let secret_key_1 = SecretKey::from_bytes(&[1u8; 32]); + let secret_key_2 = SecretKey::from_bytes(&[2u8; 32]); + let node_id_2 = secret_key_2.public(); + let secret_key_missing_node = SecretKey::from_bytes(&[255u8; 32]); + let node_id_missing_node = secret_key_missing_node.public(); + + let (ep_1, msock_1) = magicsock_ep(secret_key_1.clone()).await.unwrap(); + + // Generate an address not present in the NodeMap. + let bad_addr = QuicMappedAddr::generate(); + + // 500ms is rather fast here. Running this locally it should always be the correct + // timeout. If this is too slow however the test will not become flaky as we are + // expecting the timeout, we might just get the timeout for the wrong reason. But + // this speeds up the test. + let res = tokio::time::timeout( + Duration::from_millis(500), + magicsock_connect(&ep_1, secret_key_1.clone(), bad_addr, node_id_missing_node), + ) + .await; + assert!(res.is_err(), "expecting timeout"); + + // Now check we can still create another connection with this endpoint. + let (ep_2, msock_2) = magicsock_ep(secret_key_2.clone()).await.unwrap(); + + // This needs an accept task + let accept_task = tokio::spawn({ + async fn accept(ep: quinn::Endpoint) -> Result<()> { + let incoming = ep.accept().await.ok_or(anyhow!("no incoming"))?; + let _conn = incoming.accept()?.await?; + + // Keep this connection alive for a while + tokio::time::sleep(Duration::from_secs(10)).await; + info!("accept finished"); + Ok(()) + } + let ep_2 = ep_2.clone(); + async move { + if let Err(err) = accept(ep_2).await { + error!("{err:#}"); + } + } + .instrument(info_span!("ep2.accept, me = node_id_2.fmt_short()")) + }); + let _accept_task = AbortOnDropHandle::new(accept_task); + + let node_addr_2 = NodeAddr { + node_id: node_id_2, + info: AddrInfo { + relay_url: None, + direct_addresses: msock_2 + .direct_addresses() + .next() + .await + .expect("no direct addrs") + .into_iter() + .map(|x| x.addr) + .collect(), + }, + }; + msock_1 + .add_node_addr( + node_addr_2, + Source::NamedApp { + name: "test".into(), + }, + ) + .unwrap(); + let addr = msock_1.get_mapping_addr(node_id_2).unwrap(); + let res = tokio::time::timeout( + Duration::from_secs(10), + magicsock_connect(&ep_1, secret_key_1.clone(), addr, node_id_2), + ) + .await + .expect("timeout while connecting"); + + // aka assert!(res.is_ok()) but with nicer error reporting. + res.unwrap(); + + // TODO: Now check if we can connect to a repaired ep_3, but we can't modify that + // much internal state for now. + } + + #[tokio::test] + async fn test_try_send_no_udp_addr_or_relay_url() { + // This specifically tests the `if udp_addr.is_none() && relay_url.is_none()` + // behaviour of MagicSock::try_send. + let _logging_guard = iroh_test::logging::setup(); + + let secret_key_1 = SecretKey::from_bytes(&[1u8; 32]); + let secret_key_2 = SecretKey::from_bytes(&[2u8; 32]); + let node_id_2 = secret_key_2.public(); + + let (ep_1, msock_1) = magicsock_ep(secret_key_1.clone()).await.unwrap(); + let (ep_2, msock_2) = magicsock_ep(secret_key_2.clone()).await.unwrap(); + + // We need a task to accept the connection. + let accept_task = tokio::spawn({ + async fn accept(ep: quinn::Endpoint) -> Result<()> { + let incoming = ep.accept().await.ok_or(anyhow!("no incoming"))?; + let conn = incoming.accept()?.await?; + let mut stream = conn.accept_uni().await?; + stream.read_to_end(1 << 16).await?; + info!("accept finished"); + Ok(()) + } + let ep_2 = ep_2.clone(); + async move { + if let Err(err) = accept(ep_2).await { + error!("{err:#}"); + } + } + .instrument(info_span!("ep2.accept", me = node_id_2.fmt_short())) + }); + let _accept_task = AbortOnDropHandle::new(accept_task); + + // Add an empty entry in the NodeMap of ep_1 + msock_1.node_map.add_node_addr( + NodeAddr { + node_id: node_id_2, + info: AddrInfo::default(), + }, + Source::NamedApp { + name: "test".into(), + }, + ); + let addr_2 = msock_1.get_mapping_addr(node_id_2).unwrap(); + + // Set a low max_idle_timeout so quinn gives up on this quickly and our test does + // not take forever. You need to check the log output to verify this is really + // triggering the correct error. + // In test_try_send_no_send_addr() above you may have noticed we used + // tokio::time::timeout() on the connection attempt instead. Here however we want + // Quinn itself to have fully given up on the connection attempt because we will + // later connect to **the same** node. If Quinn did not give up on the connection + // we'd close it on drop, and the retransmits of the close packets would interfere + // with the next handshake, closing it during the handshake. This makes the test a + // little slower though. + let mut transport_config = quinn::TransportConfig::default(); + transport_config.max_idle_timeout(Some(Duration::from_millis(200).try_into().unwrap())); + let res = magicsock_connet_with_transport_config( + &ep_1, + secret_key_1.clone(), + addr_2, + node_id_2, + Arc::new(transport_config), + ) + .await; + assert!(res.is_err(), "expected timeout"); + info!("first connect timed out as expected"); + + // Provide correct addressing information + msock_1.node_map.add_node_addr( + NodeAddr { + node_id: node_id_2, + info: AddrInfo { + relay_url: None, + direct_addresses: msock_2 + .direct_addresses() + .next() + .await + .expect("no direct addrs") + .into_iter() + .map(|x| x.addr) + .collect(), + }, + }, + Source::NamedApp { + name: "test".into(), + }, + ); + + // We can now connect + tokio::time::timeout(Duration::from_secs(10), async move { + info!("establishing new connection"); + let conn = magicsock_connect(&ep_1, secret_key_1.clone(), addr_2, node_id_2) + .await + .unwrap(); + info!("have connection"); + let mut stream = conn.open_uni().await.unwrap(); + stream.write_all(b"hello").await.unwrap(); + stream.finish().unwrap(); + stream.stopped().await.unwrap(); + info!("finished stream"); + }) + .await + .expect("connection timed out"); + + // TODO: could remove the addresses again, send, add it back and see it recover. + // But we don't have that much private access to the NodeMap. This will do for now. + } +} From deadb1b84df1d89eaae551f1ae4e0584fbd76f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cramfox=E2=80=9D?= <“kasey@n0.computer”> Date: Wed, 11 Dec 2024 00:13:45 -0500 Subject: [PATCH 3/5] clean up dumb files that shouldnt be included --- magicsock.rs | 3939 ---------------------------------------- net-report-defaults.rs | 49 - net-report-lib.rs | 1383 -------------- probes.rs | 877 --------- reportgen.rs | 1597 ---------------- 5 files changed, 7845 deletions(-) delete mode 100644 magicsock.rs delete mode 100644 net-report-defaults.rs delete mode 100644 net-report-lib.rs delete mode 100644 probes.rs delete mode 100644 reportgen.rs diff --git a/magicsock.rs b/magicsock.rs deleted file mode 100644 index b025cfbaf58..00000000000 --- a/magicsock.rs +++ /dev/null @@ -1,3939 +0,0 @@ - -//! Implements a socket that can change its communication path while in use, actively searching for the best way to communicate. -//! -//! Based on tailscale/wgengine/magicsock -//! -//! ### `DEV_RELAY_ONLY` env var: -//! When present at *compile time*, this env var will force all packets -//! to be sent over the relay connection, regardless of whether or -//! not we have a direct UDP address for the given node. -//! -//! The intended use is for testing the relay protocol inside the MagicSock -//! to ensure that we can rely on the relay to send packets when two nodes -//! are unable to find direct UDP connections to each other. -//! -//! This also prevent this node from attempting to hole punch and prevents it -//! from responding to any hole punching attempts. This node will still, -//! however, read any packets that come off the UDP sockets. - -use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - fmt::Display, - io, - net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, - pin::Pin, - sync::{ - atomic::{AtomicBool, AtomicU16, AtomicU64, Ordering}, - Arc, RwLock, - }, - task::{Context, Poll, Waker}, - time::{Duration, Instant}, -}; - -use anyhow::{anyhow, Context as _, Result}; -use bytes::Bytes; -use futures_lite::{FutureExt, Stream, StreamExt}; -use futures_util::stream::BoxStream; -use iroh_base::key::NodeId; -use iroh_metrics::{inc, inc_by}; -use iroh_relay::protos::stun; -use netwatch::{interfaces, ip::LocalAddresses, netmon}; -use quinn::AsyncUdpSocket; -use rand::{seq::SliceRandom, Rng, SeedableRng}; -use smallvec::{smallvec, SmallVec}; -use tokio::{ - sync::{self, mpsc, Mutex}, - task::JoinSet, - time, -}; -use tokio_util::sync::CancellationToken; -use tracing::{ - debug, error, error_span, event, info, info_span, instrument, trace, trace_span, warn, - Instrument, Level, Span, -}; -use url::Url; -use watchable::Watchable; - -use self::{ - metrics::Metrics as MagicsockMetrics, - node_map::{NodeMap, PingAction, PingRole, SendPing}, - relay_actor::{RelayActor, RelayActorMessage, RelayReadResult}, - udp_conn::UdpConn, -}; -use crate::{ - defaults::timeouts::NET_REPORT_TIMEOUT, - disco::{self, CallMeMaybe, SendAddr}, - discovery::{Discovery, DiscoveryItem}, - dns::DnsResolver, - endpoint::NodeAddr, - key::{PublicKey, SecretKey, SharedSecret}, - AddrInfo, RelayMap, RelayUrl, -}; - -mod metrics; -mod node_map; -mod relay_actor; -mod timer; -mod udp_conn; - -pub use node_map::Source; - -pub(super) use self::timer::Timer; -pub use self::{ - metrics::Metrics, - node_map::{ConnectionType, ConnectionTypeStream, ControlMsg, DirectAddrInfo, RemoteInfo}, -}; - -/// How long we consider a STUN-derived endpoint valid for. UDP NAT mappings typically -/// expire at 30 seconds, so this is a few seconds shy of that. -const ENDPOINTS_FRESH_ENOUGH_DURATION: Duration = Duration::from_secs(27); - -const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); - -/// Contains options for `MagicSock::listen`. -#[derive(derive_more::Debug)] -pub(crate) struct Options { - /// The IPv4 address to listen on. - /// - /// If set to `None` it will choose a random port and listen on `0.0.0.0:0`. - pub(crate) addr_v4: Option, - /// The IPv6 address to listen on. - /// - /// If set to `None` it will choose a random port and listen on `[::]:0`. - pub(crate) addr_v6: Option, - - /// Secret key for this node. - pub(crate) secret_key: SecretKey, - - /// The [`RelayMap`] to use, leave empty to not use a relay server. - pub(crate) relay_map: RelayMap, - - /// An optional [`NodeMap`], to restore information about nodes. - pub(crate) node_map: Option>, - - /// Optional node discovery mechanism. - pub(crate) discovery: Option>, - - /// A DNS resolver to use for resolving relay URLs. - /// - /// You can use [`crate::dns::default_resolver`] for a resolver that uses the system's DNS - /// configuration. - pub(crate) dns_resolver: DnsResolver, - - /// Proxy configuration. - pub(crate) proxy_url: Option, - - /// Skip verification of SSL certificates from relay servers - /// - /// May only be used in tests. - #[cfg(any(test, feature = "test-utils"))] - #[cfg_attr(iroh_docsrs, doc(cfg(any(test, feature = "test-utils"))))] - pub(crate) insecure_skip_relay_cert_verify: bool, -} - -impl Default for Options { - fn default() -> Self { - Options { - addr_v4: None, - addr_v6: None, - secret_key: SecretKey::generate(), - relay_map: RelayMap::empty(), - node_map: None, - discovery: None, - proxy_url: None, - dns_resolver: crate::dns::default_resolver().clone(), - #[cfg(any(test, feature = "test-utils"))] - insecure_skip_relay_cert_verify: false, - } - } -} - -/// Contents of a relay message. Use a SmallVec to avoid allocations for the very -/// common case of a single packet. -type RelayContents = SmallVec<[Bytes; 1]>; - -/// Handle for [`MagicSock`]. -/// -/// Dereferences to [`MagicSock`], and handles closing. -#[derive(Clone, Debug, derive_more::Deref)] -pub(crate) struct Handle { - #[deref(forward)] - msock: Arc, - // Empty when closed - actor_tasks: Arc>>, -} - -/// Iroh connectivity layer. -/// -/// This is responsible for routing packets to nodes based on node IDs, it will initially -/// route packets via a relay and transparently try and establish a node-to-node -/// connection and upgrade to it. It will also keep looking for better connections as the -/// network details of both nodes change. -/// -/// It is usually only necessary to use a single [`MagicSock`] instance in an application, it -/// means any QUIC endpoints on top will be sharing as much information about nodes as -/// possible. -#[derive(derive_more::Debug)] -pub(crate) struct MagicSock { - actor_sender: mpsc::Sender, - relay_actor_sender: mpsc::Sender, - /// String representation of the node_id of this node. - me: String, - /// Proxy - proxy_url: Option, - - /// Used for receiving relay messages. - relay_recv_receiver: parking_lot::Mutex>, - /// Stores wakers, to be called when relay_recv_ch receives new data. - network_recv_wakers: parking_lot::Mutex>, - network_send_wakers: Arc>>, - - /// The DNS resolver to be used in this magicsock. - dns_resolver: DnsResolver, - - /// Key for this node. - secret_key: SecretKey, - - /// Cached version of the Ipv4 and Ipv6 addrs of the current connection. - local_addrs: std::sync::RwLock<(SocketAddr, Option)>, - - /// Preferred port from `Options::port`; 0 means auto. - port: AtomicU16, - - /// Close is in progress (or done) - closing: AtomicBool, - /// Close was called. - closed: AtomicBool, - /// If the last net_report report, reports IPv6 to be available. - ipv6_reported: Arc, - - /// None (or zero nodes) means relay is disabled. - relay_map: RelayMap, - /// Nearest relay node ID; 0 means none/unknown. - my_relay: Watchable>, - /// Tracks the networkmap node entity for each node discovery key. - node_map: NodeMap, - /// UDP IPv4 socket - pconn4: UdpConn, - /// UDP IPv6 socket - pconn6: Option, - /// NetReport client - net_reporter: net_report::Addr, - /// Handle to the underlying quinn::Endpoint. - /// - /// Used in netcheck for QUIC address discovery. - quic_endpoint: Arc>>, - /// The state for an active DiscoKey. - disco_secrets: DiscoSecrets, - - /// UDP disco (ping) queue - udp_disco_sender: mpsc::Sender<(SocketAddr, PublicKey, disco::Message)>, - - /// Optional discovery service - discovery: Option>, - - /// Our discovered direct addresses. - direct_addrs: DiscoveredDirectAddrs, - - /// List of CallMeMaybe disco messages that should be sent out after the next endpoint update - /// completes - pending_call_me_maybes: parking_lot::Mutex>, - - /// Indicates the direct addr update state. - direct_addr_update_state: DirectAddrUpdateState, - - /// Skip verification of SSL certificates from relay servers - /// - /// May only be used in tests. - #[cfg(any(test, feature = "test-utils"))] - #[cfg_attr(iroh_docsrs, doc(cfg(any(test, feature = "test-utils"))))] - insecure_skip_relay_cert_verify: bool, -} - -impl MagicSock { - /// Creates a magic [`MagicSock`] listening on [`Options::addr_v4`] and [`Options::addr_v6`]. - pub(crate) async fn spawn(opts: Options) -> Result { - Handle::new(opts).await - } - - /// Returns the relay node we are connected to, that has the best latency. - /// - /// If `None`, then we are not connected to any relay nodes. - pub(crate) fn my_relay(&self) -> Option { - self.my_relay.get() - } - - /// Get the current proxy configuration. - pub(crate) fn proxy_url(&self) -> Option<&Url> { - self.proxy_url.as_ref() - } - - /// Sets the relay node with the best latency. - /// - /// If we are not connected to any relay nodes, set this to `None`. - fn set_my_relay(&self, my_relay: Option) -> Option { - self.my_relay.replace(my_relay) - } - - /// Sets the internal `quinn::Endpoint` that is used for QUIC address - /// discovery. - pub(crate) fn set_quic_endpoint(&self, endpoint: Option) { - let mut ep = self - .quic_endpoint - .write() - .expect("MagicSock::endpoint RwLock is poisoned"); - *ep = endpoint; - } - - fn is_closing(&self) -> bool { - self.closing.load(Ordering::Relaxed) - } - - fn is_closed(&self) -> bool { - self.closed.load(Ordering::SeqCst) - } - - fn public_key(&self) -> PublicKey { - self.secret_key.public() - } - - /// Get the cached version of the Ipv4 and Ipv6 addrs of the current connection. - pub(crate) fn local_addr(&self) -> (SocketAddr, Option) { - *self.local_addrs.read().expect("not poisoned") - } - - /// Returns `true` if we have at least one candidate address where we can send packets to. - pub(crate) fn has_send_address(&self, node_key: PublicKey) -> bool { - self.remote_info(node_key) - .map(|info| info.has_send_address()) - .unwrap_or(false) - } - - /// Return the [`RemoteInfo`]s of all nodes in the node map. - pub(crate) fn list_remote_infos(&self) -> Vec { - self.node_map.list_remote_infos(Instant::now()) - } - - /// Return the [`RemoteInfo`] for a single node in the node map. - pub(crate) fn remote_info(&self, node_id: NodeId) -> Option { - self.node_map.remote_info(node_id) - } - - /// Returns the direct addresses as a stream. - /// - /// The [`MagicSock`] continuously monitors the direct addresses, the network addresses - /// it might be able to be contacted on, for changes. Whenever changes are detected - /// this stream will yield a new list of addresses. - /// - /// Upon the first creation on the [`MagicSock`] it may not yet have completed a first - /// direct addresses discovery, in this case the first item of the stream will not be - /// immediately available. Once this first set of direct addresses are discovered the - /// stream will always return the first set of addresses immediately, which are the most - /// recently discovered addresses. - /// - /// To get the current direct addresses, drop the stream after the first item was - /// received. - pub(crate) fn direct_addresses(&self) -> DirectAddrsStream { - self.direct_addrs.updates_stream() - } - - /// Watch for changes to the home relay. - /// - /// Note that this can be used to wait for the initial home relay to be known. If the home - /// relay is known at this point, it will be the first item in the stream. - pub(crate) fn watch_home_relay(&self) -> impl Stream { - let current = futures_lite::stream::iter(self.my_relay()); - let changes = self - .my_relay - .watch() - .into_stream() - .filter_map(|maybe_relay| maybe_relay); - current.chain(changes) - } - - /// Returns a stream that reports the [`ConnectionType`] we have to the - /// given `node_id`. - /// - /// The `NodeMap` continuously monitors the `node_id`'s endpoint for - /// [`ConnectionType`] changes, and sends the latest [`ConnectionType`] - /// on the stream. - /// - /// The current [`ConnectionType`] will the the initial entry on the stream. - /// - /// # Errors - /// - /// Will return an error if there is no address information known about the - /// given `node_id`. - pub(crate) fn conn_type_stream(&self, node_id: NodeId) -> Result { - self.node_map.conn_type_stream(node_id) - } - - /// Returns the socket address which can be used by the QUIC layer to dial this node. - pub(crate) fn get_mapping_addr(&self, node_id: NodeId) -> Option { - self.node_map.get_quic_mapped_addr_for_node_key(node_id) - } - - /// Add addresses for a node to the magic socket's addresbook. - #[instrument(skip_all, fields(me = %self.me))] - pub fn add_node_addr(&self, mut addr: NodeAddr, source: node_map::Source) -> Result<()> { - let mut pruned = 0; - for my_addr in self.direct_addrs.sockaddrs() { - if addr.info.direct_addresses.remove(&my_addr) { - warn!( node_id=addr.node_id.fmt_short(), %my_addr, %source, "not adding our addr for node"); - pruned += 1; - } - } - if !addr.info.is_empty() { - self.node_map.add_node_addr(addr, source); - Ok(()) - } else if pruned != 0 { - Err(anyhow::anyhow!( - "empty addressing info, {pruned} direct addresses have been pruned" - )) - } else { - Err(anyhow::anyhow!("empty addressing info")) - } - } - - /// Stores a new set of direct addresses. - /// - /// If the direct addresses have changed from the previous set, they are published to - /// discovery. - pub(super) fn store_direct_addresses(&self, addrs: BTreeSet) { - let updated = self.direct_addrs.update(addrs); - if updated { - self.node_map - .on_direct_addr_discovered(self.direct_addrs.sockaddrs()); - self.publish_my_addr(); - } - } - - /// Get a reference to the DNS resolver used in this [`MagicSock`]. - pub(crate) fn dns_resolver(&self) -> &DnsResolver { - &self.dns_resolver - } - - /// Reference to optional discovery service - pub(crate) fn discovery(&self) -> Option<&dyn Discovery> { - self.discovery.as_ref().map(Box::as_ref) - } - - /// Call to notify the system of potential network changes. - pub(crate) async fn network_change(&self) { - self.actor_sender - .send(ActorMessage::NetworkChange) - .await - .ok(); - } - - #[cfg(test)] - async fn force_network_change(&self, is_major: bool) { - self.actor_sender - .send(ActorMessage::ForceNetworkChange(is_major)) - .await - .ok(); - } - - #[cfg_attr(windows, allow(dead_code))] - fn normalized_local_addr(&self) -> io::Result { - let (v4, v6) = self.local_addr(); - let addr = if let Some(v6) = v6 { v6 } else { v4 }; - Ok(addr) - } - - fn create_io_poller(&self) -> Pin> { - // To do this properly the MagicSock would need a registry of pollers. For each - // node we would look up the poller or create one. Then on each try_send we can - // look up the correct poller and configure it to poll the paths it needs. - // - // Note however that the current quinn impl calls UdpPoller::poll_writable() - // **before** it calls try_send(), as opposed to how it is documented. That is a - // problem as we would not yet know the path that needs to be polled. To avoid such - // ambiguity the API could be changed to a .poll_send(&self, cx: &mut Context, - // io_poller: Pin<&mut dyn UdpPoller>, transmit: &Transmit) -> Poll> - // instead of the existing .try_send() because then we would have control over this. - // - // Right now however we have one single poller behaving the same for each - // connection. It checks all paths and returns Poll::Ready as soon as any path is - // ready. - let ipv4_poller = Arc::new(self.pconn4.clone()).create_io_poller(); - let ipv6_poller = self - .pconn6 - .as_ref() - .map(|sock| Arc::new(sock.clone()).create_io_poller()); - let relay_sender = self.relay_actor_sender.clone(); - Box::pin(IoPoller { - ipv4_poller, - ipv6_poller, - relay_sender, - relay_send_waker: self.network_send_wakers.clone(), - }) - } - - /// Implementation for AsyncUdpSocket::try_send - #[instrument(skip_all)] - fn try_send(&self, transmit: &quinn_udp::Transmit) -> io::Result<()> { - inc_by!(MagicsockMetrics, send_data, transmit.contents.len() as _); - - if self.is_closed() { - inc_by!( - MagicsockMetrics, - send_data_network_down, - transmit.contents.len() as _ - ); - return Err(io::Error::new( - io::ErrorKind::NotConnected, - "connection closed", - )); - } - - let dest = QuicMappedAddr(transmit.destination); - trace!( - dst = %dest, - src = ?transmit.src_ip, - len = %transmit.contents.len(), - "sending", - ); - let mut transmit = transmit.clone(); - match self - .node_map - .get_send_addrs(dest, self.ipv6_reported.load(Ordering::Relaxed)) - { - Some((node_id, udp_addr, relay_url, msgs)) => { - let mut pings_sent = false; - // If we have pings to send, we *have* to send them out first. - if !msgs.is_empty() { - if let Err(err) = self.try_send_ping_actions(msgs) { - warn!( - node = %node_id.fmt_short(), - "failed to handle ping actions: {err:#}", - ); - } - pings_sent = true; - } - - let mut udp_sent = false; - let mut udp_error = None; - let mut relay_sent = false; - let mut relay_error = None; - - // send udp - if let Some(addr) = udp_addr { - // rewrite target address - transmit.destination = addr; - match self.try_send_udp(addr, &transmit) { - Ok(()) => { - trace!(node = %node_id.fmt_short(), dst = %addr, - "sent transmit over UDP"); - udp_sent = true; - } - Err(err) => { - error!(node = %node_id.fmt_short(), dst = %addr, - "failed to send udp: {err:#}"); - udp_error = Some(err); - } - } - } - - // send relay - if let Some(ref relay_url) = relay_url { - match self.try_send_relay(relay_url, node_id, split_packets(&transmit)) { - Ok(()) => { - relay_sent = true; - } - Err(err) => { - relay_error = Some(err); - } - } - } - - let udp_pending = udp_error - .as_ref() - .map(|err| err.kind() == io::ErrorKind::WouldBlock) - .unwrap_or_default(); - let relay_pending = relay_error - .as_ref() - .map(|err| err.kind() == io::ErrorKind::WouldBlock) - .unwrap_or_default(); - if udp_pending && relay_pending { - // Handle backpressure. - Err(io::Error::new(io::ErrorKind::WouldBlock, "pending")) - } else { - if relay_sent || udp_sent { - trace!( - node = %node_id.fmt_short(), - send_udp = ?udp_addr, - send_relay = ?relay_url, - "sent transmit", - ); - } else if !pings_sent { - // Returning Ok here means we let QUIC handle a timeout for a lost - // packet, same would happen if we returned any errors. The - // philosophy of quinn-udp is that a UDP connection could come back - // at any time so these errors should be treated as transient and - // are just timeouts. Hence we opt for returning Ok. See - // test_try_send_no_udp_addr_or_relay_url to explore this further. - error!( - node = %node_id.fmt_short(), - "no UDP or relay paths available for node", - ); - } - Ok(()) - } - } - None => { - error!(%dest, "no NodeState for mapped address"); - // Returning Ok here means we let QUIC timeout. Returning WouldBlock - // triggers a hot loop. Returning an error would immediately fail a - // connection. The philosophy of quinn-udp is that a UDP connection could - // come back at any time or missing should be transient so chooses to let - // these kind of errors time out. See test_try_send_no_send_addr to try - // this out. - Ok(()) - } - } - } - - fn try_send_relay( - &self, - url: &RelayUrl, - node: NodeId, - contents: RelayContents, - ) -> io::Result<()> { - trace!( - node = %node.fmt_short(), - relay_url = %url, - count = contents.len(), - len = contents.iter().map(|c| c.len()).sum::(), - "send relay", - ); - let msg = RelayActorMessage::Send { - url: url.clone(), - contents, - remote_node: node, - }; - match self.relay_actor_sender.try_send(msg) { - Ok(_) => { - trace!(node = %node.fmt_short(), relay_url = %url, - "send relay: message queued"); - Ok(()) - } - Err(mpsc::error::TrySendError::Closed(_)) => { - warn!(node = %node.fmt_short(), relay_url = %url, - "send relay: message dropped, channel to actor is closed"); - Err(io::Error::new( - io::ErrorKind::ConnectionReset, - "channel to actor is closed", - )) - } - Err(mpsc::error::TrySendError::Full(_)) => { - warn!(node = %node.fmt_short(), relay_url = %url, - "send relay: message dropped, channel to actor is full"); - Err(io::Error::new( - io::ErrorKind::WouldBlock, - "channel to actor is full", - )) - } - } - } - - fn try_send_udp(&self, addr: SocketAddr, transmit: &quinn_udp::Transmit) -> io::Result<()> { - let conn = self.conn_for_addr(addr)?; - conn.try_send(transmit)?; - let total_bytes: u64 = transmit.contents.len() as u64; - if addr.is_ipv6() { - inc_by!(MagicsockMetrics, send_ipv6, total_bytes); - } else { - inc_by!(MagicsockMetrics, send_ipv4, total_bytes); - } - Ok(()) - } - - fn conn_for_addr(&self, addr: SocketAddr) -> io::Result<&UdpConn> { - let sock = match addr { - SocketAddr::V4(_) => &self.pconn4, - SocketAddr::V6(_) => self - .pconn6 - .as_ref() - .ok_or(io::Error::new(io::ErrorKind::Other, "no IPv6 connection"))?, - }; - Ok(sock) - } - - /// NOTE: Receiving on a [`Self::closed`] socket will return [`Poll::Pending`] indefinitely. - #[instrument(skip_all)] - fn poll_recv( - &self, - cx: &mut Context, - bufs: &mut [io::IoSliceMut<'_>], - metas: &mut [quinn_udp::RecvMeta], - ) -> Poll> { - // FIXME: currently ipv4 load results in ipv6 traffic being ignored - debug_assert_eq!(bufs.len(), metas.len(), "non matching bufs & metas"); - if self.is_closed() { - return Poll::Pending; - } - - // order of polling is: UDPv4, UDPv6, relay - let (msgs, from_ipv4) = match self.pconn4.poll_recv(cx, bufs, metas)? { - Poll::Pending | Poll::Ready(0) => match &self.pconn6 { - Some(conn) => match conn.poll_recv(cx, bufs, metas)? { - Poll::Pending | Poll::Ready(0) => { - return self.poll_recv_relay(cx, bufs, metas); - } - Poll::Ready(n) => (n, false), - }, - None => { - return self.poll_recv_relay(cx, bufs, metas); - } - }, - Poll::Ready(n) => (n, true), - }; - - // Adding the IP address we received something on results in Quinn using this - // address on the send path to send from. However we let Quinn use a - // QuicMappedAddress, not a real address. So we used to substitute our bind address - // here so that Quinn would send on the right address. But that would sometimes - // result in the wrong address family and Windows trips up on that. - // - // What should be done is that this dst_ip from the RecvMeta is stored in the - // NodeState/PathState. Then on the send path it should be retrieved from the - // NodeState/PathSate together with the send address and substituted at send time. - // This is relevant for IPv6 link-local addresses where the OS otherwise does not - // know which intervace to send from. - #[cfg(not(windows))] - let dst_ip = self.normalized_local_addr().ok().map(|addr| addr.ip()); - // Reasoning for this here: - // https://github.com/n0-computer/iroh/pull/2595#issuecomment-2290947319 - #[cfg(windows)] - let dst_ip = None; - - let mut quic_packets_total = 0; - - for (meta, buf) in metas.iter_mut().zip(bufs.iter_mut()).take(msgs) { - let mut is_quic = false; - let mut quic_packets_count = 0; - if meta.len > meta.stride { - trace!(%meta.len, %meta.stride, "GRO datagram received"); - inc!(MagicsockMetrics, recv_gro_datagrams); - } - - // find disco and stun packets and forward them to the actor - for packet in buf[..meta.len].chunks_mut(meta.stride) { - if packet.len() < meta.stride { - trace!( - len = %packet.len(), - %meta.stride, - "Last GRO datagram smaller than stride", - ); - } - - let packet_is_quic = if stun::is(packet) { - trace!(src = %meta.addr, len = %meta.stride, "UDP recv: stun packet"); - let packet2 = Bytes::copy_from_slice(packet); - self.net_reporter.receive_stun_packet(packet2, meta.addr); - false - } else if let Some((sender, sealed_box)) = disco::source_and_box(packet) { - // Disco? - trace!(src = %meta.addr, len = %meta.stride, "UDP recv: disco packet"); - self.handle_disco_message( - sender, - sealed_box, - DiscoMessageSource::Udp(meta.addr), - ); - false - } else { - trace!(src = %meta.addr, len = %meta.stride, "UDP recv: quic packet"); - if from_ipv4 { - inc_by!(MagicsockMetrics, recv_data_ipv4, packet.len() as _); - } else { - inc_by!(MagicsockMetrics, recv_data_ipv6, packet.len() as _); - } - true - }; - - if packet_is_quic { - quic_packets_count += 1; - is_quic = true; - } else { - // overwrite the first byte of the packets with zero. - // this makes quinn reliably and quickly ignore the packet as long as - // [`quinn::EndpointConfig::grease_quic_bit`] is set to `false` - // (which we always do in Endpoint::bind). - packet[0] = 0u8; - } - } - - if is_quic { - // remap addr - match self.node_map.receive_udp(meta.addr) { - None => { - warn!(src = ?meta.addr, count = %quic_packets_count, len = meta.len, "UDP recv quic packets: no node state found, skipping"); - // if we have no node state for the from addr, set len to 0 to make quinn skip the buf completely. - meta.len = 0; - } - Some((node_id, quic_mapped_addr)) => { - trace!(src = ?meta.addr, node = %node_id.fmt_short(), count = %quic_packets_count, len = meta.len, "UDP recv quic packets"); - quic_packets_total += quic_packets_count; - meta.addr = quic_mapped_addr.0; - } - } - } else { - // if there is no non-stun,non-disco packet in the chunk, set len to zero to make - // quinn skip the buf completely. - meta.len = 0; - } - // Normalize local_ip - meta.dst_ip = dst_ip; - } - - if quic_packets_total > 0 { - inc_by!(MagicsockMetrics, recv_datagrams, quic_packets_total as _); - trace!("UDP recv: {} packets", quic_packets_total); - } - - Poll::Ready(Ok(msgs)) - } - - #[instrument(skip_all)] - fn poll_recv_relay( - &self, - cx: &mut Context, - bufs: &mut [io::IoSliceMut<'_>], - metas: &mut [quinn_udp::RecvMeta], - ) -> Poll> { - let mut num_msgs = 0; - for (buf_out, meta_out) in bufs.iter_mut().zip(metas.iter_mut()) { - if self.is_closed() { - break; - } - let mut relay_recv_receiver = self.relay_recv_receiver.lock(); - match relay_recv_receiver.try_recv() { - Err(mpsc::error::TryRecvError::Empty) => { - self.network_recv_wakers.lock().replace(cx.waker().clone()); - break; - } - Err(mpsc::error::TryRecvError::Disconnected) => { - return Poll::Ready(Err(io::Error::new( - io::ErrorKind::NotConnected, - "connection closed", - ))); - } - Ok(Err(err)) => return Poll::Ready(Err(err)), - Ok(Ok((node_id, meta, bytes))) => { - inc_by!(MagicsockMetrics, recv_data_relay, bytes.len() as _); - trace!(src = %meta.addr, node = %node_id.fmt_short(), count = meta.len / meta.stride, len = meta.len, "recv quic packets from relay"); - buf_out[..bytes.len()].copy_from_slice(&bytes); - *meta_out = meta; - num_msgs += 1; - } - } - } - - // If we have any msgs to report, they are in the first `num_msgs_total` slots - if num_msgs > 0 { - inc_by!(MagicsockMetrics, recv_datagrams, num_msgs as _); - Poll::Ready(Ok(num_msgs)) - } else { - Poll::Pending - } - } - - /// Handles a discovery message. - #[instrument("disco_in", skip_all, fields(node = %sender.fmt_short(), %src))] - fn handle_disco_message(&self, sender: PublicKey, sealed_box: &[u8], src: DiscoMessageSource) { - trace!("handle_disco_message start"); - if self.is_closed() { - return; - } - - // We're now reasonably sure we're expecting communication from - // this node, do the heavy crypto lifting to see what they want. - let dm = match self.disco_secrets.unseal_and_decode( - &self.secret_key, - sender, - sealed_box.to_vec(), - ) { - Ok(dm) => dm, - Err(DiscoBoxError::Open(err)) => { - warn!(?err, "failed to open disco box"); - inc!(MagicsockMetrics, recv_disco_bad_key); - return; - } - Err(DiscoBoxError::Parse(err)) => { - // Couldn't parse it, but it was inside a correctly - // signed box, so just ignore it, assuming it's from a - // newer version of Tailscale that we don't - // understand. Not even worth logging about, lest it - // be too spammy for old clients. - - inc!(MagicsockMetrics, recv_disco_bad_parse); - debug!(?err, "failed to parse disco message"); - return; - } - }; - - if src.is_relay() { - inc!(MagicsockMetrics, recv_disco_relay); - } else { - inc!(MagicsockMetrics, recv_disco_udp); - } - - let span = trace_span!("handle_disco", ?dm); - let _guard = span.enter(); - trace!("receive disco message"); - match dm { - disco::Message::Ping(ping) => { - inc!(MagicsockMetrics, recv_disco_ping); - self.handle_ping(ping, sender, src); - } - disco::Message::Pong(pong) => { - inc!(MagicsockMetrics, recv_disco_pong); - self.node_map.handle_pong(sender, &src, pong); - } - disco::Message::CallMeMaybe(cm) => { - inc!(MagicsockMetrics, recv_disco_call_me_maybe); - match src { - DiscoMessageSource::Relay { url, .. } => { - event!( - target: "events.net.call-me-maybe.recv", - Level::DEBUG, - remote_node = sender.fmt_short(), - via = ?url, - their_addrs = ?cm.my_numbers, - ); - } - _ => { - warn!("call-me-maybe packets should only come via relay"); - return; - } - } - let ping_actions = self.node_map.handle_call_me_maybe(sender, cm); - for action in ping_actions { - match action { - PingAction::SendCallMeMaybe { .. } => { - warn!("Unexpected CallMeMaybe as response of handling a CallMeMaybe"); - } - PingAction::SendPing(ping) => { - self.send_ping_queued(ping); - } - } - } - } - } - trace!("disco message handled"); - } - - /// Handle a ping message. - fn handle_ping(&self, dm: disco::Ping, sender: NodeId, src: DiscoMessageSource) { - // Insert the ping into the node map, and return whether a ping with this tx_id was already - // received. - let addr: SendAddr = src.clone().into(); - let handled = self.node_map.handle_ping(sender, addr.clone(), dm.tx_id); - match handled.role { - PingRole::Duplicate => { - debug!(%src, tx = %hex::encode(dm.tx_id), "received ping: path already confirmed, skip"); - return; - } - PingRole::LikelyHeartbeat => {} - PingRole::NewPath => { - debug!(%src, tx = %hex::encode(dm.tx_id), "received ping: new path"); - } - PingRole::Activate => { - debug!(%src, tx = %hex::encode(dm.tx_id), "received ping: path active"); - } - } - - // Send a pong. - debug!(tx = %hex::encode(dm.tx_id), %addr, dstkey = %sender.fmt_short(), - "sending pong"); - let pong = disco::Message::Pong(disco::Pong { - tx_id: dm.tx_id, - ping_observed_addr: addr.clone(), - }); - event!( - target: "events.net.pong.sent", - Level::DEBUG, - remote_node = %sender.fmt_short(), - dst = ?addr, - txn = ?dm.tx_id, - ); - - if !self.send_disco_message_queued(addr.clone(), sender, pong) { - warn!(%addr, "failed to queue pong"); - } - - if let Some(ping) = handled.needs_ping_back { - debug!( - %addr, - dstkey = %sender.fmt_short(), - "sending direct ping back", - ); - self.send_ping_queued(ping); - } - } - - fn encode_disco_message(&self, dst_key: PublicKey, msg: &disco::Message) -> Bytes { - self.disco_secrets - .encode_and_seal(&self.secret_key, dst_key, msg) - } - - fn send_ping_queued(&self, ping: SendPing) { - let SendPing { - id, - dst, - dst_node, - tx_id, - purpose, - } = ping; - let msg = disco::Message::Ping(disco::Ping { - tx_id, - node_key: self.public_key(), - }); - let sent = match dst { - SendAddr::Udp(addr) => self - .udp_disco_sender - .try_send((addr, dst_node, msg)) - .is_ok(), - SendAddr::Relay(ref url) => self.send_disco_message_relay(url, dst_node, msg), - }; - if sent { - let msg_sender = self.actor_sender.clone(); - trace!(%dst, tx = %hex::encode(tx_id), ?purpose, "ping sent (queued)"); - self.node_map - .notify_ping_sent(id, dst, tx_id, purpose, msg_sender); - } else { - warn!(dst = ?dst, tx = %hex::encode(tx_id), ?purpose, "failed to send ping: queues full"); - } - } - - /// Tries to send the ping actions. - /// - /// Note that on failure the (remaining) ping actions are simply dropped. That's bad! - /// The Endpoint will think a full ping was done and not request a new full-ping for a - /// while. We should probably be buffering the pings. - fn try_send_ping_actions(&self, msgs: Vec) -> io::Result<()> { - for msg in msgs { - // Abort sending as soon as we know we are shutting down. - if self.is_closing() || self.is_closed() { - return Ok(()); - } - match msg { - PingAction::SendCallMeMaybe { - ref relay_url, - dst_node, - } => { - self.send_or_queue_call_me_maybe(relay_url, dst_node); - } - PingAction::SendPing(ping) => { - self.try_send_ping(ping)?; - } - } - } - Ok(()) - } - - /// Send a disco message. UDP messages will be queued. - /// - /// If `dst` is [`SendAddr::Relay`], the message will be pushed into the relay client channel. - /// If `dst` is [`SendAddr::Udp`], the message will be pushed into the udp disco send channel. - /// - /// Returns true if the channel had capacity for the message, and false if the message was - /// dropped. - fn send_disco_message_queued( - &self, - dst: SendAddr, - dst_key: PublicKey, - msg: disco::Message, - ) -> bool { - match dst { - SendAddr::Udp(addr) => self.udp_disco_sender.try_send((addr, dst_key, msg)).is_ok(), - SendAddr::Relay(ref url) => self.send_disco_message_relay(url, dst_key, msg), - } - } - - /// Send a disco message. UDP messages will be polled to send directly on the UDP socket. - fn try_send_disco_message( - &self, - dst: SendAddr, - dst_key: PublicKey, - msg: disco::Message, - ) -> io::Result<()> { - match dst { - SendAddr::Udp(addr) => { - self.try_send_disco_message_udp(addr, dst_key, &msg)?; - } - SendAddr::Relay(ref url) => { - self.send_disco_message_relay(url, dst_key, msg); - } - } - Ok(()) - } - - fn send_disco_message_relay(&self, url: &RelayUrl, dst: NodeId, msg: disco::Message) -> bool { - debug!(node = %dst.fmt_short(), %url, %msg, "send disco message (relay)"); - let pkt = self.encode_disco_message(dst, &msg); - inc!(MagicsockMetrics, send_disco_relay); - match self.try_send_relay(url, dst, smallvec![pkt]) { - Ok(()) => { - if let disco::Message::CallMeMaybe(CallMeMaybe { ref my_numbers }) = msg { - event!( - target: "events.net.call-me-maybe.sent", - Level::DEBUG, - remote_node = %dst.fmt_short(), - via = ?url, - addrs = ?my_numbers, - ); - } - inc!(MagicsockMetrics, sent_disco_relay); - disco_message_sent(&msg); - true - } - Err(_) => false, - } - } - - async fn send_disco_message_udp( - &self, - dst: SocketAddr, - dst_node: NodeId, - msg: &disco::Message, - ) -> io::Result<()> { - futures_lite::future::poll_fn(move |cx| { - loop { - match self.try_send_disco_message_udp(dst, dst_node, msg) { - Ok(()) => return Poll::Ready(Ok(())), - Err(err) if err.kind() == io::ErrorKind::WouldBlock => { - // This is the socket .try_send_disco_message_udp used. - let sock = self.conn_for_addr(dst)?; - let sock = Arc::new(sock.clone()); - let mut poller = sock.create_io_poller(); - match poller.as_mut().poll_writable(cx)? { - Poll::Ready(()) => continue, - Poll::Pending => return Poll::Pending, - } - } - Err(err) => return Poll::Ready(Err(err)), - } - } - }) - .await - } - - fn try_send_disco_message_udp( - &self, - dst: SocketAddr, - dst_node: NodeId, - msg: &disco::Message, - ) -> std::io::Result<()> { - trace!(%dst, %msg, "send disco message (UDP)"); - if self.is_closed() { - return Err(io::Error::new( - io::ErrorKind::NotConnected, - "connection closed", - )); - } - let pkt = self.encode_disco_message(dst_node, msg); - // TODO: These metrics will be wrong with the poll impl - // Also - do we need it? I'd say the `sent_disco_udp` below is enough. - inc!(MagicsockMetrics, send_disco_udp); - let transmit = quinn_udp::Transmit { - destination: dst, - contents: &pkt, - ecn: None, - segment_size: None, - src_ip: None, // TODO - }; - let sent = self.try_send_udp(dst, &transmit); - match sent { - Ok(()) => { - trace!(%dst, node = %dst_node.fmt_short(), %msg, "sent disco message"); - inc!(MagicsockMetrics, sent_disco_udp); - disco_message_sent(msg); - Ok(()) - } - Err(err) => { - warn!(%dst, node = %dst_node.fmt_short(), ?msg, ?err, - "failed to send disco message"); - Err(err) - } - } - } - - #[instrument(skip_all)] - async fn handle_ping_actions(&mut self, msgs: Vec) { - // TODO: This used to make sure that all ping actions are sent. Though on the - // poll_send/try_send path we also do fire-and-forget. try_send_ping_actions() - // really should store any unsent pings on the Inner and send them at the next - // possible time. - if let Err(err) = self.try_send_ping_actions(msgs) { - warn!("Not all ping actions were sent: {err:#}"); - } - } - - fn try_send_ping(&self, ping: SendPing) -> io::Result<()> { - let SendPing { - id, - dst, - dst_node, - tx_id, - purpose, - } = ping; - let msg = disco::Message::Ping(disco::Ping { - tx_id, - node_key: self.public_key(), - }); - self.try_send_disco_message(dst.clone(), dst_node, msg)?; - debug!(%dst, tx = %hex::encode(tx_id), ?purpose, "ping sent (polled)"); - let msg_sender = self.actor_sender.clone(); - self.node_map - .notify_ping_sent(id, dst.clone(), tx_id, purpose, msg_sender); - Ok(()) - } - - fn poll_send_relay( - &self, - url: &RelayUrl, - node: PublicKey, - contents: RelayContents, - ) -> Poll { - trace!(node = %node.fmt_short(), relay_url = %url, count = contents.len(), len = contents.iter().map(|c| c.len()).sum::(), "send relay"); - let msg = RelayActorMessage::Send { - url: url.clone(), - contents, - remote_node: node, - }; - match self.relay_actor_sender.try_send(msg) { - Ok(_) => { - trace!(node = %node.fmt_short(), relay_url = %url, "send relay: message queued"); - Poll::Ready(true) - } - Err(mpsc::error::TrySendError::Closed(_)) => { - warn!(node = %node.fmt_short(), relay_url = %url, "send relay: message dropped, channel to actor is closed"); - Poll::Ready(false) - } - Err(mpsc::error::TrySendError::Full(_)) => { - warn!(node = %node.fmt_short(), relay_url = %url, "send relay: message dropped, channel to actor is full"); - Poll::Pending - } - } - } - - fn send_queued_call_me_maybes(&self) { - let msg = self.direct_addrs.to_call_me_maybe_message(); - let msg = disco::Message::CallMeMaybe(msg); - for (public_key, url) in self.pending_call_me_maybes.lock().drain() { - if !self.send_disco_message_relay(&url, public_key, msg.clone()) { - warn!(node = %public_key.fmt_short(), "relay channel full, dropping call-me-maybe"); - } - } - } - - /// Sends the call-me-maybe DISCO message, queuing if addresses are too stale. - /// - /// To send the call-me-maybe message, we need to know our current direct addresses. If - /// this information is too stale, the call-me-maybe is queued while a net_report run is - /// scheduled. Once this run finishes, the call-me-maybe will be sent. - fn send_or_queue_call_me_maybe(&self, url: &RelayUrl, dst_node: NodeId) { - match self.direct_addrs.fresh_enough() { - Ok(()) => { - let msg = self.direct_addrs.to_call_me_maybe_message(); - let msg = disco::Message::CallMeMaybe(msg); - if !self.send_disco_message_relay(url, dst_node, msg) { - warn!(dstkey = %dst_node.fmt_short(), relayurl = %url, - "relay channel full, dropping call-me-maybe"); - } else { - debug!(dstkey = %dst_node.fmt_short(), relayurl = %url, "call-me-maybe sent"); - } - } - Err(last_refresh_ago) => { - self.pending_call_me_maybes - .lock() - .insert(dst_node, url.clone()); - debug!( - ?last_refresh_ago, - "want call-me-maybe but direct addrs stale; queuing after restun", - ); - self.re_stun("refresh-for-peering"); - } - } - } - - /// Triggers an address discovery. The provided why string is for debug logging only. - #[instrument(skip_all)] - fn re_stun(&self, why: &'static str) { - debug!("re_stun: {}", why); - inc!(MagicsockMetrics, re_stun_calls); - self.direct_addr_update_state.schedule_run(why); - } - - /// Publishes our address to a discovery service, if configured. - /// - /// Called whenever our addresses or home relay node changes. - fn publish_my_addr(&self) { - if let Some(ref discovery) = self.discovery { - let info = AddrInfo { - relay_url: self.my_relay(), - direct_addresses: self.direct_addrs.sockaddrs(), - }; - discovery.publish(&info); - } - } -} - -#[derive(Clone, Debug)] -enum DiscoMessageSource { - Udp(SocketAddr), - Relay { url: RelayUrl, key: PublicKey }, -} - -impl Display for DiscoMessageSource { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::Udp(addr) => write!(f, "Udp({addr})"), - Self::Relay { ref url, key } => write!(f, "Relay({url}, {})", key.fmt_short()), - } - } -} - -impl From for SendAddr { - fn from(value: DiscoMessageSource) -> Self { - match value { - DiscoMessageSource::Udp(addr) => SendAddr::Udp(addr), - DiscoMessageSource::Relay { url, .. } => SendAddr::Relay(url), - } - } -} - -impl From<&DiscoMessageSource> for SendAddr { - fn from(value: &DiscoMessageSource) -> Self { - match value { - DiscoMessageSource::Udp(addr) => SendAddr::Udp(*addr), - DiscoMessageSource::Relay { url, .. } => SendAddr::Relay(url.clone()), - } - } -} - -impl DiscoMessageSource { - fn is_relay(&self) -> bool { - matches!(self, DiscoMessageSource::Relay { .. }) - } -} - -/// Manages currently running direct addr discovery, aka net_report runs. -/// -/// Invariants: -/// - only one direct addr update must be running at a time -/// - if an update is scheduled while another one is running, remember that -/// and start a new one when the current one has finished -#[derive(Debug)] -struct DirectAddrUpdateState { - /// If running, set to the reason for the currently the update. - running: sync::watch::Sender>, - /// If set, start a new update as soon as the current one is finished. - want_update: parking_lot::Mutex>, -} - -impl DirectAddrUpdateState { - fn new() -> Self { - let (running, _) = sync::watch::channel(None); - DirectAddrUpdateState { - running, - want_update: Default::default(), - } - } - - /// Schedules a new run, either starting it immediately if none is running or - /// scheduling it for later. - fn schedule_run(&self, why: &'static str) { - if self.is_running() { - let _ = self.want_update.lock().insert(why); - } else { - self.run(why); - } - } - - /// Returns `true` if an update is currently in progress. - fn is_running(&self) -> bool { - self.running.borrow().is_some() - } - - /// Trigger a new run. - fn run(&self, why: &'static str) { - self.running.send(Some(why)).ok(); - } - - /// Clears the current running state. - fn finish_run(&self) { - self.running.send(None).ok(); - } - - /// Returns the next update, if one is set. - fn next_update(&self) -> Option<&'static str> { - self.want_update.lock().take() - } -} - -impl Handle { - /// Creates a magic [`MagicSock`] listening on [`Options::addr_v4`] and [`Options::addr_v6`]. - async fn new(opts: Options) -> Result { - let me = opts.secret_key.public().fmt_short(); - if crate::util::relay_only_mode() { - warn!( - "creating a MagicSock that will only send packets over a relay relay connection." - ); - } - - Self::with_name(me, opts) - .instrument(error_span!("magicsock")) - .await - } - - async fn with_name(me: String, opts: Options) -> Result { - let port_mapper = portmapper::Client::default(); - - let Options { - addr_v4, - addr_v6, - secret_key, - relay_map, - node_map, - discovery, - dns_resolver, - proxy_url, - #[cfg(any(test, feature = "test-utils"))] - insecure_skip_relay_cert_verify, - } = opts; - - let (relay_recv_sender, relay_recv_receiver) = mpsc::channel(128); - - let (pconn4, pconn6) = bind(addr_v4, addr_v6)?; - let port = pconn4.port(); - - // NOTE: we can end up with a zero port if `std::net::UdpSocket::socket_addr` fails - match port.try_into() { - Ok(non_zero_port) => { - port_mapper.update_local_port(non_zero_port); - } - Err(_zero_port) => debug!("Skipping port mapping with zero local port"), - } - let ipv4_addr = pconn4.local_addr()?; - let ipv6_addr = pconn6.as_ref().and_then(|c| c.local_addr().ok()); - - let net_reporter = - net_report::Client::new(Some(port_mapper.clone()), dns_resolver.clone())?; - - let (actor_sender, actor_receiver) = mpsc::channel(256); - let (relay_actor_sender, relay_actor_receiver) = mpsc::channel(256); - let (udp_disco_sender, mut udp_disco_receiver) = mpsc::channel(256); - - // load the node data - let node_map = node_map.unwrap_or_default(); - let node_map = NodeMap::load_from_vec(node_map); - - let inner = Arc::new(MagicSock { - me, - port: AtomicU16::new(port), - secret_key, - proxy_url, - local_addrs: std::sync::RwLock::new((ipv4_addr, ipv6_addr)), - closing: AtomicBool::new(false), - closed: AtomicBool::new(false), - relay_recv_receiver: parking_lot::Mutex::new(relay_recv_receiver), - network_recv_wakers: parking_lot::Mutex::new(None), - network_send_wakers: Arc::new(parking_lot::Mutex::new(None)), - actor_sender: actor_sender.clone(), - ipv6_reported: Arc::new(AtomicBool::new(false)), - relay_map, - my_relay: Default::default(), - pconn4: pconn4.clone(), - pconn6: pconn6.clone(), - net_reporter: net_reporter.addr(), - disco_secrets: DiscoSecrets::default(), - node_map, - relay_actor_sender: relay_actor_sender.clone(), - udp_disco_sender, - discovery, - direct_addrs: Default::default(), - pending_call_me_maybes: Default::default(), - direct_addr_update_state: DirectAddrUpdateState::new(), - dns_resolver, - #[cfg(any(test, feature = "test-utils"))] - insecure_skip_relay_cert_verify, - quic_endpoint: Arc::new(RwLock::new(None)), - }); - - let mut actor_tasks = JoinSet::default(); - - let relay_actor = RelayActor::new(inner.clone(), actor_sender.clone()); - let relay_actor_cancel_token = relay_actor.cancel_token(); - actor_tasks.spawn( - async move { - relay_actor.run(relay_actor_receiver).await; - } - .instrument(info_span!("relay-actor")), - ); - - let inner2 = inner.clone(); - actor_tasks.spawn(async move { - while let Some((dst, dst_key, msg)) = udp_disco_receiver.recv().await { - if let Err(err) = inner2.send_disco_message_udp(dst, dst_key, &msg).await { - warn!(%dst, node = %dst_key.fmt_short(), ?err, "failed to send disco message (UDP)"); - } - } - }); - - let inner2 = inner.clone(); - let network_monitor = netmon::Monitor::new().await?; - actor_tasks.spawn( - async move { - let actor = Actor { - msg_receiver: actor_receiver, - msg_sender: actor_sender, - relay_actor_sender, - relay_actor_cancel_token, - msock: inner2, - relay_recv_sender, - periodic_re_stun_timer: new_re_stun_timer(false), - net_info_last: None, - port_mapper, - pconn4, - pconn6, - no_v4_send: false, - net_reporter, - network_monitor, - }; - - if let Err(err) = actor.run().await { - warn!("relay handler errored: {:?}", err); - } - } - .instrument(info_span!("actor")), - ); - - let c = Handle { - msock: inner, - actor_tasks: Arc::new(Mutex::new(actor_tasks)), - }; - - Ok(c) - } - - /// Closes the connection. - /// - /// Only the first close does anything. Any later closes return nil. - /// Polling the socket ([`AsyncUdpSocket::poll_recv`]) will return [`Poll::Pending`] - /// indefinitely after this call. - #[instrument(skip_all, fields(me = %self.msock.me))] - pub(crate) async fn close(&self) -> Result<()> { - if self.msock.is_closed() { - return Ok(()); - } - self.msock.closing.store(true, Ordering::Relaxed); - self.msock.actor_sender.send(ActorMessage::Shutdown).await?; - self.msock.closed.store(true, Ordering::SeqCst); - self.msock.direct_addrs.addrs.shutdown(); - - let mut tasks = self.actor_tasks.lock().await; - - // give the tasks a moment to shutdown cleanly - let tasks_ref = &mut tasks; - let shutdown_done = time::timeout(Duration::from_millis(100), async move { - while let Some(task) = tasks_ref.join_next().await { - if let Err(err) = task { - warn!("unexpected error in task shutdown: {:?}", err); - } - } - }) - .await; - if shutdown_done.is_ok() { - debug!("tasks shutdown complete"); - } else { - // shutdown all tasks - debug!("aborting remaining {}/3 tasks", tasks.len()); - tasks.shutdown().await; - } - - Ok(()) - } -} - -#[derive(Debug, Default)] -struct DiscoSecrets(parking_lot::Mutex>); - -impl DiscoSecrets { - fn get( - &self, - secret: &SecretKey, - node_id: PublicKey, - ) -> parking_lot::MappedMutexGuard { - parking_lot::MutexGuard::map(self.0.lock(), |inner| { - inner - .entry(node_id) - .or_insert_with(|| secret.shared(&node_id)) - }) - } - - pub fn encode_and_seal( - &self, - secret_key: &SecretKey, - node_id: PublicKey, - msg: &disco::Message, - ) -> Bytes { - let mut seal = msg.as_bytes(); - self.get(secret_key, node_id).seal(&mut seal); - disco::encode_message(&secret_key.public(), seal).into() - } - - pub fn unseal_and_decode( - &self, - secret: &SecretKey, - node_id: PublicKey, - mut sealed_box: Vec, - ) -> Result { - self.get(secret, node_id) - .open(&mut sealed_box) - .map_err(DiscoBoxError::Open)?; - disco::Message::from_bytes(&sealed_box).map_err(DiscoBoxError::Parse) - } -} - -#[derive(Debug, thiserror::Error)] -enum DiscoBoxError { - #[error("Failed to open crypto box")] - Open(anyhow::Error), - #[error("Failed to parse disco message")] - Parse(anyhow::Error), -} - -type RelayRecvResult = Result<(PublicKey, quinn_udp::RecvMeta, Bytes), io::Error>; - -impl AsyncUdpSocket for Handle { - fn create_io_poller(self: Arc) -> Pin> { - self.msock.create_io_poller() - } - - fn try_send(&self, transmit: &quinn_udp::Transmit) -> io::Result<()> { - self.msock.try_send(transmit) - } - - /// NOTE: Receiving on a [`Self::close`]d socket will return [`Poll::Pending`] indefinitely. - fn poll_recv( - &self, - cx: &mut Context, - bufs: &mut [io::IoSliceMut<'_>], - metas: &mut [quinn_udp::RecvMeta], - ) -> Poll> { - self.msock.poll_recv(cx, bufs, metas) - } - - fn local_addr(&self) -> io::Result { - match &*self.msock.local_addrs.read().expect("not poisoned") { - (ipv4, None) => { - // Pretend to be IPv6, because our QuinnMappedAddrs - // need to be IPv6. - let ip: IpAddr = match ipv4.ip() { - IpAddr::V4(ip) => ip.to_ipv6_mapped().into(), - IpAddr::V6(ip) => ip.into(), - }; - Ok(SocketAddr::new(ip, ipv4.port())) - } - (_, Some(ipv6)) => Ok(*ipv6), - } - } - - fn max_transmit_segments(&self) -> usize { - if let Some(pconn6) = self.pconn6.as_ref() { - std::cmp::min( - pconn6.max_transmit_segments(), - self.pconn4.max_transmit_segments(), - ) - } else { - self.pconn4.max_transmit_segments() - } - } - - fn max_receive_segments(&self) -> usize { - if let Some(pconn6) = self.pconn6.as_ref() { - // `max_receive_segments` controls the size of the `RecvMeta` buffer - // that quinn creates. Having buffers slightly bigger than necessary - // isn't terrible, and makes sure a single socket can read the maximum - // amount with a single poll. We considered adding these numbers instead, - // but we never get data from both sockets at the same time in `poll_recv` - // and it's impossible and unnecessary to be refactored that way. - std::cmp::max( - pconn6.max_receive_segments(), - self.pconn4.max_receive_segments(), - ) - } else { - self.pconn4.max_receive_segments() - } - } - - fn may_fragment(&self) -> bool { - if let Some(pconn6) = self.pconn6.as_ref() { - pconn6.may_fragment() || self.pconn4.may_fragment() - } else { - self.pconn4.may_fragment() - } - } -} - -#[derive(Debug)] -struct IoPoller { - ipv4_poller: Pin>, - ipv6_poller: Option>>, - relay_sender: mpsc::Sender, - relay_send_waker: Arc>>, -} - -impl quinn::UdpPoller for IoPoller { - fn poll_writable(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - // This version returns Ready as soon as any of them are ready. - let this = &mut *self; - match this.ipv4_poller.as_mut().poll_writable(cx) { - Poll::Ready(_) => return Poll::Ready(Ok(())), - Poll::Pending => (), - } - if let Some(ref mut ipv6_poller) = this.ipv6_poller { - match ipv6_poller.as_mut().poll_writable(cx) { - Poll::Ready(_) => return Poll::Ready(Ok(())), - Poll::Pending => (), - } - } - match this.relay_sender.capacity() { - 0 => { - self.relay_send_waker.lock().replace(cx.waker().clone()); - Poll::Pending - } - _ => Poll::Ready(Ok(())), - } - } -} - -#[derive(Debug)] -enum ActorMessage { - Shutdown, - ReceiveRelay(RelayReadResult), - EndpointPingExpired(usize, stun_rs::TransactionId), - NetReport(Result>>, &'static str), - NetworkChange, - #[cfg(test)] - ForceNetworkChange(bool), -} - -struct Actor { - msock: Arc, - msg_receiver: mpsc::Receiver, - msg_sender: mpsc::Sender, - relay_actor_sender: mpsc::Sender, - relay_actor_cancel_token: CancellationToken, - /// Channel to send received relay messages on, for processing. - relay_recv_sender: mpsc::Sender, - /// When set, is an AfterFunc timer that will call MagicSock::do_periodic_stun. - periodic_re_stun_timer: time::Interval, - /// The `NetInfo` provided in the last call to `net_info_func`. It's used to deduplicate calls to netInfoFunc. - net_info_last: Option, - - // The underlying UDP sockets used to send/rcv packets. - pconn4: UdpConn, - pconn6: Option, - - /// The NAT-PMP/PCP/UPnP prober/client, for requesting port mappings from NAT devices. - port_mapper: portmapper::Client, - - /// Whether IPv4 UDP is known to be unable to transmit - /// at all. This could happen if the socket is in an invalid state - /// (as can happen on darwin after a network link status change). - no_v4_send: bool, - - /// The prober that discovers local network conditions, including the closest relay relay and NAT mappings. - net_reporter: net_report::Client, - - network_monitor: netmon::Monitor, -} - -impl Actor { - async fn run(mut self) -> Result<()> { - // Setup network monitoring - let (link_change_s, mut link_change_r) = mpsc::channel(8); - let _token = self - .network_monitor - .subscribe(move |is_major| { - let link_change_s = link_change_s.clone(); - async move { - link_change_s.send(is_major).await.ok(); - } - .boxed() - }) - .await?; - - // Let the the heartbeat only start a couple seconds later - let mut direct_addr_heartbeat_timer = time::interval_at( - time::Instant::now() + HEARTBEAT_INTERVAL, - HEARTBEAT_INTERVAL, - ); - let mut direct_addr_update_receiver = - self.msock.direct_addr_update_state.running.subscribe(); - let mut portmap_watcher = self.port_mapper.watch_external_address(); - - let mut discovery_events: BoxStream = - Box::pin(futures_lite::stream::empty()); - if let Some(d) = self.msock.discovery() { - if let Some(events) = d.subscribe() { - discovery_events = events; - } - } - - let mut receiver_closed = false; - let mut portmap_watcher_closed = false; - let mut link_change_closed = false; - loop { - inc!(Metrics, actor_tick_main); - tokio::select! { - msg = self.msg_receiver.recv(), if !receiver_closed => { - let Some(msg) = msg else { - trace!("tick: magicsock receiver closed"); - inc!(Metrics, actor_tick_other); - - receiver_closed = true; - continue; - }; - - trace!(?msg, "tick: msg"); - inc!(Metrics, actor_tick_msg); - if self.handle_actor_message(msg).await { - return Ok(()); - } - } - tick = self.periodic_re_stun_timer.tick() => { - trace!("tick: re_stun {:?}", tick); - inc!(Metrics, actor_tick_re_stun); - self.msock.re_stun("periodic"); - } - change = portmap_watcher.changed(), if !portmap_watcher_closed => { - if change.is_err() { - trace!("tick: portmap watcher closed"); - inc!(Metrics, actor_tick_other); - - portmap_watcher_closed = true; - continue; - } - - trace!("tick: portmap changed"); - inc!(Metrics, actor_tick_portmap_changed); - let new_external_address = *portmap_watcher.borrow(); - debug!("external address updated: {new_external_address:?}"); - self.msock.re_stun("portmap_updated"); - }, - _ = direct_addr_heartbeat_timer.tick() => { - trace!( - "tick: direct addr heartbeat {} direct addrs", - self.msock.node_map.node_count(), - ); - inc!(Metrics, actor_tick_direct_addr_heartbeat); - // TODO: this might trigger too many packets at once, pace this - - self.msock.node_map.prune_inactive(); - let msgs = self.msock.node_map.nodes_stayin_alive(); - self.handle_ping_actions(msgs).await; - } - _ = direct_addr_update_receiver.changed() => { - let reason = *direct_addr_update_receiver.borrow(); - trace!("tick: direct addr update receiver {:?}", reason); - inc!(Metrics, actor_tick_direct_addr_update_receiver); - if let Some(reason) = reason { - self.refresh_direct_addrs(reason).await; - } - } - is_major = link_change_r.recv(), if !link_change_closed => { - let Some(is_major) = is_major else { - trace!("tick: link change receiver closed"); - inc!(Metrics, actor_tick_other); - - link_change_closed = true; - continue; - }; - - trace!("tick: link change {}", is_major); - inc!(Metrics, actor_link_change); - self.handle_network_change(is_major).await; - } - // Even if `discovery_events` yields `None`, it could begin to yield - // `Some` again in the future, so we don't want to disable this branch - // forever like we do with the other branches that yield `Option`s - Some(discovery_item) = discovery_events.next() => { - trace!("tick: discovery event, address discovered: {discovery_item:?}"); - let node_addr = NodeAddr {node_id: discovery_item.node_id, info: discovery_item.addr_info}; - if let Err(e) = self.msock.add_node_addr(node_addr.clone(), Source::Discovery { name: discovery_item.provenance.into() }) { - warn!(?node_addr, "unable to add discovered node address to the node map: {e:?}"); - } - } - } - } - } - - async fn handle_network_change(&mut self, is_major: bool) { - debug!("link change detected: major? {}", is_major); - - if is_major { - self.msock.dns_resolver.clear_cache(); - self.msock.re_stun("link-change-major"); - self.close_stale_relay_connections().await; - self.reset_endpoint_states(); - } else { - self.msock.re_stun("link-change-minor"); - } - } - - #[instrument(skip_all)] - async fn handle_ping_actions(&mut self, msgs: Vec) { - // TODO: This used to make sure that all ping actions are sent. Though on the - // poll_send/try_send path we also do fire-and-forget. try_send_ping_actions() - // really should store any unsent pings on the Inner and send them at the next - // possible time. - if let Err(err) = self.msock.try_send_ping_actions(msgs) { - warn!("Not all ping actions were sent: {err:#}"); - } - } - - /// Processes an incoming actor message. - /// - /// Returns `true` if it was a shutdown. - async fn handle_actor_message(&mut self, msg: ActorMessage) -> bool { - match msg { - ActorMessage::Shutdown => { - debug!("shutting down"); - - self.msock.node_map.notify_shutdown(); - self.port_mapper.deactivate(); - self.relay_actor_cancel_token.cancel(); - - // Ignore errors from pconnN - // They will frequently have been closed already by a call to connBind.Close. - debug!("stopping connections"); - if let Some(ref conn) = self.pconn6 { - conn.close().await.ok(); - } - self.pconn4.close().await.ok(); - - debug!("shutdown complete"); - return true; - } - ActorMessage::ReceiveRelay(read_result) => { - let passthroughs = self.process_relay_read_result(read_result); - for passthrough in passthroughs { - self.relay_recv_sender - .send(passthrough) - .await - .expect("missing recv sender"); - let mut wakers = self.msock.network_recv_wakers.lock(); - if let Some(waker) = wakers.take() { - waker.wake(); - } - } - } - ActorMessage::EndpointPingExpired(id, txid) => { - self.msock.node_map.notify_ping_timeout(id, txid); - } - ActorMessage::NetReport(report, why) => { - match report { - Ok(report) => { - self.handle_net_report_report(report).await; - } - Err(err) => { - warn!( - "failed to generate net_report report for: {}: {:?}", - why, err - ); - } - } - self.finalize_direct_addrs_update(why); - } - ActorMessage::NetworkChange => { - self.network_monitor.network_change().await.ok(); - } - #[cfg(test)] - ActorMessage::ForceNetworkChange(is_major) => { - self.handle_network_change(is_major).await; - } - } - - false - } - - #[cfg_attr(windows, allow(dead_code))] - fn normalized_local_addr(&self) -> io::Result { - self.msock.normalized_local_addr() - } - - fn process_relay_read_result(&mut self, dm: RelayReadResult) -> Vec { - trace!("process_relay_read {} bytes", dm.buf.len()); - if dm.buf.is_empty() { - warn!("received empty relay packet"); - return Vec::new(); - } - let url = &dm.url; - - let quic_mapped_addr = self.msock.node_map.receive_relay(url, dm.src); - - // the relay packet is made up of multiple udp packets, prefixed by a u16 be length prefix - // - // split the packet into these parts - let parts = PacketSplitIter::new(dm.buf); - // Normalize local_ip - #[cfg(not(windows))] - let dst_ip = self.normalized_local_addr().ok().map(|addr| addr.ip()); - // Reasoning for this here: https://github.com/n0-computer/iroh/pull/2595#issuecomment-2290947319 - - let mut out = Vec::new(); - for part in parts { - match part { - Ok(part) => { - if self.handle_relay_disco_message(&part, url, dm.src) { - // Message was internal, do not bubble up. - continue; - } - - let meta = quinn_udp::RecvMeta { - len: part.len(), - stride: part.len(), - addr: quic_mapped_addr.0, - dst_ip, - ecn: None, - }; - out.push(Ok((dm.src, meta, part))); - } - Err(e) => { - out.push(Err(e)); - } - } - } - - out - } - - /// Refreshes knowledge about our direct addresses. - /// - /// In other words, this triggers a net_report run. - /// - /// Note that invoking this is managed by the [`DirectAddrUpdateState`] and this should - /// never be invoked directly. Some day this will be refactored to not allow this easy - /// mistake to be made. - #[instrument(level = "debug", skip_all)] - async fn refresh_direct_addrs(&mut self, why: &'static str) { - inc!(MagicsockMetrics, update_direct_addrs); - - debug!("starting direct addr update ({})", why); - self.port_mapper.procure_mapping(); - self.update_net_info(why).await; - } - - /// Updates the direct addresses of this magic socket. - /// - /// Updates the [`DiscoveredDirectAddrs`] of this [`MagicSock`] with the current set of - /// direct addresses from: - /// - /// - The portmapper. - /// - A net_report report. - /// - The local interfaces IP addresses. - fn update_direct_addresses(&mut self, net_report_report: Option>) { - let portmap_watcher = self.port_mapper.watch_external_address(); - - // We only want to have one DirectAddr for each SocketAddr we have. So we store - // this as a map of SocketAddr -> DirectAddrType. At the end we will construct a - // DirectAddr from each entry. - let mut addrs: BTreeMap = BTreeMap::new(); - - // First add PortMapper provided addresses. - let maybe_port_mapped = *portmap_watcher.borrow(); - if let Some(portmap_ext) = maybe_port_mapped.map(SocketAddr::V4) { - addrs - .entry(portmap_ext) - .or_insert(DirectAddrType::Portmapped); - self.set_net_info_have_port_map(); - } - - // Next add STUN addresses from the net_report report. - if let Some(net_report_report) = net_report_report { - if let Some(global_v4) = net_report_report.global_v4 { - addrs - .entry(global_v4.into()) - .or_insert(DirectAddrType::Stun); - - // If they're behind a hard NAT and are using a fixed - // port locally, assume they might've added a static - // port mapping on their router to the same explicit - // port that we are running with. Worst case it's an invalid candidate mapping. - let port = self.msock.port.load(Ordering::Relaxed); - if net_report_report - .mapping_varies_by_dest_ip - .unwrap_or_default() - && port != 0 - { - let mut addr = global_v4; - addr.set_port(port); - addrs - .entry(addr.into()) - .or_insert(DirectAddrType::Stun4LocalPort); - } - } - if let Some(global_v6) = net_report_report.global_v6 { - addrs - .entry(global_v6.into()) - .or_insert(DirectAddrType::Stun); - } - } - - let local_addr_v4 = self.pconn4.local_addr().ok(); - let local_addr_v6 = self.pconn6.as_ref().and_then(|c| c.local_addr().ok()); - - let is_unspecified_v4 = local_addr_v4 - .map(|a| a.ip().is_unspecified()) - .unwrap_or(false); - let is_unspecified_v6 = local_addr_v6 - .map(|a| a.ip().is_unspecified()) - .unwrap_or(false); - - let msock = self.msock.clone(); - - // The following code can be slow, we do not want to block the caller since it would - // block the actor loop. - tokio::spawn( - async move { - // If a socket is bound to the unspecified address, create SocketAddrs for - // each local IP address by pairing it with the port the socket is bound on. - if is_unspecified_v4 || is_unspecified_v6 { - // Depending on the OS and network interfaces attached and their state - // enumerating the local interfaces can take a long time. Especially - // Windows is very slow. - let LocalAddresses { - regular: mut ips, - loopback, - } = tokio::task::spawn_blocking(LocalAddresses::new) - .await - .unwrap(); - if ips.is_empty() && addrs.is_empty() { - // Include loopback addresses only if there are no other interfaces - // or public addresses, this allows testing offline. - ips = loopback; - } - for ip in ips { - let port_if_unspecified = match ip { - IpAddr::V4(_) if is_unspecified_v4 => { - local_addr_v4.map(|addr| addr.port()) - } - IpAddr::V6(_) if is_unspecified_v6 => { - local_addr_v6.map(|addr| addr.port()) - } - _ => None, - }; - if let Some(port) = port_if_unspecified { - let addr = SocketAddr::new(ip, port); - addrs.entry(addr).or_insert(DirectAddrType::Local); - } - } - } - - // If a socket is bound to a specific address, add it. - if !is_unspecified_v4 { - if let Some(addr) = local_addr_v4 { - addrs.entry(addr).or_insert(DirectAddrType::Local); - } - } - if !is_unspecified_v6 { - if let Some(addr) = local_addr_v6 { - addrs.entry(addr).or_insert(DirectAddrType::Local); - } - } - - // Finally create and store store all these direct addresses and send any - // queued call-me-maybe messages. - msock.store_direct_addresses( - addrs - .iter() - .map(|(addr, typ)| DirectAddr { - addr: *addr, - typ: *typ, - }) - .collect(), - ); - msock.send_queued_call_me_maybes(); - } - .instrument(Span::current()), - ); - } - - /// Called when a direct addr update is done, no matter if it was successful or not. - fn finalize_direct_addrs_update(&mut self, why: &'static str) { - let new_why = self.msock.direct_addr_update_state.next_update(); - if !self.msock.is_closed() { - if let Some(new_why) = new_why { - self.msock.direct_addr_update_state.run(new_why); - return; - } - self.periodic_re_stun_timer = new_re_stun_timer(true); - } - - self.msock.direct_addr_update_state.finish_run(); - debug!("direct addr update done ({})", why); - } - - /// Updates `NetInfo.HavePortMap` to true. - #[instrument(level = "debug", skip_all)] - fn set_net_info_have_port_map(&mut self) { - if let Some(ref mut net_info_last) = self.net_info_last { - if net_info_last.have_port_map { - // No change. - return; - } - net_info_last.have_port_map = true; - self.net_info_last = Some(net_info_last.clone()); - } - } - - #[instrument(level = "debug", skip_all)] - async fn call_net_info_callback(&mut self, ni: NetInfo) { - if let Some(ref net_info_last) = self.net_info_last { - if ni.basically_equal(net_info_last) { - return; - } - } - - self.net_info_last = Some(ni); - } - - /// Calls net_report. - /// - /// Note that invoking this is managed by [`DirectAddrUpdateState`] via - /// [`Actor::refresh_direct_addrs`] and this should never be invoked directly. Some day - /// this will be refactored to not allow this easy mistake to be made. - #[instrument(level = "debug", skip_all)] - async fn update_net_info(&mut self, why: &'static str) { - if self.msock.relay_map.is_empty() { - debug!("skipping net_report, empty RelayMap"); - self.msg_sender - .send(ActorMessage::NetReport(Ok(None), why)) - .await - .ok(); - return; - } - - let relay_map = self.msock.relay_map.clone(); - let pconn4 = Some(self.pconn4.as_socket()); - let pconn6 = self.pconn6.as_ref().map(|p| p.as_socket()); - let quic_endpoint = self.msock.quic_endpoint.read().expect("poisoned").clone(); - let quic_addr_disc = match quic_endpoint { - Some(ep) => { - match crate::tls::make_client_config(&self.msock.secret_key, None, vec![], false) { - Ok(client_config) => Some(net_report::QuicAddressDiscovery { - ep, - client_config: quinn::ClientConfig::new(Arc::new(client_config)), - }), - Err(err) => { - warn!("Unable to make a QuicClientConfig to run QUIC address discovery probes in net_report: {err:?}"); - None - } - } - } - None => None, - }; - - debug!("requesting net_report report"); - match self - .net_reporter - .get_report_channel(relay_map, pconn4, pconn6, quic_addr_disc) - .await - { - Ok(rx) => { - let msg_sender = self.msg_sender.clone(); - tokio::task::spawn(async move { - let report = time::timeout(NET_REPORT_TIMEOUT, rx).await; - let report: anyhow::Result<_> = match report { - Ok(Ok(Ok(report))) => Ok(Some(report)), - Ok(Ok(Err(err))) => Err(err), - Ok(Err(_)) => Err(anyhow!("net_report report not received")), - Err(err) => Err(anyhow!("net_report report timeout: {:?}", err)), - }; - msg_sender - .send(ActorMessage::NetReport(report, why)) - .await - .ok(); - // The receiver of the NetReport message will call - // .finalize_direct_addrs_update(). - }); - } - Err(err) => { - warn!("unable to start net_report generation: {:?}", err); - self.finalize_direct_addrs_update(why); - } - } - } - - async fn handle_net_report_report(&mut self, report: Option>) { - if let Some(ref report) = report { - self.msock - .ipv6_reported - .store(report.ipv6, Ordering::Relaxed); - let r = &report; - trace!( - "setting no_v4_send {} -> {}", - self.no_v4_send, - !r.ipv4_can_send - ); - self.no_v4_send = !r.ipv4_can_send; - - let have_port_map = self.port_mapper.watch_external_address().borrow().is_some(); - let mut ni = NetInfo { - relay_latency: Default::default(), - mapping_varies_by_dest_ip: r.mapping_varies_by_dest_ip, - hair_pinning: r.hair_pinning, - portmap_probe: r.portmap_probe.clone(), - have_port_map, - working_ipv6: Some(r.ipv6), - os_has_ipv6: Some(r.os_has_ipv6), - working_udp: Some(r.udp), - working_icmp_v4: r.icmpv4, - working_icmp_v6: r.icmpv6, - preferred_relay: r.preferred_relay.clone(), - }; - for (rid, d) in r.relay_v4_latency.iter() { - ni.relay_latency - .insert(format!("{rid}-v4"), d.as_secs_f64()); - } - for (rid, d) in r.relay_v6_latency.iter() { - ni.relay_latency - .insert(format!("{rid}-v6"), d.as_secs_f64()); - } - - if ni.preferred_relay.is_none() { - // Perhaps UDP is blocked. Pick a deterministic but arbitrary one. - ni.preferred_relay = self.pick_relay_fallback(); - } - - if !self.set_nearest_relay(ni.preferred_relay.clone()) { - ni.preferred_relay = None; - } - - // TODO: set link type - self.call_net_info_callback(ni).await; - } - self.update_direct_addresses(report); - } - - fn set_nearest_relay(&mut self, relay_url: Option) -> bool { - let my_relay = self.msock.my_relay(); - if relay_url == my_relay { - // No change. - return true; - } - let old_relay = self.msock.set_my_relay(relay_url.clone()); - - if let Some(ref relay_url) = relay_url { - inc!(MagicsockMetrics, relay_home_change); - - // On change, notify all currently connected relay servers and - // start connecting to our home relay if we are not already. - info!("home is now relay {}, was {:?}", relay_url, old_relay); - self.msock.publish_my_addr(); - - self.send_relay_actor(RelayActorMessage::SetHome { - url: relay_url.clone(), - }); - } - - true - } - - /// Returns a deterministic relay node to connect to. This is only used if net_report - /// couldn't find the nearest one, for instance, if UDP is blocked and thus STUN - /// latency checks aren't working. - /// - /// If no the [`RelayMap`] is empty, returns `0`. - fn pick_relay_fallback(&self) -> Option { - // TODO: figure out which relay node most of our nodes are using, - // and use that region as our fallback. - // - // If we already had selected something in the past and it has any - // nodes, we want to stay on it. If there are no nodes at all, - // stay on whatever relay we previously picked. If we need to pick - // one and have no node info, pick a node randomly. - // - // We used to do the above for legacy clients, but never updated it for disco. - - let my_relay = self.msock.my_relay(); - if my_relay.is_some() { - return my_relay; - } - - let ids = self.msock.relay_map.urls().collect::>(); - let mut rng = rand::rngs::StdRng::seed_from_u64(0); - ids.choose(&mut rng).map(|c| (*c).clone()) - } - - /// Resets the preferred address for all nodes. - /// This is called when connectivity changes enough that we no longer trust the old routes. - #[instrument(skip_all, fields(me = %self.msock.me))] - fn reset_endpoint_states(&mut self) { - self.msock.node_map.reset_node_states() - } - - /// Tells the relay actor to close stale relay connections. - /// - /// The relay connections who's local endpoints no longer exist after a network change - /// will error out soon enough. Closing them eagerly speeds this up however and allows - /// re-establishing a relay connection faster. - async fn close_stale_relay_connections(&self) { - let ifs = interfaces::State::new().await; - let local_ips = ifs - .interfaces - .values() - .flat_map(|netif| netif.addrs()) - .map(|ipnet| ipnet.addr()) - .collect(); - self.send_relay_actor(RelayActorMessage::MaybeCloseRelaysOnRebind(local_ips)); - } - - fn send_relay_actor(&self, msg: RelayActorMessage) { - match self.relay_actor_sender.try_send(msg) { - Ok(_) => {} - Err(mpsc::error::TrySendError::Closed(_)) => { - warn!("unable to send to relay actor, already closed"); - } - Err(mpsc::error::TrySendError::Full(_)) => { - warn!("dropping message for relay actor, channel is full"); - } - } - } - - fn handle_relay_disco_message( - &mut self, - msg: &[u8], - url: &RelayUrl, - relay_node_src: PublicKey, - ) -> bool { - match disco::source_and_box(msg) { - Some((source, sealed_box)) => { - if relay_node_src != source { - // TODO: return here? - warn!("Received relay disco message from connection for {}, but with message from {}", relay_node_src.fmt_short(), source.fmt_short()); - } - self.msock.handle_disco_message( - source, - sealed_box, - DiscoMessageSource::Relay { - url: url.clone(), - key: relay_node_src, - }, - ); - true - } - None => false, - } - } -} - -fn new_re_stun_timer(initial_delay: bool) -> time::Interval { - // Pick a random duration between 20 and 26 seconds (just under 30s, - // a common UDP NAT timeout on Linux,etc) - let mut rng = rand::thread_rng(); - let d: Duration = rng.gen_range(Duration::from_secs(20)..=Duration::from_secs(26)); - if initial_delay { - debug!("scheduling periodic_stun to run in {}s", d.as_secs()); - time::interval_at(time::Instant::now() + d, d) - } else { - debug!( - "scheduling periodic_stun to run immediately and in {}s", - d.as_secs() - ); - time::interval(d) - } -} - -/// Initial connection setup. -fn bind( - addr_v4: Option, - addr_v6: Option, -) -> Result<(UdpConn, Option)> { - let addr_v4 = addr_v4.unwrap_or_else(|| SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)); - let pconn4 = UdpConn::bind(SocketAddr::V4(addr_v4)).context("bind IPv4 failed")?; - - let ip4_port = pconn4.local_addr()?.port(); - let ip6_port = ip4_port.checked_add(1).unwrap_or(ip4_port - 1); - let addr_v6 = - addr_v6.unwrap_or_else(|| SocketAddrV6::new(Ipv6Addr::UNSPECIFIED, ip6_port, 0, 0)); - let pconn6 = match UdpConn::bind(SocketAddr::V6(addr_v6)) { - Ok(conn) => Some(conn), - Err(err) => { - info!("bind ignoring IPv6 bind failure: {:?}", err); - None - } - }; - - Ok((pconn4, pconn6)) -} - -/// The discovered direct addresses of this [`MagicSock`]. -/// -/// These are all the [`DirectAddr`]s that this [`MagicSock`] is aware of for itself. -/// They include all locally bound ones as well as those discovered by other mechanisms like -/// STUN. -#[derive(derive_more::Debug, Default, Clone)] -struct DiscoveredDirectAddrs { - /// The last set of discovered direct addresses. - addrs: Watchable>, - - /// The last time the direct addresses were updated, even if there was no change. - /// - /// This is only ever None at startup. - updated_at: Arc>>, -} - -impl DiscoveredDirectAddrs { - /// Updates the direct addresses, returns `true` if they changed, `false` if not. - fn update(&self, addrs: BTreeSet) -> bool { - *self.updated_at.write().unwrap() = Some(Instant::now()); - let updated = self.addrs.update(addrs).is_ok(); - if updated { - event!( - target: "events.net.direct_addrs", - Level::DEBUG, - addrs = ?self.addrs.get(), - ); - } - updated - } - - fn sockaddrs(&self) -> BTreeSet { - self.addrs.read().iter().map(|da| da.addr).collect() - } - - /// Whether the direct addr information is considered "fresh". - /// - /// If not fresh you should probably update the direct addresses before using this info. - /// - /// Returns `Ok(())` if fresh enough and `Err(elapsed)` if not fresh enough. - /// `elapsed` is the time elapsed since the direct addresses were last updated. - /// - /// If there is no direct address information `Err(Duration::ZERO)` is returned. - fn fresh_enough(&self) -> Result<(), Duration> { - match *self.updated_at.read().expect("poisoned") { - None => Err(Duration::ZERO), - Some(time) => { - let elapsed = time.elapsed(); - if elapsed <= ENDPOINTS_FRESH_ENOUGH_DURATION { - Ok(()) - } else { - Err(elapsed) - } - } - } - } - - fn to_call_me_maybe_message(&self) -> disco::CallMeMaybe { - let my_numbers = self.addrs.read().iter().map(|da| da.addr).collect(); - disco::CallMeMaybe { my_numbers } - } - - fn updates_stream(&self) -> DirectAddrsStream { - DirectAddrsStream { - initial: Some(self.addrs.get()), - inner: self.addrs.watch().into_stream(), - } - } -} - -/// Stream returning local endpoints as they change. -#[derive(Debug)] -pub struct DirectAddrsStream { - initial: Option>, - inner: watchable::WatcherStream>, -} - -impl Stream for DirectAddrsStream { - type Item = BTreeSet; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = &mut *self; - if let Some(addrs) = this.initial.take() { - if !addrs.is_empty() { - return Poll::Ready(Some(addrs)); - } - } - loop { - match Pin::new(&mut this.inner).poll_next(cx) { - Poll::Pending => break Poll::Pending, - Poll::Ready(Some(addrs)) => { - if addrs.is_empty() { - // When we start up we might initially have empty direct addrs as - // the magic socket has not yet figured this out. Later on this set - // should never be empty. However even if it was the magicsock - // would be in a state not very usable so skipping those events is - // probably fine. - // To make sure we install the right waker we loop rather than - // returning Poll::Pending immediately here. - continue; - } else { - break Poll::Ready(Some(addrs)); - } - } - Poll::Ready(None) => break Poll::Ready(None), - } - } - } -} - -/// Split a transmit containing a GSO payload into individual packets. -/// -/// This allocates the data. -/// -/// If the transmit has a segment size it contains multiple GSO packets. It will be split -/// into multiple packets according to that segment size. If it does not have a segment -/// size, the contents will be sent as a single packet. -// TODO: If quinn stayed on bytes this would probably be much cheaper, probably. Need to -// figure out where they allocate the Vec. -fn split_packets(transmit: &quinn_udp::Transmit) -> RelayContents { - let mut res = SmallVec::with_capacity(1); - let contents = transmit.contents; - if let Some(segment_size) = transmit.segment_size { - for chunk in contents.chunks(segment_size) { - res.push(Bytes::from(chunk.to_vec())); - } - } else { - res.push(Bytes::from(contents.to_vec())); - } - res -} - -/// Splits a packet into its component items. -#[derive(Debug)] -struct PacketSplitIter { - bytes: Bytes, -} - -impl PacketSplitIter { - /// Create a new PacketSplitIter from a packet. - /// - /// Returns an error if the packet is too big. - fn new(bytes: Bytes) -> Self { - Self { bytes } - } - - fn fail(&mut self) -> Option> { - self.bytes.clear(); - Some(Err(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "", - ))) - } -} - -impl Iterator for PacketSplitIter { - type Item = std::io::Result; - - fn next(&mut self) -> Option { - use bytes::Buf; - if self.bytes.has_remaining() { - if self.bytes.remaining() < 2 { - return self.fail(); - } - let len = self.bytes.get_u16_le() as usize; - if self.bytes.remaining() < len { - return self.fail(); - } - let item = self.bytes.split_to(len); - Some(Ok(item)) - } else { - None - } - } -} - -/// The fake address used by the QUIC layer to address a node. -/// -/// You can consider this as nothing more than a lookup key for a node the [`MagicSock`] knows -/// about. -/// -/// [`MagicSock`] can reach a node by several real socket addresses, or maybe even via the relay -/// node. The QUIC layer however needs to address a node by a stable [`SocketAddr`] so -/// that normal socket APIs can function. Thus when a new node is introduced to a [`MagicSock`] -/// it is given a new fake address. This is the type of that address. -/// -/// It is but a newtype. And in our QUIC-facing socket APIs like [`AsyncUdpSocket`] it -/// comes in as the inner [`SocketAddr`], in those interfaces we have to be careful to do -/// the conversion to this type. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub(crate) struct QuicMappedAddr(pub(crate) SocketAddr); - -/// Counter to always generate unique addresses for [`QuicMappedAddr`]. -static ADDR_COUNTER: AtomicU64 = AtomicU64::new(1); - -impl QuicMappedAddr { - /// The Prefix/L of our Unique Local Addresses. - const ADDR_PREFIXL: u8 = 0xfd; - /// The Global ID used in our Unique Local Addresses. - const ADDR_GLOBAL_ID: [u8; 5] = [21, 7, 10, 81, 11]; - /// The Subnet ID used in our Unique Local Addresses. - const ADDR_SUBNET: [u8; 2] = [0; 2]; - - /// Generates a globally unique fake UDP address. - /// - /// This generates and IPv6 Unique Local Address according to RFC 4193. - pub(crate) fn generate() -> Self { - let mut addr = [0u8; 16]; - addr[0] = Self::ADDR_PREFIXL; - addr[1..6].copy_from_slice(&Self::ADDR_GLOBAL_ID); - addr[6..8].copy_from_slice(&Self::ADDR_SUBNET); - - let counter = ADDR_COUNTER.fetch_add(1, Ordering::Relaxed); - addr[8..16].copy_from_slice(&counter.to_be_bytes()); - - Self(SocketAddr::new(IpAddr::V6(Ipv6Addr::from(addr)), 12345)) - } -} - -impl std::fmt::Display for QuicMappedAddr { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "QuicMappedAddr({})", self.0) - } -} -fn disco_message_sent(msg: &disco::Message) { - match msg { - disco::Message::Ping(_) => { - inc!(MagicsockMetrics, sent_disco_ping); - } - disco::Message::Pong(_) => { - inc!(MagicsockMetrics, sent_disco_pong); - } - disco::Message::CallMeMaybe(_) => { - inc!(MagicsockMetrics, sent_disco_call_me_maybe); - } - } -} - -/// A *direct address* on which an iroh-node might be contactable. -/// -/// Direct addresses are UDP socket addresses on which an iroh-net node could potentially be -/// contacted. These can come from various sources depending on the network topology of the -/// iroh-net node, see [`DirectAddrType`] for the several kinds of sources. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct DirectAddr { - /// The address. - pub addr: SocketAddr, - /// The origin of this direct address. - pub typ: DirectAddrType, -} - -/// The type of direct address. -/// -/// These are the various sources or origins from which an iroh-net node might have found a -/// possible [`DirectAddr`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum DirectAddrType { - /// Not yet determined.. - Unknown, - /// A locally bound socket address. - Local, - /// Public internet address discovered via STUN. - /// - /// When possible an iroh-net node will perform STUN to discover which is the address - /// from which it sends data on the public internet. This can be different from locally - /// bound addresses when the node is on a local network which performs NAT or similar. - Stun, - /// An address assigned by the router using port mapping. - /// - /// When possible an iroh-net node will request a port mapping from the local router to - /// get a publicly routable direct address. - Portmapped, - /// Hard NAT: STUN'ed IPv4 address + local fixed port. - /// - /// It is possible to configure iroh-net to bound to a specific port and independently - /// configure the router to forward this port to the iroh-net node. This indicates a - /// situation like this, which still uses STUN to discover the public address. - Stun4LocalPort, -} - -impl Display for DirectAddrType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - DirectAddrType::Unknown => write!(f, "?"), - DirectAddrType::Local => write!(f, "local"), - DirectAddrType::Stun => write!(f, "stun"), - DirectAddrType::Portmapped => write!(f, "portmap"), - DirectAddrType::Stun4LocalPort => write!(f, "stun4localport"), - } - } -} - -/// Contains information about the host's network state. -#[derive(Debug, Clone, PartialEq)] -struct NetInfo { - /// Says whether the host's NAT mappings vary based on the destination IP. - mapping_varies_by_dest_ip: Option, - - /// If their router does hairpinning. It reports true even if there's no NAT involved. - hair_pinning: Option, - - /// Whether the host has IPv6 internet connectivity. - working_ipv6: Option, - - /// Whether the OS supports IPv6 at all, regardless of whether IPv6 internet connectivity is available. - os_has_ipv6: Option, - - /// Whether the host has UDP internet connectivity. - working_udp: Option, - - /// Whether ICMPv4 works, `None` means not checked. - working_icmp_v4: Option, - - /// Whether ICMPv6 works, `None` means not checked. - working_icmp_v6: Option, - - /// Whether we have an existing portmap open (UPnP, PMP, or PCP). - have_port_map: bool, - - /// Probe indicating the presence of port mapping protocols on the LAN. - portmap_probe: Option, - - /// This node's preferred relay server for incoming traffic. - /// - /// The node might be be temporarily connected to multiple relay servers (to send to - /// other nodes) but this is the relay on which you can always contact this node. Also - /// known as home relay. - preferred_relay: Option, - - /// The fastest recent time to reach various relay STUN servers, in seconds. - /// - /// This should only be updated rarely, or when there's a - /// material change, as any change here also gets uploaded to the control plane. - relay_latency: BTreeMap, -} - -impl NetInfo { - /// Checks if this is probably still the same network as *other*. - /// - /// This tries to compare the network situation, without taking into account things - /// expected to change a little like e.g. latency to the relay server. - fn basically_equal(&self, other: &Self) -> bool { - let eq_icmp_v4 = match (self.working_icmp_v4, other.working_icmp_v4) { - (Some(slf), Some(other)) => slf == other, - _ => true, // ignore for comparison if only one report had this info - }; - let eq_icmp_v6 = match (self.working_icmp_v6, other.working_icmp_v6) { - (Some(slf), Some(other)) => slf == other, - _ => true, // ignore for comparison if only one report had this info - }; - self.mapping_varies_by_dest_ip == other.mapping_varies_by_dest_ip - && self.hair_pinning == other.hair_pinning - && self.working_ipv6 == other.working_ipv6 - && self.os_has_ipv6 == other.os_has_ipv6 - && self.working_udp == other.working_udp - && eq_icmp_v4 - && eq_icmp_v6 - && self.have_port_map == other.have_port_map - && self.portmap_probe == other.portmap_probe - && self.preferred_relay == other.preferred_relay - } -} - -#[cfg(test)] -mod tests { - use anyhow::Context; - use iroh_test::CallOnDrop; - use rand::RngCore; - use tokio_util::task::AbortOnDropHandle; - - use super::*; - use crate::{defaults::staging::EU_RELAY_HOSTNAME, tls, Endpoint, RelayMode}; - - const ALPN: &[u8] = b"n0/test/1"; - - impl MagicSock { - #[track_caller] - pub fn add_test_addr(&self, node_addr: NodeAddr) { - self.add_node_addr( - node_addr, - Source::NamedApp { - name: "test".into(), - }, - ) - .unwrap() - } - } - - /// Magicsock plus wrappers for sending packets - #[derive(Clone)] - struct MagicStack { - secret_key: SecretKey, - endpoint: Endpoint, - } - - impl MagicStack { - async fn new(relay_mode: RelayMode) -> Result { - let secret_key = SecretKey::generate(); - - let mut transport_config = quinn::TransportConfig::default(); - transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); - - let endpoint = Endpoint::builder() - .secret_key(secret_key.clone()) - .transport_config(transport_config) - .relay_mode(relay_mode) - .alpns(vec![ALPN.to_vec()]) - .bind() - .await?; - - Ok(Self { - secret_key, - endpoint, - }) - } - - fn tracked_endpoints(&self) -> Vec { - self.endpoint - .magic_sock() - .list_remote_infos() - .into_iter() - .map(|ep| ep.node_id) - .collect() - } - - fn public(&self) -> PublicKey { - self.secret_key.public() - } - } - - /// Monitors endpoint changes and plumbs things together. - /// - /// This is a way of connecting endpoints without a relay server. Whenever the local - /// endpoints of a magic endpoint change this address is added to the other magic - /// sockets. This function will await until the endpoints are connected the first time - /// before returning. - /// - /// When the returned drop guard is dropped, the tasks doing this updating are stopped. - #[instrument(skip_all)] - async fn mesh_stacks(stacks: Vec) -> Result { - /// Registers endpoint addresses of a node to all other nodes. - fn update_direct_addrs( - stacks: &[MagicStack], - my_idx: usize, - new_addrs: BTreeSet, - ) { - let me = &stacks[my_idx]; - for (i, m) in stacks.iter().enumerate() { - if i == my_idx { - continue; - } - - let addr = NodeAddr { - node_id: me.public(), - info: crate::AddrInfo { - relay_url: None, - direct_addresses: new_addrs.iter().map(|ep| ep.addr).collect(), - }, - }; - m.endpoint.magic_sock().add_test_addr(addr); - } - } - - // For each node, start a task which monitors its local endpoints and registers them - // with the other nodes as local endpoints become known. - let mut tasks = JoinSet::new(); - for (my_idx, m) in stacks.iter().enumerate() { - let m = m.clone(); - let stacks = stacks.clone(); - tasks.spawn(async move { - let me = m.endpoint.node_id().fmt_short(); - let mut stream = m.endpoint.direct_addresses(); - while let Some(new_eps) = stream.next().await { - info!(%me, "conn{} endpoints update: {:?}", my_idx + 1, new_eps); - update_direct_addrs(&stacks, my_idx, new_eps); - } - }); - } - let guard = CallOnDrop::new(move || { - tasks.abort_all(); - }); - - // Wait for all nodes to be registered with each other. - time::timeout(Duration::from_secs(10), async move { - let all_node_ids: Vec<_> = stacks.iter().map(|ms| ms.endpoint.node_id()).collect(); - loop { - let mut ready = Vec::with_capacity(stacks.len()); - for ms in stacks.iter() { - let endpoints = ms.tracked_endpoints(); - let my_node_id = ms.endpoint.node_id(); - let all_nodes_meshed = all_node_ids - .iter() - .filter(|node_id| **node_id != my_node_id) - .all(|node_id| endpoints.contains(node_id)); - ready.push(all_nodes_meshed); - } - if ready.iter().all(|meshed| *meshed) { - break; - } - tokio::time::sleep(Duration::from_millis(200)).await; - } - }) - .await - .context("failed to connect nodes")?; - info!("all nodes meshed"); - Ok(guard) - } - - #[instrument(skip_all, fields(me = %ep.endpoint.node_id().fmt_short()))] - async fn echo_receiver(ep: MagicStack) -> Result<()> { - info!("accepting conn"); - let conn = ep.endpoint.accept().await.expect("no conn"); - - info!("connecting"); - let conn = conn.await.context("[receiver] connecting")?; - info!("accepting bi"); - let (mut send_bi, mut recv_bi) = - conn.accept_bi().await.context("[receiver] accepting bi")?; - - info!("reading"); - let val = recv_bi - .read_to_end(usize::MAX) - .await - .context("[receiver] reading to end")?; - - info!("replying"); - for chunk in val.chunks(12) { - send_bi - .write_all(chunk) - .await - .context("[receiver] sending chunk")?; - } - - info!("finishing"); - send_bi.finish().context("[receiver] finishing")?; - send_bi.stopped().await.context("[receiver] stopped")?; - - let stats = conn.stats(); - info!("stats: {:#?}", stats); - // TODO: ensure panics in this function are reported ok - assert!( - stats.path.lost_packets < 10, - "[receiver] should not loose many packets", - ); - - info!("close"); - conn.close(0u32.into(), b"done"); - info!("wait idle"); - ep.endpoint.endpoint().wait_idle().await; - - Ok(()) - } - - #[instrument(skip_all, fields(me = %ep.endpoint.node_id().fmt_short()))] - async fn echo_sender(ep: MagicStack, dest_id: PublicKey, msg: &[u8]) -> Result<()> { - info!("connecting to {}", dest_id.fmt_short()); - let dest = NodeAddr::new(dest_id); - let conn = ep - .endpoint - .connect(dest, ALPN) - .await - .context("[sender] connect")?; - - info!("opening bi"); - let (mut send_bi, mut recv_bi) = conn.open_bi().await.context("[sender] open bi")?; - - info!("writing message"); - send_bi.write_all(msg).await.context("[sender] write all")?; - - info!("finishing"); - send_bi.finish().context("[sender] finish")?; - send_bi.stopped().await.context("[sender] stopped")?; - - info!("reading_to_end"); - let val = recv_bi.read_to_end(usize::MAX).await.context("[sender]")?; - assert_eq!( - val, - msg, - "[sender] expected {}, got {}", - hex::encode(msg), - hex::encode(&val) - ); - - let stats = conn.stats(); - info!("stats: {:#?}", stats); - assert!( - stats.path.lost_packets < 10, - "[sender] should not loose many packets", - ); - - info!("close"); - conn.close(0u32.into(), b"done"); - info!("wait idle"); - ep.endpoint.endpoint().wait_idle().await; - Ok(()) - } - - /// Runs a roundtrip between the [`echo_sender`] and [`echo_receiver`]. - async fn run_roundtrip(sender: MagicStack, receiver: MagicStack, payload: &[u8]) { - let send_node_id = sender.endpoint.node_id(); - let recv_node_id = receiver.endpoint.node_id(); - info!("\nroundtrip: {send_node_id:#} -> {recv_node_id:#}"); - - let receiver_task = tokio::spawn(echo_receiver(receiver)); - let sender_res = echo_sender(sender, recv_node_id, payload).await; - let sender_is_err = match sender_res { - Ok(()) => false, - Err(err) => { - eprintln!("[sender] Error:\n{err:#?}"); - true - } - }; - let receiver_is_err = match receiver_task.await { - Ok(Ok(())) => false, - Ok(Err(err)) => { - eprintln!("[receiver] Error:\n{err:#?}"); - true - } - Err(joinerr) => { - if joinerr.is_panic() { - std::panic::resume_unwind(joinerr.into_panic()); - } else { - eprintln!("[receiver] Error:\n{joinerr:#?}"); - } - true - } - }; - if sender_is_err || receiver_is_err { - panic!("Sender or receiver errored"); - } - } - - #[tokio::test(flavor = "multi_thread")] - async fn test_two_devices_roundtrip_quinn_magic() -> Result<()> { - iroh_test::logging::setup_multithreaded(); - - let m1 = MagicStack::new(RelayMode::Disabled).await?; - let m2 = MagicStack::new(RelayMode::Disabled).await?; - - let _guard = mesh_stacks(vec![m1.clone(), m2.clone()]).await?; - - for i in 0..5 { - info!("\n-- round {i}"); - run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await; - run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await; - - info!("\n-- larger data"); - let mut data = vec![0u8; 10 * 1024]; - rand::thread_rng().fill_bytes(&mut data); - run_roundtrip(m1.clone(), m2.clone(), &data).await; - run_roundtrip(m2.clone(), m1.clone(), &data).await; - } - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread")] - async fn test_two_devices_roundtrip_network_change() -> Result<()> { - time::timeout( - Duration::from_secs(90), - test_two_devices_roundtrip_network_change_impl(), - ) - .await? - } - - /// Same structure as `test_two_devices_roundtrip_quinn_magic`, but interrupts regularly - /// with (simulated) network changes. - async fn test_two_devices_roundtrip_network_change_impl() -> Result<()> { - iroh_test::logging::setup_multithreaded(); - - let m1 = MagicStack::new(RelayMode::Disabled).await?; - let m2 = MagicStack::new(RelayMode::Disabled).await?; - - let _guard = mesh_stacks(vec![m1.clone(), m2.clone()]).await?; - - let offset = || { - let delay = rand::thread_rng().gen_range(10..=500); - Duration::from_millis(delay) - }; - let rounds = 5; - - // Regular network changes to m1 only. - let m1_network_change_guard = { - let m1 = m1.clone(); - let task = tokio::spawn(async move { - loop { - println!("[m1] network change"); - m1.endpoint.magic_sock().force_network_change(true).await; - time::sleep(offset()).await; - } - }); - CallOnDrop::new(move || { - task.abort(); - }) - }; - - for i in 0..rounds { - println!("-- [m1 changes] round {}", i + 1); - run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await; - run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await; - - println!("-- [m1 changes] larger data"); - let mut data = vec![0u8; 10 * 1024]; - rand::thread_rng().fill_bytes(&mut data); - run_roundtrip(m1.clone(), m2.clone(), &data).await; - run_roundtrip(m2.clone(), m1.clone(), &data).await; - } - - std::mem::drop(m1_network_change_guard); - - // Regular network changes to m2 only. - let m2_network_change_guard = { - let m2 = m2.clone(); - let task = tokio::spawn(async move { - loop { - println!("[m2] network change"); - m2.endpoint.magic_sock().force_network_change(true).await; - time::sleep(offset()).await; - } - }); - CallOnDrop::new(move || { - task.abort(); - }) - }; - - for i in 0..rounds { - println!("-- [m2 changes] round {}", i + 1); - run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await; - run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await; - - println!("-- [m2 changes] larger data"); - let mut data = vec![0u8; 10 * 1024]; - rand::thread_rng().fill_bytes(&mut data); - run_roundtrip(m1.clone(), m2.clone(), &data).await; - run_roundtrip(m2.clone(), m1.clone(), &data).await; - } - - std::mem::drop(m2_network_change_guard); - - // Regular network changes to both m1 and m2 only. - let m1_m2_network_change_guard = { - let m1 = m1.clone(); - let m2 = m2.clone(); - let task = tokio::spawn(async move { - println!("-- [m1] network change"); - m1.endpoint.magic_sock().force_network_change(true).await; - println!("-- [m2] network change"); - m2.endpoint.magic_sock().force_network_change(true).await; - time::sleep(offset()).await; - }); - CallOnDrop::new(move || { - task.abort(); - }) - }; - - for i in 0..rounds { - println!("-- [m1 & m2 changes] round {}", i + 1); - run_roundtrip(m1.clone(), m2.clone(), b"hello m1").await; - run_roundtrip(m2.clone(), m1.clone(), b"hello m2").await; - - println!("-- [m1 & m2 changes] larger data"); - let mut data = vec![0u8; 10 * 1024]; - rand::thread_rng().fill_bytes(&mut data); - run_roundtrip(m1.clone(), m2.clone(), &data).await; - run_roundtrip(m2.clone(), m1.clone(), &data).await; - } - - std::mem::drop(m1_m2_network_change_guard); - Ok(()) - } - - #[tokio::test(flavor = "multi_thread")] - async fn test_two_devices_setup_teardown() -> Result<()> { - iroh_test::logging::setup_multithreaded(); - for i in 0..10 { - println!("-- round {i}"); - println!("setting up magic stack"); - let m1 = MagicStack::new(RelayMode::Disabled).await?; - let m2 = MagicStack::new(RelayMode::Disabled).await?; - - let _guard = mesh_stacks(vec![m1.clone(), m2.clone()]).await?; - - println!("closing endpoints"); - let msock1 = m1.endpoint.magic_sock(); - let msock2 = m2.endpoint.magic_sock(); - m1.endpoint.close(0u32.into(), b"done").await?; - m2.endpoint.close(0u32.into(), b"done").await?; - - assert!(msock1.msock.is_closed()); - assert!(msock2.msock.is_closed()); - } - Ok(()) - } - - #[tokio::test] - async fn test_two_devices_roundtrip_quinn_raw() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - let make_conn = |addr: SocketAddr| -> anyhow::Result { - let key = SecretKey::generate(); - let conn = std::net::UdpSocket::bind(addr)?; - - let quic_server_config = tls::make_server_config(&key, vec![ALPN.to_vec()], false)?; - let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_server_config)); - let mut transport_config = quinn::TransportConfig::default(); - transport_config.keep_alive_interval(Some(Duration::from_secs(5))); - transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); - server_config.transport_config(Arc::new(transport_config)); - let mut quic_ep = quinn::Endpoint::new( - quinn::EndpointConfig::default(), - Some(server_config), - conn, - Arc::new(quinn::TokioRuntime), - )?; - - let quic_client_config = - tls::make_client_config(&key, None, vec![ALPN.to_vec()], false)?; - let mut client_config = quinn::ClientConfig::new(Arc::new(quic_client_config)); - let mut transport_config = quinn::TransportConfig::default(); - transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); - client_config.transport_config(Arc::new(transport_config)); - quic_ep.set_default_client_config(client_config); - - Ok(quic_ep) - }; - - let m1 = make_conn("127.0.0.1:0".parse().unwrap())?; - let m2 = make_conn("127.0.0.1:0".parse().unwrap())?; - - // msg from a -> b - macro_rules! roundtrip { - ($a:expr, $b:expr, $msg:expr) => { - let a = $a.clone(); - let b = $b.clone(); - let a_name = stringify!($a); - let b_name = stringify!($b); - println!("{} -> {} ({} bytes)", a_name, b_name, $msg.len()); - - let a_addr = a.local_addr()?; - let b_addr = b.local_addr()?; - - println!("{}: {}, {}: {}", a_name, a_addr, b_name, b_addr); - - let b_task = tokio::task::spawn(async move { - println!("[{b_name}] accepting conn"); - let conn = b.accept().await.expect("no conn"); - println!("[{}] connecting", b_name); - let conn = conn - .await - .with_context(|| format!("[{b_name}] connecting"))?; - println!("[{}] accepting bi", b_name); - let (mut send_bi, mut recv_bi) = conn - .accept_bi() - .await - .with_context(|| format!("[{b_name}] accepting bi"))?; - - println!("[{b_name}] reading"); - let val = recv_bi - .read_to_end(usize::MAX) - .await - .with_context(|| format!("[{b_name}] reading to end"))?; - println!("[{b_name}] finishing"); - send_bi - .finish() - .with_context(|| format!("[{b_name}] finishing"))?; - send_bi - .stopped() - .await - .with_context(|| format!("[b_name] stopped"))?; - - println!("[{b_name}] close"); - conn.close(0u32.into(), b"done"); - println!("[{b_name}] closed"); - - Ok::<_, anyhow::Error>(val) - }); - - println!("[{a_name}] connecting to {b_addr}"); - let conn = a - .connect(b_addr, "localhost")? - .await - .with_context(|| format!("[{a_name}] connect"))?; - - println!("[{a_name}] opening bi"); - let (mut send_bi, mut recv_bi) = conn - .open_bi() - .await - .with_context(|| format!("[{a_name}] open bi"))?; - println!("[{a_name}] writing message"); - send_bi - .write_all(&$msg[..]) - .await - .with_context(|| format!("[{a_name}] write all"))?; - - println!("[{a_name}] finishing"); - send_bi - .finish() - .with_context(|| format!("[{a_name}] finish"))?; - send_bi - .stopped() - .await - .with_context(|| format!("[{a_name}] stopped"))?; - - println!("[{a_name}] reading_to_end"); - let _ = recv_bi - .read_to_end(usize::MAX) - .await - .with_context(|| format!("[{a_name}] reading_to_end"))?; - println!("[{a_name}] close"); - conn.close(0u32.into(), b"done"); - println!("[{a_name}] wait idle"); - a.wait_idle().await; - - drop(send_bi); - - // make sure the right values arrived - println!("[{a_name}] waiting for channel"); - let val = b_task.await??; - anyhow::ensure!( - val == $msg, - "expected {}, got {}", - hex::encode($msg), - hex::encode(val) - ); - }; - } - - for i in 0..10 { - println!("-- round {}", i + 1); - roundtrip!(m1, m2, b"hello m1"); - roundtrip!(m2, m1, b"hello m2"); - - println!("-- larger data"); - - let mut data = vec![0u8; 10 * 1024]; - rand::thread_rng().fill_bytes(&mut data); - roundtrip!(m1, m2, data); - roundtrip!(m2, m1, data); - } - - Ok(()) - } - - #[tokio::test] - async fn test_two_devices_roundtrip_quinn_rebinding_conn() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - fn make_conn(addr: SocketAddr) -> anyhow::Result { - let key = SecretKey::generate(); - let conn = UdpConn::bind(addr)?; - - let quic_server_config = tls::make_server_config(&key, vec![ALPN.to_vec()], false)?; - let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(quic_server_config)); - let mut transport_config = quinn::TransportConfig::default(); - transport_config.keep_alive_interval(Some(Duration::from_secs(5))); - transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); - server_config.transport_config(Arc::new(transport_config)); - let mut quic_ep = quinn::Endpoint::new_with_abstract_socket( - quinn::EndpointConfig::default(), - Some(server_config), - Arc::new(conn), - Arc::new(quinn::TokioRuntime), - )?; - - let quic_client_config = - tls::make_client_config(&key, None, vec![ALPN.to_vec()], false)?; - let mut client_config = quinn::ClientConfig::new(Arc::new(quic_client_config)); - let mut transport_config = quinn::TransportConfig::default(); - transport_config.max_idle_timeout(Some(Duration::from_secs(10).try_into().unwrap())); - client_config.transport_config(Arc::new(transport_config)); - quic_ep.set_default_client_config(client_config); - - Ok(quic_ep) - } - - let m1 = make_conn("127.0.0.1:7770".parse().unwrap())?; - let m2 = make_conn("127.0.0.1:7771".parse().unwrap())?; - - // msg from a -> b - macro_rules! roundtrip { - ($a:expr, $b:expr, $msg:expr) => { - let a = $a.clone(); - let b = $b.clone(); - let a_name = stringify!($a); - let b_name = stringify!($b); - println!("{} -> {} ({} bytes)", a_name, b_name, $msg.len()); - - let a_addr: SocketAddr = format!("127.0.0.1:{}", a.local_addr()?.port()) - .parse() - .unwrap(); - let b_addr: SocketAddr = format!("127.0.0.1:{}", b.local_addr()?.port()) - .parse() - .unwrap(); - - println!("{}: {}, {}: {}", a_name, a_addr, b_name, b_addr); - - let b_task = tokio::task::spawn(async move { - println!("[{}] accepting conn", b_name); - let conn = b.accept().await.expect("no conn"); - println!("[{}] connecting", b_name); - let conn = conn - .await - .with_context(|| format!("[{}] connecting", b_name))?; - println!("[{}] accepting bi", b_name); - let (mut send_bi, mut recv_bi) = conn - .accept_bi() - .await - .with_context(|| format!("[{}] accepting bi", b_name))?; - - println!("[{}] reading", b_name); - let val = recv_bi - .read_to_end(usize::MAX) - .await - .with_context(|| format!("[{}] reading to end", b_name))?; - println!("[{}] finishing", b_name); - send_bi - .finish() - .with_context(|| format!("[{}] finishing", b_name))?; - send_bi - .stopped() - .await - .with_context(|| format!("[{b_name}] stopped"))?; - - println!("[{}] close", b_name); - conn.close(0u32.into(), b"done"); - println!("[{}] closed", b_name); - - Ok::<_, anyhow::Error>(val) - }); - - println!("[{}] connecting to {}", a_name, b_addr); - let conn = a - .connect(b_addr, "localhost")? - .await - .with_context(|| format!("[{}] connect", a_name))?; - - println!("[{}] opening bi", a_name); - let (mut send_bi, mut recv_bi) = conn - .open_bi() - .await - .with_context(|| format!("[{}] open bi", a_name))?; - println!("[{}] writing message", a_name); - send_bi - .write_all(&$msg[..]) - .await - .with_context(|| format!("[{}] write all", a_name))?; - - println!("[{}] finishing", a_name); - send_bi - .finish() - .with_context(|| format!("[{}] finish", a_name))?; - send_bi - .stopped() - .await - .with_context(|| format!("[{a_name}] stopped"))?; - - println!("[{}] reading_to_end", a_name); - let _ = recv_bi - .read_to_end(usize::MAX) - .await - .with_context(|| format!("[{}]", a_name))?; - println!("[{}] close", a_name); - conn.close(0u32.into(), b"done"); - println!("[{}] wait idle", a_name); - a.wait_idle().await; - - drop(send_bi); - - // make sure the right values arrived - println!("[{}] waiting for channel", a_name); - let val = b_task.await??; - anyhow::ensure!( - val == $msg, - "expected {}, got {}", - hex::encode($msg), - hex::encode(val) - ); - }; - } - - for i in 0..10 { - println!("-- round {}", i + 1); - roundtrip!(m1, m2, b"hello m1"); - roundtrip!(m2, m1, b"hello m2"); - - println!("-- larger data"); - - let mut data = vec![0u8; 10 * 1024]; - rand::thread_rng().fill_bytes(&mut data); - roundtrip!(m1, m2, data); - roundtrip!(m2, m1, data); - } - - Ok(()) - } - - #[test] - fn test_split_packets() { - fn mk_transmit(contents: &[u8], segment_size: Option) -> quinn_udp::Transmit<'_> { - let destination = "127.0.0.1:0".parse().unwrap(); - quinn_udp::Transmit { - destination, - ecn: None, - contents, - segment_size, - src_ip: None, - } - } - fn mk_expected(parts: impl IntoIterator) -> RelayContents { - parts - .into_iter() - .map(|p| p.as_bytes().to_vec().into()) - .collect() - } - // no split - assert_eq!( - split_packets(&mk_transmit(b"hello", None)), - mk_expected(["hello"]) - ); - // split without rest - assert_eq!( - split_packets(&mk_transmit(b"helloworld", Some(5))), - mk_expected(["hello", "world"]) - ); - // split with rest and second transmit - assert_eq!( - split_packets(&mk_transmit(b"hello world", Some(5))), - mk_expected(["hello", " worl", "d"]) // spellchecker:disable-line - ); - // split that results in 1 packet - assert_eq!( - split_packets(&mk_transmit(b"hello world", Some(1000))), - mk_expected(["hello world"]) - ); - } - - #[tokio::test] - async fn test_local_endpoints() { - let _guard = iroh_test::logging::setup(); - let ms = Handle::new(Default::default()).await.unwrap(); - - // See if we can get endpoints. - let eps0 = ms.direct_addresses().next().await.unwrap(); - println!("{eps0:?}"); - assert!(!eps0.is_empty()); - - // Getting the endpoints again immediately should give the same results. - let eps1 = ms.direct_addresses().next().await.unwrap(); - println!("{eps1:?}"); - assert_eq!(eps0, eps1); - } - - #[tokio::test] - async fn test_watch_home_relay() { - // use an empty relay map to get full control of the changes during the test - let ops = Options { - relay_map: RelayMap::empty(), - ..Default::default() - }; - let msock = MagicSock::spawn(ops).await.unwrap(); - let mut relay_stream = msock.watch_home_relay(); - - // no relay, nothing to report - assert_eq!( - futures_lite::future::poll_once(relay_stream.next()).await, - None - ); - - let url: RelayUrl = format!("https://{}", EU_RELAY_HOSTNAME).parse().unwrap(); - msock.set_my_relay(Some(url.clone())); - - assert_eq!(relay_stream.next().await, Some(url.clone())); - - // drop the stream and query it again, the result should be immediately available - - let mut relay_stream = msock.watch_home_relay(); - assert_eq!( - futures_lite::future::poll_once(relay_stream.next()).await, - Some(Some(url)) - ); - } - - /// Creates a new [`quinn::Endpoint`] hooked up to a [`MagicSock`]. - /// - /// This is without involving [`crate::endpoint::Endpoint`]. The socket will accept - /// connections using [`ALPN`]. - /// - /// Use [`magicsock_connect`] to establish connections. - #[instrument(name = "ep", skip_all, fields(me = secret_key.public().fmt_short()))] - async fn magicsock_ep(secret_key: SecretKey) -> anyhow::Result<(quinn::Endpoint, Handle)> { - let opts = Options { - addr_v4: None, - addr_v6: None, - secret_key: secret_key.clone(), - relay_map: RelayMap::empty(), - node_map: None, - discovery: None, - dns_resolver: crate::dns::default_resolver().clone(), - proxy_url: None, - insecure_skip_relay_cert_verify: true, - }; - let msock = MagicSock::spawn(opts).await?; - let server_config = crate::endpoint::make_server_config( - &secret_key, - vec![ALPN.to_vec()], - Arc::new(quinn::TransportConfig::default()), - true, - )?; - let mut endpoint_config = quinn::EndpointConfig::default(); - endpoint_config.grease_quic_bit(false); - let endpoint = quinn::Endpoint::new_with_abstract_socket( - endpoint_config, - Some(server_config), - Arc::new(msock.clone()), - Arc::new(quinn::TokioRuntime), - )?; - Ok((endpoint, msock)) - } - - /// Connects from `ep` returned by [`magicsock_ep`] to the `node_id`. - /// - /// Uses [`ALPN`], `node_id`, must match `addr`. - #[instrument(name = "connect", skip_all, fields(me = ep_secret_key.public().fmt_short()))] - async fn magicsock_connect( - ep: &quinn::Endpoint, - ep_secret_key: SecretKey, - addr: QuicMappedAddr, - node_id: NodeId, - ) -> Result { - // Endpoint::connect sets this, do the same to have similar behaviour. - let mut transport_config = quinn::TransportConfig::default(); - transport_config.keep_alive_interval(Some(Duration::from_secs(1))); - - magicsock_connet_with_transport_config( - ep, - ep_secret_key, - addr, - node_id, - Arc::new(transport_config), - ) - .await - } - - /// Connects from `ep` returned by [`magicsock_ep`] to the `node_id`. - /// - /// This version allows customising the transport config. - /// - /// Uses [`ALPN`], `node_id`, must match `addr`. - #[instrument(name = "connect", skip_all, fields(me = ep_secret_key.public().fmt_short()))] - async fn magicsock_connet_with_transport_config( - ep: &quinn::Endpoint, - ep_secret_key: SecretKey, - addr: QuicMappedAddr, - node_id: NodeId, - transport_config: Arc, - ) -> Result { - let alpns = vec![ALPN.to_vec()]; - let quic_client_config = - tls::make_client_config(&ep_secret_key, Some(node_id), alpns, true)?; - let mut client_config = quinn::ClientConfig::new(Arc::new(quic_client_config)); - client_config.transport_config(transport_config); - let connect = ep.connect_with(client_config, addr.0, "localhost")?; - let connection = connect.await?; - Ok(connection) - } - - #[tokio::test] - async fn test_try_send_no_send_addr() { - // Regression test: if there is no send_addr we should keep being able to use the - // Endpoint. - let _guard = iroh_test::logging::setup(); - - let secret_key_1 = SecretKey::from_bytes(&[1u8; 32]); - let secret_key_2 = SecretKey::from_bytes(&[2u8; 32]); - let node_id_2 = secret_key_2.public(); - let secret_key_missing_node = SecretKey::from_bytes(&[255u8; 32]); - let node_id_missing_node = secret_key_missing_node.public(); - - let (ep_1, msock_1) = magicsock_ep(secret_key_1.clone()).await.unwrap(); - - // Generate an address not present in the NodeMap. - let bad_addr = QuicMappedAddr::generate(); - - // 500ms is rather fast here. Running this locally it should always be the correct - // timeout. If this is too slow however the test will not become flaky as we are - // expecting the timeout, we might just get the timeout for the wrong reason. But - // this speeds up the test. - let res = tokio::time::timeout( - Duration::from_millis(500), - magicsock_connect(&ep_1, secret_key_1.clone(), bad_addr, node_id_missing_node), - ) - .await; - assert!(res.is_err(), "expecting timeout"); - - // Now check we can still create another connection with this endpoint. - let (ep_2, msock_2) = magicsock_ep(secret_key_2.clone()).await.unwrap(); - - // This needs an accept task - let accept_task = tokio::spawn({ - async fn accept(ep: quinn::Endpoint) -> Result<()> { - let incoming = ep.accept().await.ok_or(anyhow!("no incoming"))?; - let _conn = incoming.accept()?.await?; - - // Keep this connection alive for a while - tokio::time::sleep(Duration::from_secs(10)).await; - info!("accept finished"); - Ok(()) - } - let ep_2 = ep_2.clone(); - async move { - if let Err(err) = accept(ep_2).await { - error!("{err:#}"); - } - } - .instrument(info_span!("ep2.accept, me = node_id_2.fmt_short()")) - }); - let _accept_task = AbortOnDropHandle::new(accept_task); - - let node_addr_2 = NodeAddr { - node_id: node_id_2, - info: AddrInfo { - relay_url: None, - direct_addresses: msock_2 - .direct_addresses() - .next() - .await - .expect("no direct addrs") - .into_iter() - .map(|x| x.addr) - .collect(), - }, - }; - msock_1 - .add_node_addr( - node_addr_2, - Source::NamedApp { - name: "test".into(), - }, - ) - .unwrap(); - let addr = msock_1.get_mapping_addr(node_id_2).unwrap(); - let res = tokio::time::timeout( - Duration::from_secs(10), - magicsock_connect(&ep_1, secret_key_1.clone(), addr, node_id_2), - ) - .await - .expect("timeout while connecting"); - - // aka assert!(res.is_ok()) but with nicer error reporting. - res.unwrap(); - - // TODO: Now check if we can connect to a repaired ep_3, but we can't modify that - // much internal state for now. - } - - #[tokio::test] - async fn test_try_send_no_udp_addr_or_relay_url() { - // This specifically tests the `if udp_addr.is_none() && relay_url.is_none()` - // behaviour of MagicSock::try_send. - let _logging_guard = iroh_test::logging::setup(); - - let secret_key_1 = SecretKey::from_bytes(&[1u8; 32]); - let secret_key_2 = SecretKey::from_bytes(&[2u8; 32]); - let node_id_2 = secret_key_2.public(); - - let (ep_1, msock_1) = magicsock_ep(secret_key_1.clone()).await.unwrap(); - let (ep_2, msock_2) = magicsock_ep(secret_key_2.clone()).await.unwrap(); - - // We need a task to accept the connection. - let accept_task = tokio::spawn({ - async fn accept(ep: quinn::Endpoint) -> Result<()> { - let incoming = ep.accept().await.ok_or(anyhow!("no incoming"))?; - let conn = incoming.accept()?.await?; - let mut stream = conn.accept_uni().await?; - stream.read_to_end(1 << 16).await?; - info!("accept finished"); - Ok(()) - } - let ep_2 = ep_2.clone(); - async move { - if let Err(err) = accept(ep_2).await { - error!("{err:#}"); - } - } - .instrument(info_span!("ep2.accept", me = node_id_2.fmt_short())) - }); - let _accept_task = AbortOnDropHandle::new(accept_task); - - // Add an empty entry in the NodeMap of ep_1 - msock_1.node_map.add_node_addr( - NodeAddr { - node_id: node_id_2, - info: AddrInfo::default(), - }, - Source::NamedApp { - name: "test".into(), - }, - ); - let addr_2 = msock_1.get_mapping_addr(node_id_2).unwrap(); - - // Set a low max_idle_timeout so quinn gives up on this quickly and our test does - // not take forever. You need to check the log output to verify this is really - // triggering the correct error. - // In test_try_send_no_send_addr() above you may have noticed we used - // tokio::time::timeout() on the connection attempt instead. Here however we want - // Quinn itself to have fully given up on the connection attempt because we will - // later connect to **the same** node. If Quinn did not give up on the connection - // we'd close it on drop, and the retransmits of the close packets would interfere - // with the next handshake, closing it during the handshake. This makes the test a - // little slower though. - let mut transport_config = quinn::TransportConfig::default(); - transport_config.max_idle_timeout(Some(Duration::from_millis(200).try_into().unwrap())); - let res = magicsock_connet_with_transport_config( - &ep_1, - secret_key_1.clone(), - addr_2, - node_id_2, - Arc::new(transport_config), - ) - .await; - assert!(res.is_err(), "expected timeout"); - info!("first connect timed out as expected"); - - // Provide correct addressing information - msock_1.node_map.add_node_addr( - NodeAddr { - node_id: node_id_2, - info: AddrInfo { - relay_url: None, - direct_addresses: msock_2 - .direct_addresses() - .next() - .await - .expect("no direct addrs") - .into_iter() - .map(|x| x.addr) - .collect(), - }, - }, - Source::NamedApp { - name: "test".into(), - }, - ); - - // We can now connect - tokio::time::timeout(Duration::from_secs(10), async move { - info!("establishing new connection"); - let conn = magicsock_connect(&ep_1, secret_key_1.clone(), addr_2, node_id_2) - .await - .unwrap(); - info!("have connection"); - let mut stream = conn.open_uni().await.unwrap(); - stream.write_all(b"hello").await.unwrap(); - stream.finish().unwrap(); - stream.stopped().await.unwrap(); - info!("finished stream"); - }) - .await - .expect("connection timed out"); - - // TODO: could remove the addresses again, send, add it back and see it recover. - // But we don't have that much private access to the NodeMap. This will do for now. - } -} diff --git a/net-report-defaults.rs b/net-report-defaults.rs deleted file mode 100644 index 26f62274156..00000000000 --- a/net-report-defaults.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! Default values used in net_report. - -/// The default STUN port used by the Relay server. -/// -/// The STUN port as defined by [RFC 8489]() -pub use iroh_base::relay_map::DEFAULT_STUN_PORT; - -/// The default QUIC port used by the Relay server to accept QUIC connections -/// for QUIC address discovery -/// -/// The port is "QUIC" typed on a phone keypad. -pub use iroh_base::relay_map::DEFAULT_QUIC_PORT; - -/// Contains all timeouts that we use in `iroh-net_report`. -pub(crate) mod timeouts { - use std::time::Duration; - - // Timeouts for net_report - - /// The maximum amount of time net_report will spend gathering a single report. - pub(crate) const OVERALL_REPORT_TIMEOUT: Duration = Duration::from_secs(5); - - /// The total time we wait for all the probes. - /// - /// This includes the STUN, ICMP and HTTPS probes, which will all - /// start at different times based on the ProbePlan. - pub(crate) const PROBES_TIMEOUT: Duration = Duration::from_secs(3); - - /// How long to await for a captive-portal result. - /// - /// This delay is chosen so it starts after good-working STUN probes - /// would have finished, but not too long so the delay is bearable if - /// STUN is blocked. - pub(crate) const CAPTIVE_PORTAL_DELAY: Duration = Duration::from_millis(200); - - /// Timeout for captive portal checks - /// - /// Must be lower than [`OVERALL_REPORT_TIMEOUT`] minus - /// [`CAPTIVE_PORTAL_DELAY`]. - pub(crate) const CAPTIVE_PORTAL_TIMEOUT: Duration = Duration::from_secs(2); - - pub(crate) const DNS_TIMEOUT: Duration = Duration::from_secs(3); - - /// The amount of time we wait for a hairpinned packet to come back. - pub(crate) const HAIRPIN_CHECK_TIMEOUT: Duration = Duration::from_millis(100); - - /// Default Pinger timeout - pub(crate) const DEFAULT_PINGER_TIMEOUT: Duration = Duration::from_secs(5); -} diff --git a/net-report-lib.rs b/net-report-lib.rs deleted file mode 100644 index 53986322ae8..00000000000 --- a/net-report-lib.rs +++ /dev/null @@ -1,1383 +0,0 @@ -//! Checks the network conditions from the current host. -//! -//! NetReport is responsible for finding out the network conditions of the current host, like -//! whether it is connected to the internet via IPv4 and/or IPv6, what the NAT situation is -//! etc and reachability to the configured relays. -// Based on - -use std::{ - collections::{BTreeMap, HashMap}, - fmt::{self, Debug}, - net::{SocketAddr, SocketAddrV4, SocketAddrV6}, - sync::Arc, -}; - -use anyhow::{anyhow, Context as _, Result}; -use bytes::Bytes; -use hickory_resolver::TokioAsyncResolver as DnsResolver; -use iroh_base::relay_map::{RelayMap, RelayNode, RelayUrl}; -#[cfg(feature = "metrics")] -use iroh_metrics::inc; -use iroh_relay::protos::stun; -use netwatch::{IpFamily, UdpSocket}; -pub use reportgen::QuicAddressDiscovery; -use tokio::{ - sync::{self, mpsc, oneshot}, - time::{Duration, Instant}, -}; -use tokio_util::{sync::CancellationToken, task::AbortOnDropHandle}; -use tracing::{debug, error, info_span, trace, warn, Instrument}; - -mod defaults; -mod dns; -#[cfg(feature = "metrics")] -mod metrics; -mod ping; -mod reportgen; - -#[cfg(feature = "metrics")] -pub use metrics::Metrics; - -const FULL_REPORT_INTERVAL: Duration = Duration::from_secs(5 * 60); - -/// The maximum latency of all nodes, if none are found yet. -/// -/// Normally the max latency of all nodes is computed, but if we don't yet know any nodes -/// latencies we return this as default. This is the value of the initial STUN probe -/// delays. It is only used as time to wait for further latencies to arrive, which *should* -/// never happen unless there already is at least one latency. Yet here we are, defining a -/// default which will never be used. -const DEFAULT_MAX_LATENCY: Duration = Duration::from_millis(100); - -/// A net_report report. -/// -/// Can be obtained by calling [`Client::get_report`]. -#[derive(Default, Debug, PartialEq, Eq, Clone)] -pub struct Report { - /// A UDP STUN round trip completed. - pub udp: bool, - /// An IPv6 STUN round trip completed. - pub ipv6: bool, - /// An IPv4 STUN round trip completed. - pub ipv4: bool, - /// An IPv6 packet was able to be sent - pub ipv6_can_send: bool, - /// an IPv4 packet was able to be sent - pub ipv4_can_send: bool, - /// could bind a socket to ::1 - pub os_has_ipv6: bool, - /// An ICMPv4 round trip completed, `None` if not checked. - pub icmpv4: Option, - /// An ICMPv6 round trip completed, `None` if not checked. - pub icmpv6: Option, - /// Whether STUN results depend on which STUN server you're talking to (on IPv4). - pub mapping_varies_by_dest_ip: Option, - /// Whether STUN results depend on which STUN server you're talking to (on IPv6). - /// - /// Note that we don't really expect this to happen and are merely logging this if - /// detecting rather than using it. For now. - pub mapping_varies_by_dest_ipv6: Option, - /// Whether the router supports communicating between two local devices through the NATted - /// public IP address (on IPv4). - pub hair_pinning: Option, - /// Probe indicating the presence of port mapping protocols on the LAN. - pub portmap_probe: Option, - /// `None` for unknown - pub preferred_relay: Option, - /// keyed by relay Url - pub relay_latency: RelayLatencies, - /// keyed by relay Url - pub relay_v4_latency: RelayLatencies, - /// keyed by relay Url - pub relay_v6_latency: RelayLatencies, - /// ip:port of global IPv4 - pub global_v4: Option, - /// `[ip]:port` of global IPv6 - pub global_v6: Option, - /// CaptivePortal is set when we think there's a captive portal that is - /// intercepting HTTP traffic. - pub captive_portal: Option, -} - -impl fmt::Display for Report { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } -} - -/// Latencies per relay node. -#[derive(Debug, Default, PartialEq, Eq, Clone)] -pub struct RelayLatencies(BTreeMap); - -impl RelayLatencies { - fn new() -> Self { - Default::default() - } - - /// Updates a relay's latency, if it is faster than before. - fn update_relay(&mut self, url: RelayUrl, latency: Duration) { - let val = self.0.entry(url).or_insert(latency); - if latency < *val { - *val = latency; - } - } - - /// Merges another [`RelayLatencies`] into this one. - /// - /// For each relay the latency is updated using [`RelayLatencies::update_relay`]. - fn merge(&mut self, other: &RelayLatencies) { - for (url, latency) in other.iter() { - self.update_relay(url.clone(), latency); - } - } - - /// Returns the maximum latency for all relays. - /// - /// If there are not yet any latencies this will return [`DEFAULT_MAX_LATENCY`]. - fn max_latency(&self) -> Duration { - self.0 - .values() - .max() - .copied() - .unwrap_or(DEFAULT_MAX_LATENCY) - } - - /// Returns an iterator over all the relays and their latencies. - pub fn iter(&self) -> impl Iterator + '_ { - self.0.iter().map(|(k, v)| (k, *v)) - } - - fn len(&self) -> usize { - self.0.len() - } - - fn is_empty(&self) -> bool { - self.0.is_empty() - } - - fn get(&self, url: &RelayUrl) -> Option { - self.0.get(url).copied() - } -} - -/// Client to run net_reports. -/// -/// Creating this creates a net_report actor which runs in the background. Most of the time -/// it is idle unless [`Client::get_report`] is called, which is the main interface. -/// -/// The [`Client`] struct can be cloned and results multiple handles to the running actor. -/// If all [`Client`]s are dropped the actor stops running. -/// -/// While running the net_report actor expects to be passed all received stun packets using -/// `Addr::receive_stun_packet`. -#[derive(Debug)] -pub struct Client { - /// Channel to send message to the [`Actor`]. - /// - /// If all senders are dropped, in other words all clones of this struct are dropped, - /// the actor will terminate. - addr: Addr, - /// Ensures the actor is terminated when the client is dropped. - _drop_guard: Arc>, -} - -#[derive(Debug)] -struct Reports { - /// Do a full relay scan, even if last is `Some`. - next_full: bool, - /// Some previous reports. - prev: HashMap>, - /// Most recent report. - last: Option>, - /// Time of last full (non-incremental) report. - last_full: Instant, -} - -impl Default for Reports { - fn default() -> Self { - Self { - next_full: Default::default(), - prev: Default::default(), - last: Default::default(), - last_full: Instant::now(), - } - } -} - -impl Client { - /// Creates a new net_report client. - /// - /// This starts a connected actor in the background. Once the client is dropped it will - /// stop running. - pub fn new(port_mapper: Option, dns_resolver: DnsResolver) -> Result { - let mut actor = Actor::new(port_mapper, dns_resolver)?; - let addr = actor.addr(); - let task = tokio::spawn( - async move { actor.run().await }.instrument(info_span!("net_report.actor")), - ); - let drop_guard = AbortOnDropHandle::new(task); - Ok(Client { - addr, - _drop_guard: Arc::new(drop_guard), - }) - } - - /// Returns a new address to send messages to this actor. - /// - /// Unlike the client itself the returned [`Addr`] does not own the actor task, it only - /// allows sending messages to the actor. - pub fn addr(&self) -> Addr { - self.addr.clone() - } - - /// Runs a net_report, returning the report. - /// - /// It may not be called concurrently with itself, `&mut self` takes care of that. - /// - /// The *stun_conn4* and *stun_conn6* endpoints are bound UDP sockets to use to send out - /// STUN packets. This function **will not read from the sockets**, as they may be - /// receiving other traffic as well, normally they are the sockets carrying the real - /// traffic. Thus all stun packets received on those sockets should be passed to - /// `Addr::receive_stun_packet` in order for this function to receive the stun - /// responses and function correctly. - /// - /// If these are not passed in this will bind sockets for STUN itself, though results - /// may not be as reliable. - pub async fn get_report( - &mut self, - dm: RelayMap, - stun_conn4: Option>, - stun_conn6: Option>, - quic_addr_disc: Option, - ) -> Result> { - let rx = self - .get_report_channel(dm, stun_conn4, stun_conn6, quic_addr_disc) - .await?; - match rx.await { - Ok(res) => res, - Err(_) => Err(anyhow!("channel closed, actor awol")), - } - } - - /// Get report with channel - pub async fn get_report_channel( - &mut self, - dm: RelayMap, - stun_conn4: Option>, - stun_conn6: Option>, - quic_addr_disc: Option, - ) -> Result>>> { - // TODO: consider if RelayMap should be made to easily clone? It seems expensive - // right now. - let (tx, rx) = oneshot::channel(); - self.addr - .send(Message::RunCheck { - relay_map: dm, - stun_sock_v4: stun_conn4, - stun_sock_v6: stun_conn6, - quic_addr_disc, - response_tx: tx, - }) - .await?; - Ok(rx) - } -} - -#[derive(Debug)] -pub(crate) struct Inflight { - /// The STUN transaction ID. - txn: stun::TransactionId, - /// The time the STUN probe was sent. - start: Instant, - /// Response to send STUN results: latency of STUN response and the discovered address. - s: sync::oneshot::Sender<(Duration, SocketAddr)>, -} - -/// Messages to send to the [`Actor`]. -#[derive(Debug)] -pub(crate) enum Message { - /// Run a net_report. - /// - /// Only one net_report can be run at a time, trying to run multiple concurrently will - /// fail. - RunCheck { - /// The relay configuration. - relay_map: RelayMap, - /// Socket to send IPv4 STUN probes from. - /// - /// Responses are never read from this socket, they must be passed in via the - /// [`Message::StunPacket`] message since the socket is also used to receive - /// other packets from in the magicsocket (`MagicSock`). - /// - /// If not provided this will attempt to bind a suitable socket itself. - stun_sock_v4: Option>, - /// Socket to send IPv6 STUN probes from. - /// - /// Like `stun_sock_v4` but for IPv6. - stun_sock_v6: Option>, - /// Endpoint and client configuration to create a QUIC - /// connection to do QUIC address discovery. - /// - /// If not provided, will not do QUIC address discovery. - quic_addr_disc: Option, - /// Channel to receive the response. - response_tx: oneshot::Sender>>, - }, - /// A report produced by the [`reportgen`] actor. - ReportReady { report: Box }, - /// The [`reportgen`] actor failed to produce a report. - ReportAborted { err: anyhow::Error }, - /// An incoming STUN packet to parse. - StunPacket { - /// The raw UDP payload. - payload: Bytes, - /// The address this was claimed to be received from. - from_addr: SocketAddr, - }, - /// A probe wants to register an in-flight STUN request. - /// - /// The sender is signalled once the STUN packet is registered with the actor and will - /// correctly accept the STUN response. - InFlightStun(Inflight, oneshot::Sender<()>), -} - -/// Sender to the main service. -/// -/// Unlike [`Client`] this is the raw channel to send messages over. Keeping this alive -/// will not keep the actor alive, which makes this handy to pass to internal tasks. -#[derive(Debug, Clone)] -pub struct Addr { - sender: mpsc::Sender, -} - -impl Addr { - /// Pass a received STUN packet to the net_reporter. - /// - /// Normally the UDP sockets to send STUN messages from are passed in so that STUN - /// packets are sent from the sockets that carry the real traffic. However because - /// these sockets carry real traffic they will also receive non-STUN traffic, thus the - /// net_report actor does not read from the sockets directly. If you receive a STUN - /// packet on the socket you should pass it to this method. - /// - /// It is safe to call this even when the net_report actor does not currently have any - /// in-flight STUN probes. The actor will simply ignore any stray STUN packets. - /// - /// There is an implicit queue here which may drop packets if the actor does not keep up - /// consuming them. - pub fn receive_stun_packet(&self, payload: Bytes, src: SocketAddr) { - if let Err(mpsc::error::TrySendError::Full(_)) = self.sender.try_send(Message::StunPacket { - payload, - from_addr: src, - }) { - #[cfg(feature = "metrics")] - inc!(Metrics, stun_packets_dropped); - warn!("dropping stun packet from {}", src); - } - } - - async fn send(&self, msg: Message) -> Result<(), mpsc::error::SendError> { - self.sender.send(msg).await.inspect_err(|_| { - error!("net_report actor lost"); - }) - } -} - -/// The net_report actor. -/// -/// This actor runs for the entire duration there's a [`Client`] connected. -#[derive(Debug)] -struct Actor { - // Actor plumbing. - /// Actor messages channel. - /// - /// If there are no more senders the actor stops. - receiver: mpsc::Receiver, - /// The sender side of the messages channel. - /// - /// This allows creating new [`Addr`]s from the actor. - sender: mpsc::Sender, - /// A collection of previously generated reports. - /// - /// Sometimes it is useful to look at past reports to decide what to do. - reports: Reports, - - // Actor configuration. - /// The port mapper client, if those are requested. - /// - /// The port mapper is responsible for talking to routers via UPnP and the like to try - /// and open ports. - port_mapper: Option, - - // Actor state. - /// Information about the currently in-flight STUN requests. - /// - /// This is used to complete the STUN probe when receiving STUN packets. - in_flight_stun_requests: HashMap, - /// The [`reportgen`] actor currently generating a report. - current_report_run: Option, - - /// The DNS resolver to use for probes that need to perform DNS lookups - dns_resolver: DnsResolver, -} - -impl Actor { - /// Creates a new actor. - /// - /// This does not start the actor, see [`Actor::run`] for this. You should not - /// normally create this directly but rather create a [`Client`]. - fn new(port_mapper: Option, dns_resolver: DnsResolver) -> Result { - // TODO: consider an instrumented flume channel so we have metrics. - let (sender, receiver) = mpsc::channel(32); - Ok(Self { - receiver, - sender, - reports: Default::default(), - port_mapper, - in_flight_stun_requests: Default::default(), - current_report_run: None, - dns_resolver, - }) - } - - /// Returns the channel to send messages to the actor. - fn addr(&self) -> Addr { - Addr { - sender: self.sender.clone(), - } - } - - /// Run the actor. - /// - /// It will now run and handle messages. Once the connected [`Client`] (including all - /// its clones) is dropped this will terminate. - async fn run(&mut self) { - debug!("net_report actor starting"); - while let Some(msg) = self.receiver.recv().await { - trace!(?msg, "handling message"); - match msg { - Message::RunCheck { - relay_map, - stun_sock_v4, - stun_sock_v6, - quic_addr_disc, - response_tx, - } => { - self.handle_run_check( - relay_map, - stun_sock_v4, - stun_sock_v6, - quic_addr_disc, - response_tx, - ); - } - Message::ReportReady { report } => { - self.handle_report_ready(report); - } - Message::ReportAborted { err } => { - self.handle_report_aborted(err); - } - Message::StunPacket { payload, from_addr } => { - self.handle_stun_packet(&payload, from_addr); - } - Message::InFlightStun(inflight, response_tx) => { - self.handle_in_flight_stun(inflight, response_tx); - } - } - } - } - - /// Starts a check run as requested by the [`Message::RunCheck`] message. - /// - /// If *stun_sock_v4* or *stun_sock_v6* are not provided this will bind the sockets - /// itself. This is not ideal since really you want to send STUN probes from the - /// sockets you will be using. - fn handle_run_check( - &mut self, - relay_map: RelayMap, - stun_sock_v4: Option>, - stun_sock_v6: Option>, - quic_addr_disc: Option, - response_tx: oneshot::Sender>>, - ) { - if self.current_report_run.is_some() { - response_tx - .send(Err(anyhow!( - "ignoring RunCheck request: reportgen actor already running" - ))) - .ok(); - return; - } - - let now = Instant::now(); - - let cancel_token = CancellationToken::new(); - let stun_sock_v4 = match stun_sock_v4 { - Some(sock) => Some(sock), - None => bind_local_stun_socket(IpFamily::V4, self.addr(), cancel_token.clone()), - }; - let stun_sock_v6 = match stun_sock_v6 { - Some(sock) => Some(sock), - None => bind_local_stun_socket(IpFamily::V6, self.addr(), cancel_token.clone()), - }; - let mut do_full = self.reports.next_full - || now.duration_since(self.reports.last_full) > FULL_REPORT_INTERVAL; - - // If the last report had a captive portal and reported no UDP access, - // it's possible that we didn't get a useful net_report due to the - // captive portal blocking us. If so, make this report a full (non-incremental) one. - if !do_full { - if let Some(ref last) = self.reports.last { - do_full = !last.udp && last.captive_portal.unwrap_or_default(); - } - } - if do_full { - self.reports.last = None; // causes ProbePlan::new below to do a full (initial) plan - self.reports.next_full = false; - self.reports.last_full = now; - #[cfg(feature = "metrics")] - inc!(Metrics, reports_full); - } - #[cfg(feature = "metrics")] - inc!(Metrics, reports); - - let actor = reportgen::Client::new( - self.addr(), - self.reports.last.clone(), - self.port_mapper.clone(), - relay_map, - stun_sock_v4, - stun_sock_v6, - quic_addr_disc, - self.dns_resolver.clone(), - ); - - self.current_report_run = Some(ReportRun { - _reportgen: actor, - _drop_guard: cancel_token.drop_guard(), - report_tx: response_tx, - }); - } - - fn handle_report_ready(&mut self, report: Box) { - let report = self.finish_and_store_report(*report); - self.in_flight_stun_requests.clear(); - if let Some(ReportRun { report_tx, .. }) = self.current_report_run.take() { - report_tx.send(Ok(report)).ok(); - } - } - - fn handle_report_aborted(&mut self, err: anyhow::Error) { - self.in_flight_stun_requests.clear(); - if let Some(ReportRun { report_tx, .. }) = self.current_report_run.take() { - report_tx.send(Err(err.context("report aborted"))).ok(); - } - } - - /// Handles [`Message::StunPacket`]. - /// - /// If there are currently no in-flight stun requests registered this is dropped, - /// otherwise forwarded to the probe. - fn handle_stun_packet(&mut self, pkt: &[u8], src: SocketAddr) { - trace!(%src, "received STUN packet"); - if self.in_flight_stun_requests.is_empty() { - return; - } - - #[cfg(feature = "metrics")] - match &src { - SocketAddr::V4(_) => { - inc!(Metrics, stun_packets_recv_ipv4); - } - SocketAddr::V6(_) => { - inc!(Metrics, stun_packets_recv_ipv6); - } - } - - match stun::parse_response(pkt) { - Ok((txn, addr_port)) => match self.in_flight_stun_requests.remove(&txn) { - Some(inf) => { - debug!(%src, %txn, "received known STUN packet"); - let elapsed = inf.start.elapsed(); - inf.s.send((elapsed, addr_port)).ok(); - } - None => { - debug!(%src, %txn, "received unexpected STUN message response"); - } - }, - Err(err) => { - match stun::parse_binding_request(pkt) { - Ok(txn) => { - // Is this our hairpin request? - match self.in_flight_stun_requests.remove(&txn) { - Some(inf) => { - debug!(%src, %txn, "received our hairpin STUN request"); - let elapsed = inf.start.elapsed(); - inf.s.send((elapsed, src)).ok(); - } - None => { - debug!(%src, %txn, "unknown STUN request"); - } - } - } - Err(_) => { - debug!(%src, "received invalid STUN response: {err:#}"); - } - } - } - } - } - - /// Handles [`Message::InFlightStun`]. - /// - /// The in-flight request is added to [`Actor::in_flight_stun_requests`] so that - /// [`Actor::handle_stun_packet`] can forward packets correctly. - /// - /// *response_tx* is to signal the actor message has been handled. - fn handle_in_flight_stun(&mut self, inflight: Inflight, response_tx: oneshot::Sender<()>) { - self.in_flight_stun_requests.insert(inflight.txn, inflight); - response_tx.send(()).ok(); - } - - fn finish_and_store_report(&mut self, report: Report) -> Arc { - let report = self.add_report_history_and_set_preferred_relay(report); - debug!("{report:?}"); - report - } - - /// Adds `r` to the set of recent Reports and mutates `r.preferred_relay` to contain the best recent one. - /// `r` is stored ref counted and a reference is returned. - fn add_report_history_and_set_preferred_relay(&mut self, mut r: Report) -> Arc { - let mut prev_relay = None; - if let Some(ref last) = self.reports.last { - prev_relay.clone_from(&last.preferred_relay); - } - let now = Instant::now(); - const MAX_AGE: Duration = Duration::from_secs(5 * 60); - - // relay ID => its best recent latency in last MAX_AGE - let mut best_recent = RelayLatencies::new(); - - // chain the current report as we are still mutating it - let prevs_iter = self - .reports - .prev - .iter() - .map(|(a, b)| -> (&Instant, &Report) { (a, b) }) - .chain(std::iter::once((&now, &r))); - - let mut to_remove = Vec::new(); - for (t, pr) in prevs_iter { - if now.duration_since(*t) > MAX_AGE { - to_remove.push(*t); - continue; - } - best_recent.merge(&pr.relay_latency); - } - - for t in to_remove { - self.reports.prev.remove(&t); - } - - // Then, pick which currently-alive relay server from the - // current report has the best latency over the past MAX_AGE. - let mut best_any = Duration::default(); - let mut old_relay_cur_latency = Duration::default(); - { - for (url, duration) in r.relay_latency.iter() { - if Some(url) == prev_relay.as_ref() { - old_relay_cur_latency = duration; - } - if let Some(best) = best_recent.get(url) { - if r.preferred_relay.is_none() || best < best_any { - best_any = best; - r.preferred_relay.replace(url.clone()); - } - } - } - - // If we're changing our preferred relay but the old one's still - // accessible and the new one's not much better, just stick with - // where we are. - if prev_relay.is_some() - && r.preferred_relay != prev_relay - && !old_relay_cur_latency.is_zero() - && best_any > old_relay_cur_latency / 3 * 2 - { - r.preferred_relay = prev_relay; - } - } - - let r = Arc::new(r); - self.reports.prev.insert(now, r.clone()); - self.reports.last = Some(r.clone()); - - r - } -} - -/// State the net_report actor needs for an in-progress report generation. -#[derive(Debug)] -struct ReportRun { - /// The handle of the [`reportgen`] actor, cancels the actor on drop. - _reportgen: reportgen::Client, - /// Drop guard to optionally kill workers started by net_report to support reportgen. - _drop_guard: tokio_util::sync::DropGuard, - /// Where to send the completed report. - report_tx: oneshot::Sender>>, -} - -/// Attempts to bind a local socket to send STUN packets from. -/// -/// If successful this returns the bound socket and will forward STUN responses to the -/// provided *actor_addr*. The *cancel_token* serves to stop the packet forwarding when the -/// socket is no longer needed. -fn bind_local_stun_socket( - network: IpFamily, - actor_addr: Addr, - cancel_token: CancellationToken, -) -> Option> { - let sock = match UdpSocket::bind(network, 0) { - Ok(sock) => Arc::new(sock), - Err(err) => { - debug!("failed to bind STUN socket: {}", err); - return None; - } - }; - let span = info_span!( - "stun_udp_listener", - local_addr = sock - .local_addr() - .map(|a| a.to_string()) - .unwrap_or(String::from("-")), - ); - { - let sock = sock.clone(); - tokio::spawn( - async move { - debug!("udp stun socket listener started"); - // TODO: Can we do better for buffers here? Probably doesn't matter much. - let mut buf = vec![0u8; 64 << 10]; - loop { - tokio::select! { - biased; - _ = cancel_token.cancelled() => break, - res = recv_stun_once(&sock, &mut buf, &actor_addr) => { - if let Err(err) = res { - warn!(%err, "stun recv failed"); - break; - } - } - } - } - debug!("udp stun socket listener stopped"); - } - .instrument(span), - ); - } - Some(sock) -} - -/// Receive STUN response from a UDP socket, pass it to the actor. -async fn recv_stun_once(sock: &UdpSocket, buf: &mut [u8], actor_addr: &Addr) -> Result<()> { - let (count, mut from_addr) = sock - .recv_from(buf) - .await - .context("Error reading from stun socket")?; - let payload = &buf[..count]; - from_addr.set_ip(from_addr.ip().to_canonical()); - let msg = Message::StunPacket { - payload: Bytes::from(payload.to_vec()), - from_addr, - }; - actor_addr.send(msg).await.context("actor stopped") -} - -/// Test if IPv6 works at all, or if it's been hard disabled at the OS level. -pub fn os_has_ipv6() -> bool { - UdpSocket::bind_local_v6(0).is_ok() -} - -#[cfg(test)] -mod test_utils { - //! Creates a relay server against which to perform tests - - use std::sync::Arc; - - use iroh_relay::server::{ - self, - testing::{quic_config, relay_config}, - ServerConfig, - }; - - use crate::RelayNode; - - pub(crate) async fn relay() -> (server::Server, Arc) { - let server = server::Server::spawn(server::testing::server_config()) - .await - .expect("should serve relay"); - let node_desc = RelayNode { - url: server.https_url().expect("should work as relay"), - stun_only: false, // the checks above and below guarantee both stun and relay - stun_port: server.stun_addr().expect("server should serve stun").port(), - quic_only: false, - quic_port: server - .quic_addr() - .expect("server should serve quic address discovery") - .port(), - }; - - (server, Arc::new(node_desc)) - } - - pub(crate) async fn relay_with_quic() -> (server::Server, Arc) { - let server_config = ServerConfig { - relay: Some(relay_config()), - stun: None, - quic: Some(quic_config()), - metrics_addr: None, - }; - let server = server::Server::spawn(server_config) - .await - .expect("should serve relay"); - let node_desc = RelayNode { - url: server.https_url().expect("should work as relay"), - stun_only: false, - stun_port: 0, - quic_only: false, - quic_port: server - .quic_addr() - .expect("server should serve quic address discovery") - .port(), - }; - - (server, Arc::new(node_desc)) - } - - /// Create a [`crate::RelayMap`] of the given size. - /// - /// This function uses [`relay`]. Note that the returned map uses internal order that will - /// often _not_ match the order of the servers. - pub(crate) async fn relay_map(relays: usize) -> (Vec, crate::RelayMap) { - let mut servers = Vec::with_capacity(relays); - let mut nodes = Vec::with_capacity(relays); - for _ in 0..relays { - let (relay_server, node) = relay().await; - servers.push(relay_server); - nodes.push(node); - } - let map = crate::RelayMap::from_nodes(nodes).expect("unuque urls"); - (servers, map) - } -} - -#[cfg(test)] -mod tests { - use std::net::Ipv4Addr; - - use bytes::BytesMut; - use tokio::time; - use tracing::info; - - use super::*; - use crate::ping::Pinger; - - mod stun_utils { - //! Utils for testing that expose a simple stun server. - - use std::{net::IpAddr, sync::Arc}; - - use anyhow::Result; - use tokio::{ - net, - sync::{oneshot, Mutex}, - }; - use tracing::{debug, trace}; - - use super::*; - use crate::{RelayMap, RelayNode, RelayUrl}; - - /// A drop guard to clean up test infrastructure. - /// - /// After dropping the test infrastructure will asynchronously shutdown and release its - /// resources. - // Nightly sees the sender as dead code currently, but we only rely on Drop of the - // sender. - #[derive(Debug)] - pub struct CleanupDropGuard { - _guard: oneshot::Sender<()>, - } - - // (read_ipv4, read_ipv6) - #[derive(Debug, Default, Clone)] - pub struct StunStats(Arc>); - - impl StunStats { - pub async fn total(&self) -> usize { - let s = self.0.lock().await; - s.0 + s.1 - } - } - - pub fn relay_map_of(stun: impl Iterator) -> RelayMap { - relay_map_of_opts(stun.map(|addr| (addr, true))) - } - - pub fn relay_map_of_opts(stun: impl Iterator) -> RelayMap { - let nodes = stun.map(|(addr, stun_only)| { - let host = addr.ip(); - let port = addr.port(); - - let url: RelayUrl = format!("http://{host}:{port}").parse().unwrap(); - RelayNode { - url, - stun_port: port, - stun_only, - quic_only: false, - quic_port: 0, - } - }); - RelayMap::from_nodes(nodes).expect("generated invalid nodes") - } - - /// Sets up a simple STUN server binding to `0.0.0.0:0`. - /// - /// See [`serve`] for more details. - pub(crate) async fn serve_v4() -> Result<(SocketAddr, StunStats, CleanupDropGuard)> { - serve(std::net::Ipv4Addr::UNSPECIFIED.into()).await - } - - /// Sets up a simple STUN server. - pub(crate) async fn serve(ip: IpAddr) -> Result<(SocketAddr, StunStats, CleanupDropGuard)> { - let stats = StunStats::default(); - - let pc = net::UdpSocket::bind((ip, 0)).await?; - let mut addr = pc.local_addr()?; - match addr.ip() { - IpAddr::V4(ip) => { - if ip.octets() == [0, 0, 0, 0] { - addr.set_ip("127.0.0.1".parse().unwrap()); - } - } - _ => unreachable!("using ipv4"), - } - - println!("STUN listening on {}", addr); - let (_guard, r) = oneshot::channel(); - let stats_c = stats.clone(); - tokio::task::spawn(async move { - run_stun(pc, stats_c, r).await; - }); - - Ok((addr, stats, CleanupDropGuard { _guard })) - } - - async fn run_stun(pc: net::UdpSocket, stats: StunStats, mut done: oneshot::Receiver<()>) { - let mut buf = vec![0u8; 64 << 10]; - loop { - trace!("read loop"); - tokio::select! { - _ = &mut done => { - debug!("shutting down"); - break; - } - res = pc.recv_from(&mut buf) => match res { - Ok((n, addr)) => { - trace!("read packet {}bytes from {}", n, addr); - let pkt = &buf[..n]; - if !stun::is(pkt) { - debug!("received non STUN pkt"); - continue; - } - if let Ok(txid) = stun::parse_binding_request(pkt) { - debug!("received binding request"); - let mut s = stats.0.lock().await; - if addr.is_ipv4() { - s.0 += 1; - } else { - s.1 += 1; - } - drop(s); - - let res = stun::response(txid, addr); - if let Err(err) = pc.send_to(&res, addr).await { - eprintln!("STUN server write failed: {:?}", err); - } - } - } - Err(err) => { - eprintln!("failed to read: {:?}", err); - } - } - } - } - } - } - - #[tokio::test] - async fn test_basic() -> Result<()> { - let _guard = iroh_test::logging::setup(); - let (stun_addr, stun_stats, _cleanup_guard) = - stun_utils::serve("127.0.0.1".parse().unwrap()).await?; - - let resolver = crate::dns::tests::resolver(); - let mut client = Client::new(None, resolver.clone())?; - let dm = stun_utils::relay_map_of([stun_addr].into_iter()); - - // Note that the ProbePlan will change with each iteration. - for i in 0..5 { - println!("--round {}", i); - let r = client.get_report(dm.clone(), None, None, None).await?; - - assert!(r.udp, "want UDP"); - assert_eq!( - r.relay_latency.len(), - 1, - "expected 1 key in RelayLatency; got {}", - r.relay_latency.len() - ); - assert!( - r.relay_latency.iter().next().is_some(), - "expected key 1 in RelayLatency; got {:?}", - r.relay_latency - ); - assert!(r.global_v4.is_some(), "expected globalV4 set"); - assert!(r.preferred_relay.is_some(),); - } - - assert!( - stun_stats.total().await >= 5, - "expected at least 5 stun, got {}", - stun_stats.total().await, - ); - - Ok(()) - } - - #[tokio::test] - async fn test_udp_blocked() -> Result<()> { - let _guard = iroh_test::logging::setup(); - - // Create a "STUN server", which will never respond to anything. This is how UDP to - // the STUN server being blocked will look like from the client's perspective. - let blackhole = tokio::net::UdpSocket::bind("127.0.0.1:0").await?; - let stun_addr = blackhole.local_addr()?; - let dm = stun_utils::relay_map_of_opts([(stun_addr, false)].into_iter()); - - // Now create a client and generate a report. - let resolver = crate::dns::tests::resolver(); - let mut client = Client::new(None, resolver.clone())?; - - let r = client.get_report(dm, None, None, None).await?; - let mut r: Report = (*r).clone(); - r.portmap_probe = None; - - // This test wants to ensure that the ICMP part of the probe works when UDP is - // blocked. Unfortunately on some systems we simply don't have permissions to - // create raw ICMP pings and we'll have to silently accept this test is useless (if - // we could, this would be a skip instead). - let pinger = Pinger::new(); - let can_ping = pinger.send(Ipv4Addr::LOCALHOST.into(), b"aa").await.is_ok(); - let want_icmpv4 = match can_ping { - true => Some(true), - false => None, - }; - - let want = Report { - // The ICMP probe sets the can_ping flag. - ipv4_can_send: can_ping, - // OS IPv6 test is irrelevant here, accept whatever the current machine has. - os_has_ipv6: r.os_has_ipv6, - // Captive portal test is irrelevant; accept what the current report has. - captive_portal: r.captive_portal, - // If we can ping we expect to have this. - icmpv4: want_icmpv4, - // If we had a pinger, we'll have some latencies filled in and a preferred relay - relay_latency: can_ping - .then(|| r.relay_latency.clone()) - .unwrap_or_default(), - preferred_relay: can_ping - .then_some(r.preferred_relay.clone()) - .unwrap_or_default(), - ..Default::default() - }; - - assert_eq!(r, want); - - Ok(()) - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_add_report_history_set_preferred_relay() -> Result<()> { - fn relay_url(i: u16) -> RelayUrl { - format!("http://{i}.com").parse().unwrap() - } - - // report returns a *Report from (relay host, Duration)+ pairs. - fn report(a: impl IntoIterator) -> Option> { - let mut report = Report::default(); - for (s, d) in a { - assert!(s.starts_with('d'), "invalid relay server key"); - let id: u16 = s[1..].parse().unwrap(); - report - .relay_latency - .0 - .insert(relay_url(id), Duration::from_secs(d)); - } - - Some(Arc::new(report)) - } - struct Step { - /// Delay in seconds - after: u64, - r: Option>, - } - struct Test { - name: &'static str, - steps: Vec, - /// want PreferredRelay on final step - want_relay: Option, - // wanted len(c.prev) - want_prev_len: usize, - } - - let tests = [ - Test { - name: "first_reading", - steps: vec![Step { - after: 0, - r: report([("d1", 2), ("d2", 3)]), - }], - want_prev_len: 1, - want_relay: Some(relay_url(1)), - }, - Test { - name: "with_two", - steps: vec![ - Step { - after: 0, - r: report([("d1", 2), ("d2", 3)]), - }, - Step { - after: 1, - r: report([("d1", 4), ("d2", 3)]), - }, - ], - want_prev_len: 2, - want_relay: Some(relay_url(1)), // t0's d1 of 2 is still best - }, - Test { - name: "but_now_d1_gone", - steps: vec![ - Step { - after: 0, - r: report([("d1", 2), ("d2", 3)]), - }, - Step { - after: 1, - r: report([("d1", 4), ("d2", 3)]), - }, - Step { - after: 2, - r: report([("d2", 3)]), - }, - ], - want_prev_len: 3, - want_relay: Some(relay_url(2)), // only option - }, - Test { - name: "d1_is_back", - steps: vec![ - Step { - after: 0, - r: report([("d1", 2), ("d2", 3)]), - }, - Step { - after: 1, - r: report([("d1", 4), ("d2", 3)]), - }, - Step { - after: 2, - r: report([("d2", 3)]), - }, - Step { - after: 3, - r: report([("d1", 4), ("d2", 3)]), - }, // same as 2 seconds ago - ], - want_prev_len: 4, - want_relay: Some(relay_url(1)), // t0's d1 of 2 is still best - }, - Test { - name: "things_clean_up", - steps: vec![ - Step { - after: 0, - r: report([("d1", 1), ("d2", 2)]), - }, - Step { - after: 1, - r: report([("d1", 1), ("d2", 2)]), - }, - Step { - after: 2, - r: report([("d1", 1), ("d2", 2)]), - }, - Step { - after: 3, - r: report([("d1", 1), ("d2", 2)]), - }, - Step { - after: 10 * 60, - r: report([("d3", 3)]), - }, - ], - want_prev_len: 1, // t=[0123]s all gone. (too old, older than 10 min) - want_relay: Some(relay_url(3)), // only option - }, - Test { - name: "preferred_relay_hysteresis_no_switch", - steps: vec![ - Step { - after: 0, - r: report([("d1", 4), ("d2", 5)]), - }, - Step { - after: 1, - r: report([("d1", 4), ("d2", 3)]), - }, - ], - want_prev_len: 2, - want_relay: Some(relay_url(1)), // 2 didn't get fast enough - }, - Test { - name: "preferred_relay_hysteresis_do_switch", - steps: vec![ - Step { - after: 0, - r: report([("d1", 4), ("d2", 5)]), - }, - Step { - after: 1, - r: report([("d1", 4), ("d2", 1)]), - }, - ], - want_prev_len: 2, - want_relay: Some(relay_url(2)), // 2 got fast enough - }, - ]; - let resolver = crate::dns::tests::resolver(); - for mut tt in tests { - println!("test: {}", tt.name); - let mut actor = Actor::new(None, resolver.clone()).unwrap(); - for s in &mut tt.steps { - // trigger the timer - time::advance(Duration::from_secs(s.after)).await; - let r = Arc::try_unwrap(s.r.take().unwrap()).unwrap(); - s.r = Some(actor.add_report_history_and_set_preferred_relay(r)); - } - let last_report = tt.steps.last().unwrap().r.clone().unwrap(); - let got = actor.reports.prev.len(); - let want = tt.want_prev_len; - assert_eq!(got, want, "prev length"); - let got = &last_report.preferred_relay; - let want = &tt.want_relay; - assert_eq!(got, want, "preferred_relay"); - } - - Ok(()) - } - - #[tokio::test] - async fn test_hairpin() -> Result<()> { - // Hairpinning is initiated after we discover our own IPv4 socket address (IP + - // port) via STUN, so the test needs to have a STUN server and perform STUN over - // IPv4 first. Hairpinning detection works by sending a STUN *request* to **our own - // public socket address** (IP + port). If the router supports hairpinning the STUN - // request is returned back to us and received on our public address. This doesn't - // need to be a STUN request, but STUN already has a unique transaction ID which we - // can easily use to identify the packet. - - // Setup STUN server and create relay_map. - let (stun_addr, _stun_stats, _done) = stun_utils::serve_v4().await?; - let dm = stun_utils::relay_map_of([stun_addr].into_iter()); - dbg!(&dm); - - let resolver = crate::dns::tests::resolver().clone(); - let mut client = Client::new(None, resolver)?; - - // Set up an external socket to send STUN requests from, this will be discovered as - // our public socket address by STUN. We send back any packets received on this - // socket to the net_report client using Client::receive_stun_packet. Once we sent - // the hairpin STUN request (from a different randomly bound socket) we are sending - // it to this socket, which is forwarnding it back to our net_report client, because - // this dumb implementation just forwards anything even if it would be garbage. - // Thus hairpinning detection will declare hairpinning to work. - let sock = UdpSocket::bind_local(IpFamily::V4, 0)?; - let sock = Arc::new(sock); - info!(addr=?sock.local_addr().unwrap(), "Using local addr"); - let task = { - let sock = sock.clone(); - let addr = client.addr.clone(); - tokio::spawn( - async move { - let mut buf = BytesMut::zeroed(64 << 10); - loop { - let (count, src) = sock.recv_from(&mut buf).await.unwrap(); - info!( - addr=?sock.local_addr().unwrap(), - %count, - "Forwarding payload to net_report client", - ); - let payload = buf.split_to(count).freeze(); - addr.receive_stun_packet(payload, src); - } - } - .instrument(info_span!("pkt-fwd")), - ) - }; - - let r = client.get_report(dm, Some(sock), None, None).await?; - dbg!(&r); - assert_eq!(r.hair_pinning, Some(true)); - - task.abort(); - Ok(()) - } - - #[tokio::test] - async fn test_quic_basic() -> Result<()> { - let _logging_guard = iroh_test::logging::setup(); - // set up relay server that has quic enabled, but not stun - let (server, relay) = test_utils::relay_with_quic().await; - - // set up quic client endpoint to use in the report - let client_config = iroh_relay::client::make_dangerous_client_config(); - let client_config = quinn::ClientConfig::new(Arc::new(client_config)); - let ep = quinn::Endpoint::client(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))?; - let addr = match ep.local_addr()? { - SocketAddr::V4(ipp) => ipp, - SocketAddr::V6(_) => unreachable!(), - }; - let quic_addr_disc = QuicAddressDiscovery { - ep: ep.clone(), - client_config, - }; - - // create a net report client - let resolver = crate::dns::tests::resolver(); - let mut client = Client::new(None, resolver.clone())?; - - let relay_map = RelayMap::from_nodes(vec![relay])?; - let r = client - .get_report(relay_map, None, None, Some(quic_addr_disc)) - .await?; - assert!(r.ipv4); - assert!(r.ipv4_can_send); - assert_eq!(r.global_v4, Some(addr)); - - // cleanup - ep.wait_idle().await; - server.shutdown().await?; - Ok(()) - } -} diff --git a/probes.rs b/probes.rs deleted file mode 100644 index f2a46180cbc..00000000000 --- a/probes.rs +++ /dev/null @@ -1,877 +0,0 @@ -//! The relay probes. -//! -//! All the probes try and establish the latency to the relay servers. Preferably the STUN -//! probes work and we also learn about our public IP addresses and ports. But fallback -//! probes for HTTPS and ICMP exist as well. - -use std::{collections::BTreeSet, fmt, sync::Arc}; - -use anyhow::{ensure, Result}; -use netwatch::interfaces; -use tokio::time::Duration; - -use crate::{RelayMap, RelayNode, RelayUrl, Report}; - -/// The retransmit interval used when net_report first runs. -/// -/// We have no past context to work with, and we want answers relatively quickly, so it's -/// biased slightly more aggressive than [`DEFAULT_ACTIVE_RETRANSMIT_DELAY`]. A few extra -/// packets at startup is fine. -const DEFAULT_INITIAL_RETRANSMIT: Duration = Duration::from_millis(100); - -/// The retransmit interval used when a previous report exists but is missing latency. -/// -/// When in an active steady-state, i.e. a previous report exists, we use the latency of the -/// previous report to determine the retransmit interval. However when this previous relay -/// latency is missing this default is used. -/// -/// This is a somewhat conservative guess because if we have no data, likely the relay node -/// is very far away and we have no data because we timed out the last time we probed it. -const DEFAULT_ACTIVE_RETRANSMIT_DELAY: Duration = Duration::from_millis(200); - -/// The extra time to add to retransmits if a previous report exists. -/// -/// When in an active steady-state, i.e. a previous report exists, we add this delay -/// multiplied with the attempt to probe retries to give later attempts increasingly more -/// time. -const ACTIVE_RETRANSMIT_EXTRA_DELAY: Duration = Duration::from_millis(50); - -/// The number of fastest relays to periodically re-query during incremental net_report -/// reports. (During a full report, all relay servers are scanned.) -const NUM_INCREMENTAL_RELAYS: usize = 3; - -/// The protocol used to time a node's latency. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, derive_more::Display)] -#[repr(u8)] -pub(super) enum ProbeProto { - /// STUN IPv4 - StunIpv4, - /// STUN IPv6 - StunIpv6, - /// HTTPS - Https, - /// ICMP IPv4 - IcmpV4, - /// ICMP IPv6 - IcmpV6, - /// QUIC Address Discovery IPv4 - QuicAddrIpv4, - /// QUIC Address Discovery IPv6 - QuicAddrIpv6, -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, derive_more::Display)] -pub(super) enum Probe { - #[display("STUN Ipv4 after {delay:?} to {node}")] - StunIpv4 { - /// When the probe is started, relative to the time that `get_report` is called. - /// One probe in each `ProbePlan` should have a delay of 0. Non-zero values - /// are for retries on UDP loss or timeout. - delay: Duration, - - /// The relay server to send this probe to. - node: Arc, - }, - #[display("STUN Ipv6 after {delay:?} to {node}")] - StunIpv6 { - delay: Duration, - node: Arc, - }, - #[display("HTTPS after {delay:?} to {node}")] - Https { - delay: Duration, - node: Arc, - }, - #[display("ICMPv4 after {delay:?} to {node}")] - IcmpV4 { - delay: Duration, - node: Arc, - }, - #[display("ICMPv6 after {delay:?} to {node}")] - IcmpV6 { - delay: Duration, - node: Arc, - }, - #[display("QUIC Addr Ivp4 after {delay:?} to {node}")] - QuicAddrIpv4 { - delay: Duration, - node: Arc, - }, - #[display("QUIC Addr Ivp6 after {delay:?} to {node}")] - QuicAddrIpv6 { - delay: Duration, - node: Arc, - }, -} - -impl Probe { - pub(super) fn delay(&self) -> Duration { - match self { - Probe::StunIpv4 { delay, .. } - | Probe::StunIpv6 { delay, .. } - | Probe::Https { delay, .. } - | Probe::IcmpV4 { delay, .. } - | Probe::IcmpV6 { delay, .. } - | Probe::QuicAddrIpv4 { delay, .. } - | Probe::QuicAddrIpv6 { delay, .. } => *delay, - } - } - - pub(super) fn proto(&self) -> ProbeProto { - match self { - Probe::StunIpv4 { .. } => ProbeProto::StunIpv4, - Probe::StunIpv6 { .. } => ProbeProto::StunIpv6, - Probe::Https { .. } => ProbeProto::Https, - Probe::IcmpV4 { .. } => ProbeProto::IcmpV4, - Probe::IcmpV6 { .. } => ProbeProto::IcmpV6, - Probe::QuicAddrIpv4 { .. } => ProbeProto::QuicAddrIpv4, - Probe::QuicAddrIpv6 { .. } => ProbeProto::QuicAddrIpv6, - } - } - - pub(super) fn node(&self) -> &Arc { - match self { - Probe::StunIpv4 { node, .. } - | Probe::StunIpv6 { node, .. } - | Probe::Https { node, .. } - | Probe::IcmpV4 { node, .. } - | Probe::IcmpV6 { node, .. } - | Probe::QuicAddrIpv4 { node, .. } - | Probe::QuicAddrIpv6 { node, .. } => node, - } - } -} - -/// A probe set is a sequence of similar [`Probe`]s with delays between them. -/// -/// The probes are to the same Relayer and of the same [`ProbeProto`] but will have different -/// delays. The delays are effectively retries, though they do not wait for the previous -/// probe to be finished. The first successful probe will cancel all other probes in the -/// set. -/// -/// This is a lot of type-safety by convention. It would be so much nicer to have this -/// compile-time checked but that introduces a giant mess of generics and traits and -/// associated exploding types. -/// -/// A [`ProbeSet`] implements [`IntoIterator`] similar to how [`Vec`] does. -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] -pub(super) struct ProbeSet { - /// The [`ProbeProto`] all the probes in this set have. - proto: ProbeProto, - /// The probes in the set. - probes: Vec, -} - -impl ProbeSet { - fn new(proto: ProbeProto) -> Self { - Self { - probes: Vec::new(), - proto, - } - } - - fn push(&mut self, probe: Probe) -> Result<()> { - ensure!(probe.proto() == self.proto, "mismatching probe proto"); - self.probes.push(probe); - Ok(()) - } - - fn is_empty(&self) -> bool { - self.probes.is_empty() - } -} - -impl<'a> IntoIterator for &'a ProbeSet { - type Item = &'a Probe; - - type IntoIter = std::slice::Iter<'a, Probe>; - - fn into_iter(self) -> Self::IntoIter { - self.probes.iter() - } -} - -impl fmt::Display for ProbeSet { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - writeln!(f, r#"ProbeSet("{}") {{"#, self.proto)?; - for probe in self.probes.iter() { - writeln!(f, " {probe},")?; - } - writeln!(f, "}}") - } -} - -/// A probe plan. -/// -/// A probe plan contains a number of [`ProbeSet`]s containing probes to be executed. -/// Generally the first probe of of a set which completes aborts the remaining probes of a -/// set. Sometimes a failing probe can also abort the remaining probes of a set. -/// -/// The [`reportgen`] actor will also abort all the remaining [`ProbeSet`]s once it has -/// sufficient information for a report. -/// -/// [`reportgen`]: crate::reportgen -#[derive(Debug, PartialEq, Eq)] -pub(super) struct ProbePlan(BTreeSet); - -impl ProbePlan { - /// Creates an initial probe plan. - pub(super) fn initial(relay_map: &RelayMap, if_state: &interfaces::State) -> Self { - let mut plan = Self(BTreeSet::new()); - - // The first time we need add probes after the STUN we record this delay, so that - // further relay server can reuse this delay. - let mut max_stun_delay: Option = None; - - for relay_node in relay_map.nodes() { - let mut stun_ipv4_probes = ProbeSet::new(ProbeProto::StunIpv4); - let mut stun_ipv6_probes = ProbeSet::new(ProbeProto::StunIpv6); - let mut quic_ipv4_probes = ProbeSet::new(ProbeProto::QuicAddrIpv4); - let mut quic_ipv6_probes = ProbeSet::new(ProbeProto::QuicAddrIpv6); - - for attempt in 0..3 { - let delay = DEFAULT_INITIAL_RETRANSMIT * attempt as u32; - - if if_state.have_v4 { - stun_ipv4_probes - .push(Probe::StunIpv4 { - delay, - node: relay_node.clone(), - }) - .expect("adding StunIpv4 probe to a StunIpv4 probe set"); - quic_ipv4_probes - .push(Probe::QuicAddrIpv4 { - delay, - node: relay_node.clone(), - }) - .expect("adding QuicAddrIpv4 probe to a QuicAddrIpv4 probe set"); - } - if if_state.have_v6 { - stun_ipv6_probes - .push(Probe::StunIpv6 { - delay, - node: relay_node.clone(), - }) - .expect("adding StunIpv6 probe to a StunIpv6 probe set"); - quic_ipv6_probes - .push(Probe::QuicAddrIpv6 { - delay, - node: relay_node.clone(), - }) - .expect("adding QuicAddrIpv6 probe to a QuicAddrIpv6 probe set"); - } - } - plan.add(stun_ipv4_probes); - plan.add(stun_ipv6_probes); - plan.add(quic_ipv4_probes); - plan.add(quic_ipv6_probes); - - // The HTTP and ICMP probes only start after the STUN and QUIC probes have had a chance. - let mut https_probes = ProbeSet::new(ProbeProto::Https); - let mut icmp_probes_ipv4 = ProbeSet::new(ProbeProto::IcmpV4); - let mut icmp_probes_ipv6 = ProbeSet::new(ProbeProto::IcmpV6); - for attempt in 0..3 { - let start = *max_stun_delay.get_or_insert_with(|| plan.max_delay()) - + DEFAULT_INITIAL_RETRANSMIT; - let delay = start + DEFAULT_INITIAL_RETRANSMIT * attempt as u32; - - https_probes - .push(Probe::Https { - delay, - node: relay_node.clone(), - }) - .expect("adding Https probe to a Https probe set"); - if if_state.have_v4 { - icmp_probes_ipv4 - .push(Probe::IcmpV4 { - delay, - node: relay_node.clone(), - }) - .expect("adding Icmp probe to an Icmp probe set"); - } - if if_state.have_v6 { - icmp_probes_ipv6 - .push(Probe::IcmpV6 { - delay, - node: relay_node.clone(), - }) - .expect("adding IcmpIpv6 probe to and IcmpIpv6 probe set"); - } - } - plan.add(https_probes); - plan.add(icmp_probes_ipv4); - plan.add(icmp_probes_ipv6); - } - plan - } - - /// Creates a follow up probe plan using a previous net_report report. - pub(super) fn with_last_report( - relay_map: &RelayMap, - if_state: &interfaces::State, - last_report: &Report, - ) -> Self { - if last_report.relay_latency.is_empty() { - return Self::initial(relay_map, if_state); - } - let mut plan = Self(Default::default()); - - // The first time we need add probes after the STUN we record this delay, so that - // further relay servers can reuse this delay. - let mut max_stun_delay: Option = None; - - let had_ipv4 = !last_report.relay_v4_latency.is_empty(); - let had_ipv6 = !last_report.relay_v6_latency.is_empty(); - let had_both = if_state.have_v6 && had_ipv4 && had_ipv6; - let sorted_relays = sort_relays(relay_map, last_report); - for (ri, (url, relay_node)) in sorted_relays.into_iter().enumerate() { - if ri == NUM_INCREMENTAL_RELAYS { - break; - } - let mut do4 = if_state.have_v4; - let mut do6 = if_state.have_v6; - - // By default, each node only gets one STUN packet sent, - // except the fastest two from the previous round. - let mut attempts = 1; - let is_fastest_two = ri < 2; - - if is_fastest_two { - attempts = 2; - } else if had_both { - // For dual stack machines, make the 3rd & slower nodes alternate between - // IPv4 and IPv6 for STUN and ICMP probes. - if ri % 2 == 0 { - (do4, do6) = (true, false); - } else { - (do4, do6) = (false, true); - } - } - if !is_fastest_two && !had_ipv6 { - do6 = false; - } - if Some(url) == last_report.preferred_relay.as_ref() { - // But if we already had a relay home, try extra hard to - // make sure it's there so we don't flip flop around. - attempts = 4; - } - let retransmit_delay = last_report - .relay_latency - .get(url) - .map(|l| l * 120 / 100) // increases latency by 20%, why? - .unwrap_or(DEFAULT_ACTIVE_RETRANSMIT_DELAY); - - let mut stun_ipv4_probes = ProbeSet::new(ProbeProto::StunIpv4); - let mut stun_ipv6_probes = ProbeSet::new(ProbeProto::StunIpv6); - let mut quic_ipv4_probes = ProbeSet::new(ProbeProto::QuicAddrIpv4); - let mut quic_ipv6_probes = ProbeSet::new(ProbeProto::QuicAddrIpv6); - - for attempt in 0..attempts { - let delay = (retransmit_delay * attempt as u32) - + (ACTIVE_RETRANSMIT_EXTRA_DELAY * attempt as u32); - if do4 { - stun_ipv4_probes - .push(Probe::StunIpv4 { - delay, - node: relay_node.clone(), - }) - .expect("Pushing StunIpv4 Probe to StunIpv4 ProbeSet"); - quic_ipv4_probes - .push(Probe::QuicAddrIpv4 { - delay, - node: relay_node.clone(), - }) - .expect("adding QuicAddrIpv4 probe to a QuicAddrIpv4 probe set"); - } - if do6 { - stun_ipv6_probes - .push(Probe::StunIpv6 { - delay, - node: relay_node.clone(), - }) - .expect("Pushing StunIpv6 Probe to StunIpv6 ProbeSet"); - quic_ipv6_probes - .push(Probe::QuicAddrIpv6 { - delay, - node: relay_node.clone(), - }) - .expect("adding QuicAddrIpv6 probe to a QuicAddrIpv6 probe set"); - } - } - plan.add(stun_ipv4_probes); - plan.add(stun_ipv6_probes); - plan.add(quic_ipv4_probes); - plan.add(quic_ipv6_probes); - - // The HTTP and ICMP probes only start after the STUN probes have had a chance. - let mut https_probes = ProbeSet::new(ProbeProto::Https); - let mut icmp_v4_probes = ProbeSet::new(ProbeProto::IcmpV4); - let mut icmp_v6_probes = ProbeSet::new(ProbeProto::IcmpV6); - let start = *max_stun_delay.get_or_insert_with(|| plan.max_delay()); - for attempt in 0..attempts { - let delay = start - + (retransmit_delay * attempt as u32) - + (ACTIVE_RETRANSMIT_EXTRA_DELAY * (attempt as u32 + 1)); - https_probes - .push(Probe::Https { - delay, - node: relay_node.clone(), - }) - .expect("Pushing Https Probe to an Https ProbeSet"); - if do4 { - icmp_v4_probes - .push(Probe::IcmpV4 { - delay, - node: relay_node.clone(), - }) - .expect("Pushing IcmpV4 Probe to an Icmp ProbeSet"); - } - if do6 { - icmp_v6_probes - .push(Probe::IcmpV6 { - delay, - node: relay_node.clone(), - }) - .expect("Pusying IcmpV6 Probe to an IcmpV6 ProbeSet"); - } - } - plan.add(https_probes); - plan.add(icmp_v4_probes); - plan.add(icmp_v6_probes); - } - plan - } - - /// Returns an iterator over the [`ProbeSet`]s in this plan. - pub(super) fn iter(&self) -> impl Iterator { - self.0.iter() - } - - /// Adds a [`ProbeSet`] if it contains probes. - fn add(&mut self, set: ProbeSet) { - if !set.is_empty() { - self.0.insert(set); - } - } - - /// Returns the delay of the last probe in the probe plan. - fn max_delay(&self) -> Duration { - self.0 - .iter() - .flatten() - .map(|probe| probe.delay()) - .max() - .unwrap_or_default() - } -} - -impl fmt::Display for ProbePlan { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - writeln!(f, "ProbePlan {{")?; - for probe_set in self.0.iter() { - writeln!(f, r#" ProbeSet("{}") {{"#, probe_set.proto)?; - for probe in probe_set.probes.iter() { - writeln!(f, " {probe},")?; - } - writeln!(f, " }}")?; - } - writeln!(f, "}}") - } -} - -impl FromIterator for ProbePlan { - fn from_iter>(iter: T) -> Self { - Self(iter.into_iter().collect()) - } -} - -/// Sorts the nodes in the [`RelayMap`] from fastest to slowest. -/// -/// This uses the latencies from the last report to determine the order. Relay Nodes with no -/// data are at the end. -fn sort_relays<'a>( - relay_map: &'a RelayMap, - last_report: &Report, -) -> Vec<(&'a RelayUrl, &'a Arc)> { - let mut prev: Vec<_> = relay_map.nodes().collect(); - prev.sort_by(|a, b| { - let latencies_a = last_report.relay_latency.get(&a.url); - let latencies_b = last_report.relay_latency.get(&b.url); - match (latencies_a, latencies_b) { - (Some(_), None) => { - // Non-zero sorts before zero. - std::cmp::Ordering::Less - } - (None, Some(_)) => { - // Zero can't sort before anything else. - std::cmp::Ordering::Greater - } - (None, None) => { - // For both empty latencies sort by relay_id. - a.url.cmp(&b.url) - } - (Some(_), Some(_)) => match latencies_a.cmp(&latencies_b) { - std::cmp::Ordering::Equal => a.url.cmp(&b.url), - x => x, - }, - } - }); - - prev.into_iter().map(|n| (&n.url, n)).collect() -} - -#[cfg(test)] -mod tests { - use pretty_assertions::assert_eq; - - use super::*; - use crate::{test_utils, RelayLatencies}; - - /// Shorthand which declares a new ProbeSet. - /// - /// `$kind`: The `ProbeProto`. - /// `$node`: Expression which will be an `Arc`. - /// `$delays`: A `Vec` of the delays for this probe. - macro_rules! probeset { - (proto: ProbeProto::$kind:ident, relay: $node:expr, delays: $delays:expr,) => { - ProbeSet { - proto: ProbeProto::$kind, - probes: $delays - .iter() - .map(|delay| Probe::$kind { - delay: *delay, - node: $node, - }) - .collect(), - } - }; - } - - #[tokio::test] - async fn test_initial_probeplan() { - let (_servers, relay_map) = test_utils::relay_map(2).await; - let relay_node_1 = relay_map.nodes().next().unwrap(); - let relay_node_2 = relay_map.nodes().nth(1).unwrap(); - let if_state = interfaces::State::fake(); - let plan = ProbePlan::initial(&relay_map, &if_state); - - let expected_plan: ProbePlan = [ - probeset! { - proto: ProbeProto::StunIpv4, - relay: relay_node_1.clone(), - delays: [Duration::ZERO, - Duration::from_millis(100), - Duration::from_millis(200)], - }, - probeset! { - proto: ProbeProto::StunIpv6, - relay: relay_node_1.clone(), - delays: [Duration::ZERO, - Duration::from_millis(100), - Duration::from_millis(200)], - }, - probeset! { - proto: ProbeProto::Https, - relay: relay_node_1.clone(), - delays: [Duration::from_millis(300), - Duration::from_millis(400), - Duration::from_millis(500)], - }, - probeset! { - proto: ProbeProto::IcmpV4, - relay: relay_node_1.clone(), - delays: [Duration::from_millis(300), - Duration::from_millis(400), - Duration::from_millis(500)], - }, - probeset! { - proto: ProbeProto::IcmpV6, - relay: relay_node_1.clone(), - delays: [Duration::from_millis(300), - Duration::from_millis(400), - Duration::from_millis(500)], - }, - probeset! { - proto: ProbeProto::StunIpv4, - relay: relay_node_2.clone(), - delays: [Duration::ZERO, - Duration::from_millis(100), - Duration::from_millis(200)], - }, - probeset! { - proto: ProbeProto::StunIpv6, - relay: relay_node_2.clone(), - delays: [Duration::ZERO, - Duration::from_millis(100), - Duration::from_millis(200)], - }, - probeset! { - proto: ProbeProto::Https, - relay: relay_node_2.clone(), - delays: [Duration::from_millis(300), - Duration::from_millis(400), - Duration::from_millis(500)], - }, - probeset! { - proto: ProbeProto::IcmpV4, - relay: relay_node_2.clone(), - delays: [Duration::from_millis(300), - Duration::from_millis(400), - Duration::from_millis(500)], - }, - probeset! { - proto: ProbeProto::IcmpV6, - relay: relay_node_2.clone(), - delays: [Duration::from_millis(300), - Duration::from_millis(400), - Duration::from_millis(500)], - }, - ] - .into_iter() - .collect(); - - println!("expected:"); - println!("{expected_plan}"); - println!("actual:"); - println!("{plan}"); - // The readable error: - assert_eq!(plan.to_string(), expected_plan.to_string()); - // Just in case there's a bug in the Display impl: - assert_eq!(plan, expected_plan); - } - - #[tokio::test] - async fn test_plan_with_report() { - let _logging = iroh_test::logging::setup(); - let (_servers, relay_map) = test_utils::relay_map(2).await; - let relay_node_1 = relay_map.nodes().next().unwrap().clone(); - let relay_node_2 = relay_map.nodes().nth(1).unwrap().clone(); - let if_state = interfaces::State::fake(); - - for i in 0..10 { - println!("round {}", i); - let mut latencies = RelayLatencies::new(); - latencies.update_relay(relay_node_1.url.clone(), Duration::from_millis(2)); - latencies.update_relay(relay_node_2.url.clone(), Duration::from_millis(2)); - let last_report = Report { - udp: true, - ipv6: true, - ipv4: true, - ipv6_can_send: true, - ipv4_can_send: true, - os_has_ipv6: true, - icmpv4: None, - icmpv6: None, - mapping_varies_by_dest_ip: Some(false), - mapping_varies_by_dest_ipv6: Some(false), - hair_pinning: Some(true), - portmap_probe: None, - preferred_relay: Some(relay_node_1.url.clone()), - relay_latency: latencies.clone(), - relay_v4_latency: latencies.clone(), - relay_v6_latency: latencies.clone(), - global_v4: None, - global_v6: None, - captive_portal: None, - }; - let plan = ProbePlan::with_last_report(&relay_map, &if_state, &last_report); - let expected_plan: ProbePlan = [ - probeset! { - proto: ProbeProto::StunIpv4, - relay: relay_node_1.clone(), - delays: [Duration::ZERO, - Duration::from_micros(52_400), - Duration::from_micros(104_800), - Duration::from_micros(157_200)], - }, - probeset! { - proto: ProbeProto::StunIpv6, - relay: relay_node_1.clone(), - delays: [Duration::ZERO, - Duration::from_micros(52_400), - Duration::from_micros(104_800), - Duration::from_micros(157_200)], - }, - probeset! { - proto: ProbeProto::Https, - relay: relay_node_1.clone(), - delays: [Duration::from_micros(207_200), - Duration::from_micros(259_600), - Duration::from_micros(312_000), - Duration::from_micros(364_400)], - }, - probeset! { - proto: ProbeProto::IcmpV4, - relay: relay_node_1.clone(), - delays: [Duration::from_micros(207_200), - Duration::from_micros(259_600), - Duration::from_micros(312_000), - Duration::from_micros(364_400)], - }, - probeset! { - proto: ProbeProto::IcmpV6, - relay: relay_node_1.clone(), - delays: [Duration::from_micros(207_200), - Duration::from_micros(259_600), - Duration::from_micros(312_000), - Duration::from_micros(364_400)], - }, - probeset! { - proto: ProbeProto::StunIpv4, - relay: relay_node_2.clone(), - delays: [Duration::ZERO, - Duration::from_micros(52_400)], - }, - probeset! { - proto: ProbeProto::StunIpv6, - relay: relay_node_2.clone(), - delays: [Duration::ZERO, - Duration::from_micros(52_400)], - }, - probeset! { - proto: ProbeProto::Https, - relay: relay_node_2.clone(), - delays: [Duration::from_micros(207_200), - Duration::from_micros(259_600)], - }, - probeset! { - proto: ProbeProto::IcmpV4, - relay: relay_node_2.clone(), - delays: [Duration::from_micros(207_200), - Duration::from_micros(259_600)], - }, - probeset! { - proto: ProbeProto::IcmpV6, - relay: relay_node_2.clone(), - delays: [Duration::from_micros(207_200), - Duration::from_micros(259_600)], - }, - ] - .into_iter() - .collect(); - - println!("{} round", i); - println!("expected:"); - println!("{expected_plan}"); - println!("actual:"); - println!("{plan}"); - // The readable error: - assert_eq!(plan.to_string(), expected_plan.to_string(), "{}", i); - // Just in case there's a bug in the Display impl: - assert_eq!(plan, expected_plan, "{}", i); - } - } - - fn create_last_report( - url_1: &RelayUrl, - latency_1: Option, - url_2: &RelayUrl, - latency_2: Option, - ) -> Report { - let mut latencies = RelayLatencies::new(); - if let Some(latency_1) = latency_1 { - latencies.update_relay(url_1.clone(), latency_1); - } - if let Some(latency_2) = latency_2 { - latencies.update_relay(url_2.clone(), latency_2); - } - Report { - udp: true, - ipv6: true, - ipv4: true, - ipv6_can_send: true, - ipv4_can_send: true, - os_has_ipv6: true, - icmpv4: None, - icmpv6: None, - mapping_varies_by_dest_ip: Some(false), - mapping_varies_by_dest_ipv6: Some(false), - hair_pinning: Some(true), - portmap_probe: None, - preferred_relay: Some(url_1.clone()), - relay_latency: latencies.clone(), - relay_v4_latency: latencies.clone(), - relay_v6_latency: latencies.clone(), - global_v4: None, - global_v6: None, - captive_portal: None, - } - } - - #[tokio::test] - async fn test_relay_sort_two_latencies() { - let _logging = iroh_test::logging::setup(); - let (_servers, relay_map) = test_utils::relay_map(2).await; - let r1 = relay_map.nodes().next().unwrap(); - let r2 = relay_map.nodes().nth(1).unwrap(); - let last_report = create_last_report( - &r1.url, - Some(Duration::from_millis(1)), - &r2.url, - Some(Duration::from_millis(2)), - ); - let sorted: Vec<_> = sort_relays(&relay_map, &last_report) - .iter() - .map(|(url, _reg)| *url) - .collect(); - assert_eq!(sorted, vec![&r1.url, &r2.url]); - } - - #[tokio::test] - async fn test_relay_sort_equal_latencies() { - let _logging = iroh_test::logging::setup(); - let (_servers, relay_map) = test_utils::relay_map(2).await; - let r1 = relay_map.nodes().next().unwrap(); - let r2 = relay_map.nodes().nth(1).unwrap(); - let last_report = create_last_report( - &r1.url, - Some(Duration::from_millis(2)), - &r2.url, - Some(Duration::from_millis(2)), - ); - let sorted: Vec<_> = sort_relays(&relay_map, &last_report) - .iter() - .map(|(url, _)| *url) - .collect(); - assert_eq!(sorted, vec![&r1.url, &r2.url]); - } - - #[tokio::test] - async fn test_relay_sort_missing_latency() { - let (_servers, relay_map) = test_utils::relay_map(2).await; - let r1 = relay_map.nodes().next().unwrap(); - let r2 = relay_map.nodes().nth(1).unwrap(); - - let last_report = - create_last_report(&r1.url, None, &r2.url, Some(Duration::from_millis(2))); - let sorted: Vec<_> = sort_relays(&relay_map, &last_report) - .iter() - .map(|(url, _)| *url) - .collect(); - assert_eq!(sorted, vec![&r2.url, &r1.url]); - - let last_report = - create_last_report(&r1.url, Some(Duration::from_millis(2)), &r2.url, None); - let sorted: Vec<_> = sort_relays(&relay_map, &last_report) - .iter() - .map(|(url, _)| *url) - .collect(); - assert_eq!(sorted, vec![&r1.url, &r2.url]); - } - - #[tokio::test] - async fn test_relay_sort_no_latency() { - let _logging = iroh_test::logging::setup(); - let (_servers, relay_map) = test_utils::relay_map(2).await; - let r1 = relay_map.nodes().next().unwrap(); - let r2 = relay_map.nodes().nth(1).unwrap(); - - let last_report = create_last_report(&r1.url, None, &r2.url, None); - let sorted: Vec<_> = sort_relays(&relay_map, &last_report) - .iter() - .map(|(url, _)| *url) - .collect(); - // sorted by relay url only - assert_eq!(sorted, vec![&r1.url, &r2.url]); - } -} diff --git a/reportgen.rs b/reportgen.rs deleted file mode 100644 index 11a8328ff29..00000000000 --- a/reportgen.rs +++ /dev/null @@ -1,1597 +0,0 @@ -//! The reportgen actor is responsible for generating a single net_report report. -//! -//! It is implemented as an actor with [`Client`] as handle. -//! -//! The actor starts generating the report as soon as it is created, it does not receive any -//! messages from the client. It follows roughly these steps: -//! -//! - Determines host IPv6 support. -//! - Creates hairpin actor. -//! - Creates portmapper future. -//! - Creates captive portal detection future. -//! - Creates Probe Set futures. -//! - These send messages to the reportgen actor. -//! - Loops driving the futures and handling actor messages: -//! - Disables futures as they are completed or aborted. -//! - Stop if there are no outstanding tasks/futures, or on timeout. -//! - Sends the completed report to the net_report actor. - -use std::{ - future::Future, - net::{IpAddr, SocketAddr}, - pin::Pin, - sync::Arc, - task::{Context, Poll}, - time::Duration, -}; - -use anyhow::{anyhow, bail, Context as _, Result}; -use hickory_resolver::TokioAsyncResolver as DnsResolver; -#[cfg(feature = "metrics")] -use iroh_metrics::inc; -use iroh_relay::{http::RELAY_PROBE_PATH, protos::stun}; -use netwatch::{interfaces, UdpSocket}; -use rand::seq::IteratorRandom; -use tokio::{ - sync::{mpsc, oneshot}, - task::JoinSet, - time::{self, Instant}, -}; -use tokio_util::task::AbortOnDropHandle; -use tracing::{debug, debug_span, error, info_span, trace, warn, Instrument, Span}; -use url::Host; - -#[cfg(feature = "metrics")] -use crate::Metrics; -use crate::{ - self as net_report, - defaults::{DEFAULT_QUIC_PORT, DEFAULT_STUN_PORT}, - dns::ResolverExt, - ping::{PingError, Pinger}, - RelayMap, RelayNode, RelayUrl, Report, -}; - -mod hairpin; -mod probes; - -use probes::{Probe, ProbePlan, ProbeProto}; - -use crate::defaults::timeouts::{ - CAPTIVE_PORTAL_DELAY, CAPTIVE_PORTAL_TIMEOUT, OVERALL_REPORT_TIMEOUT, PROBES_TIMEOUT, -}; - -const ENOUGH_NODES: usize = 3; - -/// Holds the state for a single invocation of [`net_report::Client::get_report`]. -/// -/// Dropping this will cancel the actor and stop the report generation. -#[derive(Debug)] -pub(super) struct Client { - // Addr is currently only used by child actors, so not yet exposed here. - _drop_guard: AbortOnDropHandle<()>, -} - -impl Client { - /// Creates a new actor generating a single report. - /// - /// The actor starts running immediately and only generates a single report, after which - /// it shuts down. Dropping this handle will abort the actor. - pub(super) fn new( - net_report: net_report::Addr, - last_report: Option>, - port_mapper: Option, - relay_map: RelayMap, - stun_sock4: Option>, - stun_sock6: Option>, - quic_addr_disc: Option, - dns_resolver: DnsResolver, - ) -> Self { - let (msg_tx, msg_rx) = mpsc::channel(32); - let addr = Addr { - sender: msg_tx.clone(), - }; - let mut actor = Actor { - msg_tx, - msg_rx, - net_report: net_report.clone(), - last_report, - port_mapper, - relay_map, - stun_sock4, - stun_sock6, - quic_addr_disc, - report: Report::default(), - hairpin_actor: hairpin::Client::new(net_report, addr), - outstanding_tasks: OutstandingTasks::default(), - dns_resolver, - }; - let task = tokio::spawn( - async move { actor.run().await }.instrument(info_span!("reportgen.actor")), - ); - Self { - _drop_guard: AbortOnDropHandle::new(task), - } - } -} - -/// The address of the reportstate [`Actor`]. -/// -/// Unlike the [`Client`] struct itself this is the raw channel to send message over. -/// Keeping this alive will not keep the actor alive, which makes this handy to pass to -/// internal tasks. -#[derive(Debug, Clone)] -pub(super) struct Addr { - sender: mpsc::Sender, -} - -impl Addr { - /// Blocking send to the actor, to be used from a non-actor future. - async fn send(&self, msg: Message) -> Result<(), mpsc::error::SendError> { - trace!( - "sending {:?} to channel with cap {}", - msg, - self.sender.capacity() - ); - self.sender.send(msg).await - } -} - -/// Messages to send to the reportstate [`Actor`]. -#[derive(Debug)] -enum Message { - /// Set the hairpinning availability in the report. - HairpinResult(bool), - /// Check whether executing a probe would still help. - // TODO: Ideally we remove the need for this message and the logic is inverted: once we - // get a probe result we cancel all probes that are no longer needed. But for now it's - // this way around to ease conversion. - ProbeWouldHelp(Probe, Arc, oneshot::Sender), - /// Abort all remaining probes. - AbortProbes, -} - -/// The reportstate actor. -/// -/// This actor starts, generates a single report and exits. -#[derive(Debug)] -struct Actor { - /// The sender of the message channel, so we can give out [`Addr`]. - msg_tx: mpsc::Sender, - /// The receiver of the message channel. - msg_rx: mpsc::Receiver, - /// The address of the net_report actor. - net_report: super::Addr, - - // Provided state - /// The previous report, if it exists. - last_report: Option>, - /// The portmapper client, if there is one. - port_mapper: Option, - /// The relay configuration. - relay_map: RelayMap, - /// Socket to send IPv4 STUN requests from. - stun_sock4: Option>, - /// Socket so send IPv6 STUN requests from. - stun_sock6: Option>, - /// QUIC configuration to do QUIC address Discovery - quic_addr_disc: Option, - - // Internal state. - /// The report being built. - report: Report, - /// The hairpin actor. - hairpin_actor: hairpin::Client, - /// Which tasks the [`Actor`] is still waiting on. - /// - /// This is essentially the summary of all the work the [`Actor`] is doing. - outstanding_tasks: OutstandingTasks, - /// The DNS resolver to use for probes that need to resolve DNS records. - dns_resolver: DnsResolver, -} - -impl Actor { - fn addr(&self) -> Addr { - Addr { - sender: self.msg_tx.clone(), - } - } - - async fn run(&mut self) { - match self.run_inner().await { - Ok(_) => debug!("reportgen actor finished"), - Err(err) => { - self.net_report - .send(net_report::Message::ReportAborted { err }) - .await - .ok(); - } - } - } - - /// Runs the main reportgen actor logic. - /// - /// This actor runs by: - /// - /// - Creates a hairpin actor. - /// - Creates a captive portal future. - /// - Creates ProbeSet futures in a group of futures. - /// - Runs a main loop: - /// - Drives all the above futures. - /// - Receives actor messages (sent by those futures). - /// - Updates the report, cancels unneeded futures. - /// - Sends the report to the net_report actor. - async fn run_inner(&mut self) -> Result<()> { - debug!( - port_mapper = %self.port_mapper.is_some(), - "reportstate actor starting", - ); - - self.report.os_has_ipv6 = super::os_has_ipv6(); - - let mut port_mapping = self.prepare_portmapper_task(); - let mut captive_task = self.prepare_captive_portal_task(); - let mut probes = self.spawn_probes_task().await?; - - let total_timer = tokio::time::sleep(OVERALL_REPORT_TIMEOUT); - tokio::pin!(total_timer); - let probe_timer = tokio::time::sleep(PROBES_TIMEOUT); - tokio::pin!(probe_timer); - - loop { - trace!(awaiting = ?self.outstanding_tasks, "tick; awaiting tasks"); - if self.outstanding_tasks.all_done() { - debug!("all tasks done"); - break; - } - tokio::select! { - biased; - _ = &mut total_timer => { - trace!("tick: total_timer expired"); - bail!("report timed out"); - } - - _ = &mut probe_timer => { - warn!("tick: probes timed out"); - // Set new timeout to not go into this branch multiple times. We need - // the abort to finish all probes normally. PROBES_TIMEOUT is - // sufficiently far in the future. - probe_timer.as_mut().reset(Instant::now() + PROBES_TIMEOUT); - probes.abort_all(); - self.handle_abort_probes(); - } - - // Drive the portmapper. - pm = &mut port_mapping, if self.outstanding_tasks.port_mapper => { - debug!(report=?pm, "tick: portmapper probe report"); - self.report.portmap_probe = pm; - port_mapping.inner = None; - self.outstanding_tasks.port_mapper = false; - } - - // Check for probes finishing. - set_result = probes.join_next(), if self.outstanding_tasks.probes => { - trace!("tick: probes done: {:?}", set_result); - match set_result { - Some(Ok(Ok(report))) => self.handle_probe_report(report), - Some(Ok(Err(_))) => (), - Some(Err(e)) => { - warn!("probes task error: {:?}", e); - } - None => { - self.handle_abort_probes(); - } - } - trace!("tick: probes handled"); - } - - // Drive the captive task. - found = &mut captive_task, if self.outstanding_tasks.captive_task => { - trace!("tick: captive portal task done"); - self.report.captive_portal = found; - captive_task.inner = None; - self.outstanding_tasks.captive_task = false; - } - - // Handle actor messages. - msg = self.msg_rx.recv() => { - trace!("tick: msg recv: {:?}", msg); - match msg { - Some(msg) => self.handle_message(msg), - None => bail!("msg_rx closed, reportgen client must be dropped"), - } - } - } - } - - if !probes.is_empty() { - debug!( - "aborting {} probe sets, already have enough reports", - probes.len() - ); - drop(probes); - } - - debug!("Sending report to net_report actor"); - self.net_report - .send(net_report::Message::ReportReady { - report: Box::new(self.report.clone()), - }) - .await?; - - Ok(()) - } - - /// Handles an actor message. - /// - /// Returns `true` if all the probes need to be aborted. - fn handle_message(&mut self, msg: Message) { - trace!(?msg, "handling message"); - match msg { - Message::HairpinResult(works) => { - self.report.hair_pinning = Some(works); - self.outstanding_tasks.hairpin = false; - } - Message::ProbeWouldHelp(probe, relay_node, response_tx) => { - let res = self.probe_would_help(probe, relay_node); - if response_tx.send(res).is_err() { - debug!("probe dropped before ProbeWouldHelp response sent"); - } - } - Message::AbortProbes => { - self.handle_abort_probes(); - } - } - } - - fn handle_probe_report(&mut self, probe_report: ProbeReport) { - debug!(?probe_report, "finished probe"); - update_report(&mut self.report, probe_report); - - // When we discover the first IPv4 address we want to start the hairpin actor. - if let Some(ref addr) = self.report.global_v4 { - if !self.hairpin_actor.has_started() { - self.hairpin_actor.start_check(*addr); - self.outstanding_tasks.hairpin = true; - } - } - - // Once we've heard from enough relay servers (3), start a timer to give up on the other - // probes. The timer's duration is a function of whether this is our initial full - // probe or an incremental one. For incremental ones, wait for the duration of the - // slowest relay. For initial ones, double that. - let enough_relays = std::cmp::min(self.relay_map.len(), ENOUGH_NODES); - if self.report.relay_latency.len() == enough_relays { - let timeout = self.report.relay_latency.max_latency(); - let timeout = match self.last_report.is_some() { - true => timeout, - false => timeout * 2, - }; - let reportcheck = self.addr(); - debug!( - reports=self.report.relay_latency.len(), - delay=?timeout, - "Have enough probe reports, aborting further probes soon", - ); - tokio::spawn( - async move { - time::sleep(timeout).await; - // Because we do this after a timeout it is entirely normal that the - // actor is no longer there by the time we send this message. - reportcheck - .send(Message::AbortProbes) - .await - .map_err(|err| trace!("Failed to abort all probes: {err:#}")) - .ok(); - } - .instrument(Span::current()), - ); - } - } - - /// Whether running this probe would still improve our report. - fn probe_would_help(&mut self, probe: Probe, relay_node: Arc) -> bool { - // If the probe is for a relay we don't yet know about, that would help. - if self.report.relay_latency.get(&relay_node.url).is_none() { - return true; - } - - // If the probe is for IPv6 and we don't yet have an IPv6 report, that would help. - if probe.proto() == ProbeProto::StunIpv6 && self.report.relay_v6_latency.is_empty() { - return true; - } - - // For IPv4, we need at least two IPv4 results overall to - // determine whether we're behind a NAT that shows us as - // different source IPs and/or ports depending on who we're - // talking to. If we don't yet have two results yet - // (`mapping_varies_by_dest_ip` is blank), then another IPv4 probe - // would be good. - if probe.proto() == ProbeProto::StunIpv4 && self.report.mapping_varies_by_dest_ip.is_none() - { - return true; - } - - // Otherwise not interesting. - false - } - - /// Stops further probes. - /// - /// This makes sure that no further probes are run and also cancels the captive portal - /// and portmapper tasks if there were successful probes. Be sure to only handle this - /// after all the required [`ProbeReport`]s have been processed. - fn handle_abort_probes(&mut self) { - trace!("handle abort probes"); - self.outstanding_tasks.probes = false; - if self.report.udp { - self.outstanding_tasks.port_mapper = false; - self.outstanding_tasks.captive_task = false; - } - } - - /// Creates the future which will perform the portmapper task. - /// - /// The returned future will run the portmapper, if enabled, resolving to it's result. - fn prepare_portmapper_task( - &mut self, - ) -> MaybeFuture>>>> { - let mut port_mapping = MaybeFuture::default(); - if let Some(port_mapper) = self.port_mapper.clone() { - port_mapping.inner = Some(Box::pin(async move { - match port_mapper.probe().await { - Ok(Ok(res)) => Some(res), - Ok(Err(err)) => { - debug!("skipping port mapping: {err:?}"); - None - } - Err(recv_err) => { - warn!("skipping port mapping: {recv_err:?}"); - None - } - } - })); - self.outstanding_tasks.port_mapper = true; - } - port_mapping - } - - /// Creates the future which will perform the captive portal check. - fn prepare_captive_portal_task( - &mut self, - ) -> MaybeFuture>>>> { - // If we're doing a full probe, also check for a captive portal. We - // delay by a bit to wait for UDP STUN to finish, to avoid the probe if - // it's unnecessary. - if self.last_report.is_none() { - // Even if we're doing a non-incremental update, we may want to try our - // preferred relay for captive portal detection. - let preferred_relay = self - .last_report - .as_ref() - .and_then(|l| l.preferred_relay.clone()); - - let dns_resolver = self.dns_resolver.clone(); - let dm = self.relay_map.clone(); - self.outstanding_tasks.captive_task = true; - MaybeFuture { - inner: Some(Box::pin(async move { - tokio::time::sleep(CAPTIVE_PORTAL_DELAY).await; - debug!("Captive portal check started after {CAPTIVE_PORTAL_DELAY:?}"); - let captive_portal_check = tokio::time::timeout( - CAPTIVE_PORTAL_TIMEOUT, - check_captive_portal(&dns_resolver, &dm, preferred_relay) - .instrument(debug_span!("captive-portal")), - ); - match captive_portal_check.await { - Ok(Ok(found)) => Some(found), - Ok(Err(err)) => { - let err: Result = err.downcast(); - match err { - Ok(req_err) if req_err.is_connect() => { - debug!("check_captive_portal failed: {req_err:#}"); - } - Ok(req_err) => warn!("check_captive_portal error: {req_err:#}"), - Err(any_err) => warn!("check_captive_portal error: {any_err:#}"), - } - None - } - Err(_) => { - warn!("check_captive_portal timed out"); - None - } - } - })), - } - } else { - self.outstanding_tasks.captive_task = false; - MaybeFuture::default() - } - } - - /// Prepares the future which will run all the probes as per generated ProbePlan. - /// - /// Probes operate like the following: - /// - /// - A future is created for each probe in all probe sets. - /// - All probes in a set are grouped in [`JoinSet`]. - /// - All those probe sets are grouped in one overall [`JoinSet`]. - /// - This future is polled by the main actor loop to make progress. - /// - Once a probe future is polled: - /// - Many probes start with a delay, they sleep during this time. - /// - When a probe starts it first asks the reportgen [`Actor`] if it is still useful - /// to run. If not it aborts the entire probe set. - /// - When a probe finishes, its [`ProbeReport`] is yielded to the reportgen actor. - /// - Probes get aborted in several ways: - /// - A running it can fail and abort the entire probe set if it deems the - /// failure permanent. Probes in a probe set are essentially retries. - /// - Once there are [`ProbeReport`]s from enough nodes, all remaining probes are - /// aborted. That is, the main actor loop stops polling them. - async fn spawn_probes_task(&mut self) -> Result>> { - let if_state = interfaces::State::new().await; - debug!(%if_state, "Local interfaces"); - let plan = match self.last_report { - Some(ref report) => ProbePlan::with_last_report(&self.relay_map, &if_state, report), - None => ProbePlan::initial(&self.relay_map, &if_state), - }; - trace!(%plan, "probe plan"); - - // The pinger is created here so that any sockets that might be bound for it are - // shared between the probes that use it. It binds sockets lazily, so we can always - // create it. - let pinger = Pinger::new(); - - // A collection of futures running probe sets. - let mut probes = JoinSet::default(); - for probe_set in plan.iter() { - let mut set = JoinSet::default(); - for probe in probe_set { - let reportstate = self.addr(); - let stun_sock4 = self.stun_sock4.clone(); - let stun_sock6 = self.stun_sock6.clone(); - let quic_addr_disc = self.quic_addr_disc.clone(); - let relay_node = probe.node().clone(); - let probe = probe.clone(); - let net_report = self.net_report.clone(); - let pinger = pinger.clone(); - let dns_resolver = self.dns_resolver.clone(); - - set.spawn( - run_probe( - reportstate, - stun_sock4, - stun_sock6, - quic_addr_disc, - relay_node, - probe.clone(), - net_report, - pinger, - dns_resolver, - ) - .instrument(debug_span!("run_probe", %probe)), - ); - } - - // Add the probe set to all futures of probe sets. Handle aborting a probe set - // if needed, only normal errors means the set continues. - probes.spawn( - async move { - // Hack because ProbeSet is not it's own type yet. - let mut probe_proto = None; - while let Some(res) = set.join_next().await { - match res { - Ok(Ok(report)) => return Ok(report), - Ok(Err(ProbeError::Error(err, probe))) => { - probe_proto = Some(probe.proto()); - warn!(?probe, "probe failed: {:#}", err); - continue; - } - Ok(Err(ProbeError::AbortSet(err, probe))) => { - debug!(?probe, "probe set aborted: {:#}", err); - set.abort_all(); - return Err(err); - } - Err(err) => { - warn!("fatal probe set error, aborting: {:#}", err); - continue; - } - } - } - warn!(?probe_proto, "no successful probes in ProbeSet"); - Err(anyhow!("All probes in ProbeSet failed")) - } - .instrument(info_span!("probe")), - ); - } - self.outstanding_tasks.probes = true; - - Ok(probes) - } -} - -/// Tasks on which the reportgen [`Actor`] is still waiting. -/// -/// There is no particular progression, e.g. hairpin starts `false`, moves to `true` when a -/// check is started and then becomes `false` again once it is finished. -#[derive(Debug, Default)] -struct OutstandingTasks { - probes: bool, - port_mapper: bool, - captive_task: bool, - hairpin: bool, -} - -impl OutstandingTasks { - fn all_done(&self) -> bool { - !(self.probes || self.port_mapper || self.captive_task || self.hairpin) - } -} - -/// The success result of [`run_probe`]. -#[derive(Debug, Clone)] -struct ProbeReport { - /// Whether we can send IPv4 UDP packets. - ipv4_can_send: bool, - /// Whether we can send IPv6 UDP packets. - ipv6_can_send: bool, - /// Whether we can send ICMPv4 packets, `None` if not checked. - icmpv4: Option, - /// Whether we can send ICMPv6 packets, `None` if not checked. - icmpv6: Option, - /// The latency to the relay node. - latency: Option, - /// The probe that generated this report. - probe: Probe, - /// The discovered public address. - addr: Option, -} - -impl ProbeReport { - fn new(probe: Probe) -> Self { - ProbeReport { - probe, - ipv4_can_send: false, - ipv6_can_send: false, - icmpv4: None, - icmpv6: None, - latency: None, - addr: None, - } - } -} - -/// Errors for [`run_probe`]. -/// -/// The main purpose is to signal whether other probes in this probe set should still be -/// run. Recall that a probe set is normally a set of identical probes with delays, -/// effectively creating retries, and the first successful probe of a probe set will cancel -/// the others in the set. So this allows an unsuccessful probe to cancel the remainder of -/// the set or not. -#[derive(Debug)] -enum ProbeError { - /// Abort the current set. - AbortSet(anyhow::Error, Probe), - /// Continue the other probes in the set. - Error(anyhow::Error, Probe), -} - -/// Pieces needed to do QUIC address discovery. -#[derive(Debug, Clone)] -pub struct QuicAddressDiscovery { - /// A QUIC Endpoint - pub ep: quinn::Endpoint, - /// A client config. - pub client_config: quinn::ClientConfig, -} - -/// Executes a particular [`Probe`], including using a delayed start if needed. -/// -/// If *stun_sock4* and *stun_sock6* are `None` the STUN probes are disabled. -#[allow(clippy::too_many_arguments)] -async fn run_probe( - reportstate: Addr, - stun_sock4: Option>, - stun_sock6: Option>, - quic_addr_disc: Option, - relay_node: Arc, - probe: Probe, - net_report: net_report::Addr, - pinger: Pinger, - dns_resolver: DnsResolver, -) -> Result { - if !probe.delay().is_zero() { - trace!("delaying probe"); - tokio::time::sleep(probe.delay()).await; - } - debug!("starting probe"); - - let (would_help_tx, would_help_rx) = oneshot::channel(); - if let Err(err) = reportstate - .send(Message::ProbeWouldHelp( - probe.clone(), - relay_node.clone(), - would_help_tx, - )) - .await - { - // this happens on shutdown or if the report is already finished - debug!("Failed to check if probe would help: {err:#}"); - return Err(ProbeError::AbortSet(err.into(), probe.clone())); - } - - if !would_help_rx.await.map_err(|_| { - ProbeError::AbortSet( - anyhow!("ReportCheck actor dropped sender while waiting for ProbeWouldHelp response"), - probe.clone(), - ) - })? { - return Err(ProbeError::AbortSet( - anyhow!("ReportCheck says probe set no longer useful"), - probe, - )); - } - - let relay_addr = get_relay_addr(&dns_resolver, &relay_node, probe.proto()) - .await - .context("no relay node addr") - .map_err(|e| ProbeError::AbortSet(e, probe.clone()))?; - - let mut result = ProbeReport::new(probe.clone()); - match probe { - Probe::StunIpv4 { .. } | Probe::StunIpv6 { .. } => { - let maybe_sock = if matches!(probe, Probe::StunIpv4 { .. }) { - stun_sock4.as_ref() - } else { - stun_sock6.as_ref() - }; - match maybe_sock { - Some(sock) => { - result = run_stun_probe(sock, relay_addr, net_report, probe).await?; - } - None => { - return Err(ProbeError::AbortSet( - anyhow!("No socket for {}, aborting probeset", probe.proto()), - probe.clone(), - )); - } - } - } - Probe::IcmpV4 { .. } | Probe::IcmpV6 { .. } => { - result = run_icmp_probe(probe, relay_addr, pinger).await? - } - Probe::Https { ref node, .. } => { - debug!("sending probe HTTPS"); - match measure_https_latency(&dns_resolver, node, None).await { - Ok((latency, ip)) => { - result.latency = Some(latency); - // We set these IPv4 and IPv6 but they're not really used - // and we don't necessarily set them both. If UDP is blocked - // and both IPv4 and IPv6 are available over TCP, it's basically - // random which fields end up getting set here. - // Since they're not needed, that's fine for now. - match ip { - IpAddr::V4(_) => result.ipv4_can_send = true, - IpAddr::V6(_) => result.ipv6_can_send = true, - } - } - Err(err) => { - warn!("https latency measurement failed: {:?}", err); - } - } - } - Probe::QuicAddrIpv4 { ref node, .. } | Probe::QuicAddrIpv6 { ref node, .. } => { - debug!("sending QUIC address discovery prob"); - let url = node.url.clone(); - match quic_addr_disc { - Some(quic_addr_disc) => { - result = run_quic_probe(quic_addr_disc, url, relay_addr, probe).await? - } - None => { - return Err(ProbeError::AbortSet( - anyhow!("No QUIC endpoint for {}, aborting probeset", probe.proto()), - probe.clone(), - )); - } - } - } - } - - trace!("probe successful"); - Ok(result) -} - -/// Run a QUIC address discovery probe. -// TODO(ramfox): if this probe is aborted, then the connection will never be -// properly, possibly causing errors on the server. -async fn run_quic_probe( - quic_addr_disc: QuicAddressDiscovery, - url: RelayUrl, - relay_addr: SocketAddr, - probe: Probe, -) -> Result { - match probe.proto() { - ProbeProto::QuicAddrIpv4 => debug_assert!(relay_addr.is_ipv4()), - ProbeProto::QuicAddrIpv6 => debug_assert!(relay_addr.is_ipv6()), - _ => debug_assert!(false, "wrong probe"), - } - // TODO(ramfox): what to put here if no host is given? - let host = url.host_str().unwrap_or("localhost"); - let quic_client = - iroh_relay::quic::QuicClient::new(quic_addr_disc.ep, quic_addr_disc.client_config); - let (addr, latency) = quic_client - .get_addr_and_latency(relay_addr, host) - .await - .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; - let mut result = ProbeReport::new(probe.clone()); - if matches!(probe, Probe::QuicAddrIpv4 { .. }) { - result.ipv4_can_send = true; - } else { - result.ipv6_can_send = true; - } - result.addr = Some(addr); - result.latency = Some(latency); - Ok(result) -} - -/// Run a STUN IPv4 or IPv6 probe. -async fn run_stun_probe( - sock: &Arc, - relay_addr: SocketAddr, - net_report: net_report::Addr, - probe: Probe, -) -> Result { - match probe.proto() { - ProbeProto::StunIpv4 => debug_assert!(relay_addr.is_ipv4()), - ProbeProto::StunIpv6 => debug_assert!(relay_addr.is_ipv6()), - _ => debug_assert!(false, "wrong probe"), - } - let txid = stun::TransactionId::default(); - let req = stun::request(txid); - - // Setup net_report to give us back the incoming STUN response. - let (stun_tx, stun_rx) = oneshot::channel(); - let (inflight_ready_tx, inflight_ready_rx) = oneshot::channel(); - net_report - .send(net_report::Message::InFlightStun( - net_report::Inflight { - txn: txid, - start: Instant::now(), - s: stun_tx, - }, - inflight_ready_tx, - )) - .await - .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; - inflight_ready_rx - .await - .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; - - // Send the probe. - match sock.send_to(&req, relay_addr).await { - Ok(n) if n == req.len() => { - debug!(%relay_addr, %txid, "sending {} probe", probe.proto()); - let mut result = ProbeReport::new(probe.clone()); - - if matches!(probe, Probe::StunIpv4 { .. }) { - result.ipv4_can_send = true; - #[cfg(feature = "metrics")] - inc!(Metrics, stun_packets_sent_ipv4); - } else { - result.ipv6_can_send = true; - #[cfg(feature = "metrics")] - inc!(Metrics, stun_packets_sent_ipv6); - } - let (delay, addr) = stun_rx - .await - .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; - result.latency = Some(delay); - result.addr = Some(addr); - Ok(result) - } - Ok(n) => { - let err = anyhow!("Failed to send full STUN request: {}", probe.proto()); - error!(%relay_addr, sent_len=n, req_len=req.len(), "{err:#}"); - Err(ProbeError::Error(err, probe.clone())) - } - Err(err) => { - let kind = err.kind(); - let err = anyhow::Error::new(err) - .context(format!("Failed to send STUN request: {}", probe.proto())); - - // It is entirely normal that we are on a dual-stack machine with no - // routed IPv6 network. So silence that case. - // NetworkUnreachable and HostUnreachable are still experimental (io_error_more - // #86442) but it is already emitted. So hack around this. - match format!("{kind:?}").as_str() { - "NetworkUnreachable" | "HostUnreachable" => { - debug!(%relay_addr, "{err:#}"); - Err(ProbeError::AbortSet(err, probe.clone())) - } - _ => { - // No need to log this, our caller does already log this. - Err(ProbeError::Error(err, probe.clone())) - } - } - } - } -} - -/// Reports whether or not we think the system is behind a -/// captive portal, detected by making a request to a URL that we know should -/// return a "204 No Content" response and checking if that's what we get. -/// -/// The boolean return is whether we think we have a captive portal. -async fn check_captive_portal( - dns_resolver: &DnsResolver, - dm: &RelayMap, - preferred_relay: Option, -) -> Result { - // If we have a preferred relay node and we can use it for non-STUN requests, try that; - // otherwise, pick a random one suitable for non-STUN requests. - let preferred_relay = preferred_relay.and_then(|url| match dm.get_node(&url) { - Some(node) if node.stun_only => Some(url), - _ => None, - }); - - let url = match preferred_relay { - Some(url) => url, - None => { - let urls: Vec<_> = dm - .nodes() - .filter(|n| !n.stun_only) - .map(|n| n.url.clone()) - .collect(); - if urls.is_empty() { - debug!("No suitable relay node for captive portal check"); - return Ok(false); - } - - let i = (0..urls.len()) - .choose(&mut rand::thread_rng()) - .unwrap_or_default(); - urls[i].clone() - } - }; - - let mut builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); - if let Some(Host::Domain(domain)) = url.host() { - // Use our own resolver rather than getaddrinfo - // - // Be careful, a non-zero port will override the port in the URI. - // - // Ideally we would try to resolve **both** IPv4 and IPv6 rather than purely race - // them. But our resolver doesn't support that yet. - let addrs: Vec<_> = dns_resolver - .lookup_ipv4_ipv6_staggered(domain) - .await? - .map(|ipaddr| SocketAddr::new(ipaddr, 0)) - .collect(); - builder = builder.resolve_to_addrs(domain, &addrs); - } - let client = builder.build()?; - - // Note: the set of valid characters in a challenge and the total - // length is limited; see is_challenge_char in bin/iroh-relay for more - // details. - - let host_name = url.host_str().unwrap_or_default(); - let challenge = format!("ts_{}", host_name); - let portal_url = format!("http://{}/generate_204", host_name); - let res = client - .request(reqwest::Method::GET, portal_url) - .header("X-Tailscale-Challenge", &challenge) - .send() - .await?; - - let expected_response = format!("response {challenge}"); - let is_valid_response = res - .headers() - .get("X-Tailscale-Response") - .map(|s| s.to_str().unwrap_or_default()) - == Some(&expected_response); - - debug!( - "check_captive_portal url={} status_code={} valid_response={}", - res.url(), - res.status(), - is_valid_response, - ); - let has_captive = res.status() != 204 || !is_valid_response; - - Ok(has_captive) -} - -fn get_port(relay_node: &RelayNode, proto: &ProbeProto) -> u16 { - match proto { - ProbeProto::QuicAddrIpv4 | ProbeProto::QuicAddrIpv6 => { - if relay_node.quic_port == 0 { - DEFAULT_QUIC_PORT - } else { - relay_node.quic_port - } - } - _ => { - if relay_node.stun_port == 0 { - DEFAULT_STUN_PORT - } else { - relay_node.stun_port - } - } - } -} - -/// Returns the IP address to use to communicate to this relay node. -/// -/// *proto* specifies the protocol of the probe. Depending on the protocol we may return -/// different results. Obviously IPv4 vs IPv6 but a [`RelayNode`] may also have disabled -/// some protocols. -async fn get_relay_addr( - dns_resolver: &DnsResolver, - relay_node: &RelayNode, - proto: ProbeProto, -) -> Result { - let mut port: u16 = 0; - if relay_node.stun_only && !matches!(proto, ProbeProto::StunIpv4 | ProbeProto::StunIpv6) { - bail!("Relay node not suitable for non-STUN probes"); - - } - if relay_node.quic_only && !matches!(proto, ProbeProto::QuicAddrIpv4 | ProbeProto::QuicAddrIpv6) - { - bail!("Relay node not suitable for non-QUIC address discovery probes"); - } - - match proto { - ProbeProto::StunIpv4 | ProbeProto::IcmpV4 | ProbeProto::QuicAddrIpv4 => { - match relay_node.url.host() { - Some(url::Host::Domain(hostname)) => { - debug!(?proto, %hostname, "Performing DNS A lookup for relay addr"); - match dns_resolver.lookup_ipv4_staggered(hostname).await { - Ok(mut addrs) => addrs - .next() - .map(|ip| ip.to_canonical()) - .map(|addr| SocketAddr::new(addr, port)) - .ok_or(anyhow!("No suitable relay addr found")), - Err(err) => Err(err.context("No suitable relay addr found")), - } - } - Some(url::Host::Ipv4(addr)) => Ok(SocketAddr::new(addr.into(), port)), - Some(url::Host::Ipv6(_addr)) => Err(anyhow!("No suitable relay addr found")), - None => Err(anyhow!("No valid hostname in RelayUrl")), - } - } - - ProbeProto::StunIpv6 | ProbeProto::IcmpV6 | ProbeProto::QuicAddrIpv6 => { - match relay_node.url.host() { - Some(url::Host::Domain(hostname)) => { - debug!(?proto, %hostname, "Performing DNS AAAA lookup for relay addr"); - match dns_resolver.lookup_ipv6_staggered(hostname).await { - Ok(mut addrs) => addrs - .next() - .map(|ip| ip.to_canonical()) - .map(|addr| SocketAddr::new(addr, port)) - .ok_or(anyhow!("No suitable relay addr found")), - Err(err) => Err(err.context("No suitable relay addr found")), - } - } - Some(url::Host::Ipv4(_addr)) => Err(anyhow!("No suitable relay addr found")), - Some(url::Host::Ipv6(addr)) => Ok(SocketAddr::new(addr.into(), port)), - None => Err(anyhow!("No valid hostname in RelayUrl")), - } - } - - ProbeProto::Https => Err(anyhow!("Not implemented")), - } -} - -/// Runs an ICMP IPv4 or IPv6 probe. -/// -/// The `pinger` is passed in so the ping sockets are only bound once -/// for the probe set. -async fn run_icmp_probe( - probe: Probe, - relay_addr: SocketAddr, - pinger: Pinger, -) -> Result { - match probe.proto() { - ProbeProto::IcmpV4 => debug_assert!(relay_addr.is_ipv4()), - ProbeProto::IcmpV6 => debug_assert!(relay_addr.is_ipv6()), - _ => debug_assert!(false, "wrong probe"), - } - const DATA: &[u8; 15] = b"iroh icmp probe"; - debug!(dst = %relay_addr, len = DATA.len(), "ICMP Ping started"); - let latency = pinger - .send(relay_addr.ip(), DATA) - .await - .map_err(|err| match err { - PingError::Client(err) => ProbeError::AbortSet( - anyhow!("Failed to create pinger ({err:#}), aborting probeset"), - probe.clone(), - ), - PingError::Ping(err) => ProbeError::Error(err.into(), probe.clone()), - })?; - debug!(dst = %relay_addr, len = DATA.len(), ?latency, "ICMP ping done"); - let mut report = ProbeReport::new(probe); - report.latency = Some(latency); - match relay_addr { - SocketAddr::V4(_) => { - report.ipv4_can_send = true; - report.icmpv4 = Some(true); - } - SocketAddr::V6(_) => { - report.ipv6_can_send = true; - report.icmpv6 = Some(true); - } - } - Ok(report) -} - -/// Executes an HTTPS probe. -/// -/// If `certs` is provided they will be added to the trusted root certificates, allowing the -/// use of self-signed certificates for servers. Currently this is used for testing. -#[allow(clippy::unused_async)] -async fn measure_https_latency( - dns_resolver: &DnsResolver, - node: &RelayNode, - certs: Option>>, -) -> Result<(Duration, IpAddr)> { - let url = node.url.join(RELAY_PROBE_PATH)?; - - // This should also use same connection establishment as relay client itself, which - // needs to be more configurable so users can do more crazy things: - // https://github.com/n0-computer/iroh/issues/2901 - let mut builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); - if let Some(Host::Domain(domain)) = url.host() { - // Use our own resolver rather than getaddrinfo - // - // Be careful, a non-zero port will override the port in the URI. - // - // The relay Client uses `.lookup_ipv4_ipv6` to connect, so use the same function - // but staggered for reliability. Ideally this tries to resolve **both** IPv4 and - // IPv6 though. But our resolver does not have a function for that yet. - let addrs: Vec<_> = dns_resolver - .lookup_ipv4_ipv6_staggered(domain) - .await? - .map(|ipaddr| SocketAddr::new(ipaddr, 0)) - .collect(); - builder = builder.resolve_to_addrs(domain, &addrs); - } - if let Some(certs) = certs { - for cert in certs { - let cert = reqwest::Certificate::from_der(&cert)?; - builder = builder.add_root_certificate(cert); - } - } - let client = builder.build()?; - - let start = Instant::now(); - let mut response = client.request(reqwest::Method::GET, url).send().await?; - let latency = start.elapsed(); - if response.status().is_success() { - // Drain the response body to be nice to the server, up to a limit. - const MAX_BODY_SIZE: usize = 8 << 10; // 8 KiB - let mut body_size = 0; - while let Some(chunk) = response.chunk().await? { - body_size += chunk.len(); - if body_size >= MAX_BODY_SIZE { - break; - } - } - - // Only `None` if a different hyper HttpConnector in the request. - let remote_ip = response - .remote_addr() - .context("missing HttpInfo from HttpConnector")? - .ip(); - Ok((latency, remote_ip)) - } else { - Err(anyhow!( - "Error response from server: '{}'", - response.status().canonical_reason().unwrap_or_default() - )) - } -} - -/// Updates a net_report [`Report`] with a new [`ProbeReport`]. -fn update_report(report: &mut Report, probe_report: ProbeReport) { - let relay_node = probe_report.probe.node(); - if let Some(latency) = probe_report.latency { - report - .relay_latency - .update_relay(relay_node.url.clone(), latency); - - if matches!( - probe_report.probe.proto(), - ProbeProto::StunIpv4 - | ProbeProto::StunIpv6 - | ProbeProto::QuicAddrIpv4 - | ProbeProto::QuicAddrIpv6 - ) { - report.udp = true; - - match probe_report.addr { - Some(SocketAddr::V4(ipp)) => { - report.ipv4 = true; - report - .relay_v4_latency - .update_relay(relay_node.url.clone(), latency); - if report.global_v4.is_none() { - report.global_v4 = Some(ipp); - } else if report.global_v4 != Some(ipp) { - report.mapping_varies_by_dest_ip = Some(true); - } else if report.mapping_varies_by_dest_ip.is_none() { - report.mapping_varies_by_dest_ip = Some(false); - } - } - Some(SocketAddr::V6(ipp)) => { - report.ipv6 = true; - report - .relay_v6_latency - .update_relay(relay_node.url.clone(), latency); - if report.global_v6.is_none() { - report.global_v6 = Some(ipp); - } else if report.global_v6 != Some(ipp) { - report.mapping_varies_by_dest_ipv6 = Some(true); - warn!("IPv6 Address detected by STUN varies by destination"); - } else if report.mapping_varies_by_dest_ipv6.is_none() { - report.mapping_varies_by_dest_ipv6 = Some(false); - } - } - None => { - // If we are here we had a relay server latency reported from a STUN probe. - // Thus we must have a reported address. - debug_assert!(probe_report.addr.is_some()); - } - } - } - } - report.ipv4_can_send |= probe_report.ipv4_can_send; - report.ipv6_can_send |= probe_report.ipv6_can_send; - report.icmpv4 = report - .icmpv4 - .map(|val| val || probe_report.icmpv4.unwrap_or_default()) - .or(probe_report.icmpv4); - report.icmpv6 = report - .icmpv6 - .map(|val| val || probe_report.icmpv6.unwrap_or_default()) - .or(probe_report.icmpv6); -} - -/// Resolves to pending if the inner is `None`. -#[derive(Debug)] -pub(crate) struct MaybeFuture { - /// Future to be polled. - pub inner: Option, -} - -// NOTE: explicit implementation to bypass derive unnecessary bounds -impl Default for MaybeFuture { - fn default() -> Self { - MaybeFuture { inner: None } - } -} - -impl Future for MaybeFuture { - type Output = T::Output; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - match self.inner { - Some(ref mut t) => Pin::new(t).poll(cx), - None => Poll::Pending, - } - } -} - -#[cfg(test)] -mod tests { - use std::net::{Ipv4Addr, Ipv6Addr}; - - use testresult::TestResult; - - use super::{super::test_utils, *}; - - #[tokio::test] - async fn test_update_report_stun_working() { - let _logging = iroh_test::logging::setup(); - let (_server_a, relay_a) = test_utils::relay().await; - let (_server_b, relay_b) = test_utils::relay().await; - - let mut report = Report::default(); - - // A STUN IPv4 probe from the the first relay server. - let probe_report_a = ProbeReport { - ipv4_can_send: true, - ipv6_can_send: false, - icmpv4: None, - icmpv6: None, - latency: Some(Duration::from_millis(5)), - probe: Probe::StunIpv4 { - delay: Duration::ZERO, - node: relay_a.clone(), - }, - addr: Some((Ipv4Addr::new(203, 0, 113, 1), 1234).into()), - }; - update_report(&mut report, probe_report_a.clone()); - - assert!(report.udp); - assert_eq!( - report.relay_latency.get(&relay_a.url).unwrap(), - Duration::from_millis(5) - ); - assert_eq!( - report.relay_v4_latency.get(&relay_a.url).unwrap(), - Duration::from_millis(5) - ); - assert!(report.ipv4_can_send); - assert!(!report.ipv6_can_send); - - // A second STUN IPv4 probe, same external IP detected but slower. - let probe_report_b = ProbeReport { - latency: Some(Duration::from_millis(8)), - probe: Probe::StunIpv4 { - delay: Duration::ZERO, - node: relay_b.clone(), - }, - ..probe_report_a - }; - update_report(&mut report, probe_report_b); - - assert!(report.udp); - assert_eq!( - report.relay_latency.get(&relay_a.url).unwrap(), - Duration::from_millis(5) - ); - assert_eq!( - report.relay_v4_latency.get(&relay_a.url).unwrap(), - Duration::from_millis(5) - ); - assert!(report.ipv4_can_send); - assert!(!report.ipv6_can_send); - - // A STUN IPv6 probe, this one is faster. - let probe_report_a_ipv6 = ProbeReport { - ipv4_can_send: false, - ipv6_can_send: true, - icmpv4: None, - icmpv6: None, - latency: Some(Duration::from_millis(4)), - probe: Probe::StunIpv6 { - delay: Duration::ZERO, - node: relay_a.clone(), - }, - addr: Some((Ipv6Addr::new(2001, 0xdb8, 0, 0, 0, 0, 0, 1), 1234).into()), - }; - update_report(&mut report, probe_report_a_ipv6); - - assert!(report.udp); - assert_eq!( - report.relay_latency.get(&relay_a.url).unwrap(), - Duration::from_millis(4) - ); - assert_eq!( - report.relay_v6_latency.get(&relay_a.url).unwrap(), - Duration::from_millis(4) - ); - assert!(report.ipv4_can_send); - assert!(report.ipv6_can_send); - } - - #[tokio::test] - async fn test_update_report_icmp() { - let _logging = iroh_test::logging::setup(); - let (_server_a, relay_a) = test_utils::relay().await; - let (_server_b, relay_b) = test_utils::relay().await; - - let mut report = Report::default(); - - // An ICMPv4 probe from the EU relay server. - let probe_report_eu = ProbeReport { - ipv4_can_send: true, - ipv6_can_send: false, - icmpv4: Some(true), - icmpv6: None, - latency: Some(Duration::from_millis(5)), - probe: Probe::IcmpV4 { - delay: Duration::ZERO, - node: relay_a.clone(), - }, - addr: Some((Ipv4Addr::new(203, 0, 113, 1), 1234).into()), - }; - update_report(&mut report, probe_report_eu.clone()); - - assert!(!report.udp); - assert!(report.ipv4_can_send); - assert_eq!(report.icmpv4, Some(true)); - - // A second ICMPv4 probe which did not work. - let probe_report_na = ProbeReport { - ipv4_can_send: false, - ipv6_can_send: false, - icmpv4: Some(false), - icmpv6: None, - latency: None, - probe: Probe::IcmpV4 { - delay: Duration::ZERO, - node: relay_b.clone(), - }, - addr: None, - }; - update_report(&mut report, probe_report_na); - - assert_eq!(report.icmpv4, Some(true)); - - // Behold, a STUN probe arrives! - let probe_report_eu_stun = ProbeReport { - ipv4_can_send: true, - ipv6_can_send: false, - icmpv4: None, - icmpv6: None, - latency: Some(Duration::from_millis(5)), - probe: Probe::StunIpv4 { - delay: Duration::ZERO, - node: relay_a.clone(), - }, - addr: Some((Ipv4Addr::new(203, 0, 113, 1), 1234).into()), - }; - update_report(&mut report, probe_report_eu_stun); - - assert!(report.udp); - assert_eq!(report.icmpv4, Some(true)); - } - - // # ICMP permissions on Linux - // - // ## Using capabilities: CAP_NET_RAW - // - // To run ICMP tests on Linux you need CAP_NET_RAW capabilities. When running tests - // this means you first need to build the binary, set the capabilities and finally run - // the tests. - // - // Build the test binary: - // - // cargo nextest run -p iroh_net net_report::reportgen::tests --no-run - // - // Find out the test binary location: - // - // cargo nextest list --message-format json -p iroh-net net_report::reportgen::tests \ - // | jq '."rust-suites"."iroh-net"."binary-path"' | tr -d \" - // - // Set the CAP_NET_RAW permission, note that nextest runs each test in a child process - // so the capabilities need to be inherited: - // - // sudo setcap CAP_NET_RAW=eip target/debug/deps/iroh_net-abc123 - // - // Finally run the test: - // - // cargo nextest run -p iroh_net net_report::reportgen::tests - // - // This allows the pinger to create a SOCK_RAW socket for IPPROTO_ICMP. - // - // - // ## Using sysctl - // - // Now you know the hard way, you can also get this permission a little easier, but - // slightly less secure, by allowing any process running with your group ID to create a - // SOCK_DGRAM for IPPROTO_ICMP. - // - // First find out your group ID: - // - // id --group - // - // Then allow this group to send pings. Note that this is an inclusive range: - // - // sudo sysctl net.ipv4.ping_group_range="1234 1234" - // - // Note that this does not survive a reboot usually, commonly you need to edit - // /etc/sysctl.conf or /etc/sysctl.d/* to persist this across reboots. - // - // TODO: Not sure what about IPv6 pings using sysctl. - #[tokio::test] - async fn test_icmpk_probe() { - let _logging_guard = iroh_test::logging::setup(); - let pinger = Pinger::new(); - let (server, node) = test_utils::relay().await; - let addr = server.stun_addr().expect("test relay serves stun"); - let probe = Probe::IcmpV4 { - delay: Duration::from_secs(0), - node, - }; - - // A single ICMP packet might get lost. Try several and take the first. - let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - let mut tasks = JoinSet::new(); - for i in 0..8 { - let probe = probe.clone(); - let pinger = pinger.clone(); - let tx = tx.clone(); - tasks.spawn(async move { - time::sleep(Duration::from_millis(i * 100)).await; - let res = run_icmp_probe(probe, addr, pinger).await; - tx.send(res).ok(); - }); - } - let mut last_err = None; - while let Some(res) = rx.recv().await { - match res { - Ok(report) => { - dbg!(&report); - assert_eq!(report.icmpv4, Some(true)); - assert!( - report.latency.expect("should have a latency") > Duration::from_secs(0) - ); - break; - } - Err(ProbeError::Error(err, _probe)) => { - last_err = Some(err); - } - Err(ProbeError::AbortSet(_err, _probe)) => { - // We don't have permission, too bad. - // panic!("no ping permission: {err:#}"); - break; - } - } - } - if let Some(err) = last_err { - panic!("Ping error: {err:#}"); - } - } - - #[tokio::test] - async fn test_measure_https_latency() -> TestResult { - let _logging_guard = iroh_test::logging::setup(); - let (server, relay) = test_utils::relay().await; - let dns_resolver = crate::dns::tests::resolver(); - tracing::info!(relay_url = ?relay.url , "RELAY_URL"); - let (latency, ip) = - measure_https_latency(dns_resolver, &relay, server.certificates()).await?; - - assert!(latency > Duration::ZERO); - - let relay_url_ip = relay - .url - .host_str() - .context("host")? - .parse::()?; - assert_eq!(ip, relay_url_ip); - Ok(()) - } - - #[tokio::test] - async fn test_quic_probe() -> TestResult { - let _logging_guard = iroh_test::logging::setup(); - let (server, relay) = test_utils::relay().await; - let client_config = iroh_relay::client::make_dangerous_client_config(); - let client_config = quinn::ClientConfig::new(Arc::new(client_config)); - let ep = quinn::Endpoint::client(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))?; - let client_addr = ep.local_addr()?; - let quic_addr_disc = QuicAddressDiscovery { - ep: ep.clone(), - client_config, - }; - let url = relay.url.clone(); - let port = server.quic_addr().unwrap().port(); - let probe = Probe::QuicAddrIpv4 { - delay: Duration::from_secs(0), - node: relay.clone(), - }; - let probe = match run_quic_probe( - quic_addr_disc, - url, - (Ipv4Addr::LOCALHOST, port).into(), - probe, - ) - .await - { - Ok(probe) => probe, - Err(e) => match e { - ProbeError::AbortSet(err, _) | ProbeError::Error(err, _) => { - return Err(err.into()); - } - }, - }; - assert!(probe.ipv4_can_send); - assert_eq!(probe.addr.unwrap(), client_addr); - ep.wait_idle().await; - server.shutdown().await?; - Ok(()) - } -} From 4c912fc7c2e50447602a485b52a2c155639023ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cramfox=E2=80=9D?= <“kasey@n0.computer”> Date: Wed, 11 Dec 2024 00:32:35 -0500 Subject: [PATCH 4/5] clean up tests --- iroh-net-report/src/reportgen.rs | 5 ++- iroh-net-report/src/reportgen/probes.rs | 56 +++++++++++++++++++++++++ iroh-relay/src/server/testing.rs | 6 +-- 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/iroh-net-report/src/reportgen.rs b/iroh-net-report/src/reportgen.rs index f8d408a7934..3f74be07b90 100644 --- a/iroh-net-report/src/reportgen.rs +++ b/iroh-net-report/src/reportgen.rs @@ -72,6 +72,7 @@ pub(super) struct Client { } impl Client { + #[allow(clippy::too_many_arguments)] /// Creates a new actor generating a single report. /// /// The actor starts running immediately and only generates a single report, after which @@ -900,11 +901,11 @@ async fn run_quic_probe( // TODO(ramfox): what to put here if no host is given? let host = url.host_str().unwrap_or("localhost"); let quic_client = iroh_relay::quic::QuicClient::new(quic_config.ep, quic_config.client_config) - .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; + .map_err(|e| ProbeError::Error(e, probe.clone()))?; let (addr, latency) = quic_client .get_addr_and_latency(relay_addr, host) .await - .map_err(|e| ProbeError::Error(e.into(), probe.clone()))?; + .map_err(|e| ProbeError::Error(e, probe.clone()))?; let mut result = ProbeReport::new(probe.clone()); if matches!(probe, Probe::QuicIpv4 { .. }) { result.ipv4_can_send = true; diff --git a/iroh-net-report/src/reportgen/probes.rs b/iroh-net-report/src/reportgen/probes.rs index 42b717cfad5..d116471c705 100644 --- a/iroh-net-report/src/reportgen/probes.rs +++ b/iroh-net-report/src/reportgen/probes.rs @@ -570,6 +570,20 @@ mod tests { Duration::from_millis(100), Duration::from_millis(200)], }, + probeset! { + proto: ProbeProto::QuicIpv4, + relay: relay_node_1.clone(), + delays: [Duration::ZERO, + Duration::from_millis(100), + Duration::from_millis(200)], + }, + probeset! { + proto: ProbeProto::QuicIpv6, + relay: relay_node_1.clone(), + delays: [Duration::ZERO, + Duration::from_millis(100), + Duration::from_millis(200)], + }, probeset! { proto: ProbeProto::Https, relay: relay_node_1.clone(), @@ -605,6 +619,20 @@ mod tests { Duration::from_millis(100), Duration::from_millis(200)], }, + probeset! { + proto: ProbeProto::QuicIpv4, + relay: relay_node_2.clone(), + delays: [Duration::ZERO, + Duration::from_millis(100), + Duration::from_millis(200)], + }, + probeset! { + proto: ProbeProto::QuicIpv6, + relay: relay_node_2.clone(), + delays: [Duration::ZERO, + Duration::from_millis(100), + Duration::from_millis(200)], + }, probeset! { proto: ProbeProto::Https, relay: relay_node_2.clone(), @@ -692,6 +720,22 @@ mod tests { Duration::from_micros(104_800), Duration::from_micros(157_200)], }, + probeset! { + proto: ProbeProto::QuicIpv4, + relay: relay_node_1.clone(), + delays: [Duration::ZERO, + Duration::from_micros(52_400), + Duration::from_micros(104_800), + Duration::from_micros(157_200)], + }, + probeset! { + proto: ProbeProto::QuicIpv6, + relay: relay_node_1.clone(), + delays: [Duration::ZERO, + Duration::from_micros(52_400), + Duration::from_micros(104_800), + Duration::from_micros(157_200)], + }, probeset! { proto: ProbeProto::Https, relay: relay_node_1.clone(), @@ -728,6 +772,18 @@ mod tests { delays: [Duration::ZERO, Duration::from_micros(52_400)], }, + probeset! { + proto: ProbeProto::QuicIpv4, + relay: relay_node_2.clone(), + delays: [Duration::ZERO, + Duration::from_micros(52_400)], + }, + probeset! { + proto: ProbeProto::QuicIpv6, + relay: relay_node_2.clone(), + delays: [Duration::ZERO, + Duration::from_micros(52_400)], + }, probeset! { proto: ProbeProto::Https, relay: relay_node_2.clone(), diff --git a/iroh-relay/src/server/testing.rs b/iroh-relay/src/server/testing.rs index 4e95ceeec6e..ee4fa46e4de 100644 --- a/iroh-relay/src/server/testing.rs +++ b/iroh-relay/src/server/testing.rs @@ -1,5 +1,5 @@ //! Exposes functions to quickly configure a server suitable for testing. -use std::net::{Ipv4Addr, Ipv6Addr}; +use std::net::Ipv4Addr; use super::{CertConfig, QuicConfig, RelayConfig, ServerConfig, StunConfig, TlsConfig}; @@ -71,12 +71,12 @@ pub fn relay_config() -> RelayConfig<()> { /// Creates a [`QuicConfig`] suitable for testing. /// -/// - Binds to an OS assigned port on ipv6 and ipv4, if dual stack is enabled. +/// - Binds to an OS assigned port on ipv4 /// - Uses [`self_signed_tls_certs_and_config`] to create tls certificates pub fn quic_config() -> QuicConfig { let (_, server_config) = self_signed_tls_certs_and_config(); QuicConfig { - bind_addr: (Ipv6Addr::UNSPECIFIED, 0).into(), + bind_addr: (Ipv4Addr::UNSPECIFIED, 0).into(), server_config, } } From 877889166fb9d911c1691e017691db39890e55ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cramfox=E2=80=9D?= <“kasey@n0.computer”> Date: Wed, 11 Dec 2024 14:28:08 -0500 Subject: [PATCH 5/5] PR review --- iroh-net-report/src/lib.rs | 7 +++++++ iroh-net-report/src/reportgen.rs | 13 +++++++++---- iroh-net-report/src/reportgen/probes.rs | 4 ++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/iroh-net-report/src/lib.rs b/iroh-net-report/src/lib.rs index 3898a45cf9d..d0ac11379ae 100644 --- a/iroh-net-report/src/lib.rs +++ b/iroh-net-report/src/lib.rs @@ -241,6 +241,13 @@ impl Client { /// /// If these are not passed in this will bind sockets for STUN itself, though results /// may not be as reliable. + /// + /// The *quic_config* takes a [`QuicConfig`], a combination of a QUIC endpoint and + /// a client configuration that can be use for verifying the relay server connection. + /// When available, the report will attempt to get an observed public address + /// using QUIC address discovery. + /// + /// When `None`, it will disable the QUIC address discovery probes. pub async fn get_report( &mut self, dm: RelayMap, diff --git a/iroh-net-report/src/reportgen.rs b/iroh-net-report/src/reportgen.rs index 3f74be07b90..e43d6e523e6 100644 --- a/iroh-net-report/src/reportgen.rs +++ b/iroh-net-report/src/reportgen.rs @@ -885,8 +885,6 @@ async fn run_stun_probe( } /// Run a QUIC address discovery probe. -// TODO(ramfox): if this probe is aborted, then the connection will never be -// properly closed, possibly causing errors on the server. async fn run_quic_probe( quic_config: QuicConfig, url: RelayUrl, @@ -898,8 +896,15 @@ async fn run_quic_probe( ProbeProto::QuicIpv6 => debug_assert!(relay_addr.is_ipv6()), _ => debug_assert!(false, "wrong probe"), } - // TODO(ramfox): what to put here if no host is given? - let host = url.host_str().unwrap_or("localhost"); + let host = match url.host_str() { + Some(host) => host, + None => { + return Err(ProbeError::Error( + anyhow!("URL must have 'host' to use QUIC address discovery probes"), + probe.clone(), + )); + } + }; let quic_client = iroh_relay::quic::QuicClient::new(quic_config.ep, quic_config.client_config) .map_err(|e| ProbeError::Error(e, probe.clone()))?; let (addr, latency) = quic_client diff --git a/iroh-net-report/src/reportgen/probes.rs b/iroh-net-report/src/reportgen/probes.rs index d116471c705..241ad627717 100644 --- a/iroh-net-report/src/reportgen/probes.rs +++ b/iroh-net-report/src/reportgen/probes.rs @@ -92,12 +92,12 @@ pub(super) enum Probe { delay: Duration, node: Arc, }, - #[display("QUIC Address Discovery Ivp4 after {delay:?} to {node}")] + #[display("QAD Ipv4 after {delay:?} to {node}")] QuicIpv4 { delay: Duration, node: Arc, }, - #[display("QUIC Address Discovery Ivp6 after {delay:?} to {node}")] + #[display("QAD Ipv6 after {delay:?} to {node}")] QuicIpv6 { delay: Duration, node: Arc,