diff options
Diffstat (limited to 'src/transport')
| -rw-r--r-- | src/transport/identity.rs | 102 | ||||
| -rw-r--r-- | src/transport/libp2p.rs | 2579 | ||||
| -rw-r--r-- | src/transport/libp2p/behaviour.rs | 375 | ||||
| -rw-r--r-- | src/transport/libp2p/book.rs | 779 | ||||
| -rw-r--r-- | src/transport/libp2p/config.rs | 177 | ||||
| -rw-r--r-- | src/transport/libp2p/driver.rs | 624 | ||||
| -rw-r--r-- | src/transport/libp2p/errors.rs | 127 | ||||
| -rw-r--r-- | src/transport/libp2p/ignore.rs | 326 | ||||
| -rw-r--r-- | src/transport/libp2p/mod.rs | 698 | ||||
| -rw-r--r-- | src/transport/libp2p/relays.rs | 271 | ||||
| -rw-r--r-- | src/transport/libp2p/routes.rs | 203 | ||||
| -rw-r--r-- | src/transport/libp2p/tests.rs | 1026 | ||||
| -rw-r--r-- | src/transport/mod.rs | 230 | ||||
| -rw-r--r-- | src/transport/mycelium/mod.rs | 251 | ||||
| -rw-r--r-- | src/transport/paths.rs | 464 | ||||
| -rw-r--r-- | src/transport/sim.rs | 85 | ||||
| -rw-r--r-- | src/transport/stats.rs | 15 | ||||
| -rw-r--r-- | src/transport/tor.rs | 70 | ||||
| -rw-r--r-- | src/transport/traffic.rs | 244 | ||||
| -rw-r--r-- | src/transport/wire.rs | 193 |
20 files changed, 6048 insertions, 2791 deletions
diff --git a/src/transport/identity.rs b/src/transport/identity.rs index c597d77..1df84ec 100644 --- a/src/transport/identity.rs +++ b/src/transport/identity.rs @@ -44,7 +44,7 @@ use crate::transport::paths::{Gate, PathHandler, Paths, scheme_of}; use crate::transport::stats::{IdentityCounts, IdentityStats, TransportStats}; use crate::transport::traffic::Traffic; use crate::transport::{ - Destination, EncodedMessage, HandlerFn, IdentityProof, MessageHandler, MessageKind, + Connection, Destination, EncodedMessage, HandlerFn, IdentityProof, MessageHandler, MessageKind, ParsedMessage, Transport, }; @@ -58,8 +58,9 @@ const ASK_INTERVAL: Duration = Duration::from_secs(5); const OTHER_PATHS_ASK_INTERVAL: Duration = Duration::from_secs(60); /// How long resolving a seed waits for its ME. const RESOLVE_TIMEOUT: Duration = Duration::from_secs(10); -/// Addresses kept per node from hints: about one per transport. -const MAX_HINTS: usize = 8; +/// Addresses kept per node from hints: as many as a member lists +/// (`members::MAX_ADDRESSES`). +const MAX_HINTS: usize = 16; /// Addresses with a proven node, nodes with hints, and addresses asked, each at most: /// anyone can send us proofs (keys are free), so the books are bounded. const MAX_ENTRIES: usize = 4096; @@ -187,6 +188,8 @@ impl Book { } IdentityStats { proven, + // Asked of the transports outside this lock (`IdentityTransport::stats`). + relayed: std::collections::BTreeMap::new(), hinted: self .hints .keys() @@ -242,12 +245,15 @@ impl Book { impl IdentityTransport { /// Over `transport`, whose addresses have `config.scheme`, and `config.more`. - /// `firewall`: the debug firewall, if any, which paths applies by address. + /// `firewall`: the debug firewall, if any, which paths applies by address; `clock`: + /// the node's wall clock; `keepalive`: the node's keepalive interval (`Paths::new`). pub(crate) fn new( transport: Arc<dyn Transport>, config: IdentityConfig, traffic: Arc<Traffic>, firewall: Option<Arc<Firewall>>, + clock: Arc<crate::clock::Clock>, + keepalive: Duration, ) -> Self { let node_id = config.key.id(); let shared = Arc::new_cyclic(|shared: &std::sync::Weak<Shared>| { @@ -259,6 +265,8 @@ impl IdentityTransport { .collect(), Some(Arc::clone(&traffic)), firewall.is_some().then_some(gate), + clock, + keepalive, )), firewall, key: config.key, @@ -294,6 +302,22 @@ impl Shared { } impl Shared { + /// Of the nodes in `proven`, those each transport reaches only through a relay, by + /// transport (`IdentityStats::relayed`); asked of the transports, so not under the + /// book's lock. + fn relayed( + &self, + proven: &HashMap<String, Vec<String>>, + ) -> std::collections::BTreeMap<String, usize> { + let mut relayed = std::collections::BTreeMap::new(); + for address in proven.values().flatten() { + if self.paths.connection(address) == Some(Connection::Relayed) { + *relayed.entry(scheme_of(address).to_owned()).or_insert(0) += 1; + } + } + relayed + } + fn lock(&self) -> MutexGuard<'_, Book> { self.book.lock_or_recover() } @@ -435,24 +459,25 @@ impl Shared { } /// Where to send to `peer`: an address it proved itself at, else one it is said to - /// be at; the most preferred transport's first. - fn destination(&self, peer: &str) -> Option<String> { - self.proven_at(peer) + /// be at; the most preferred transport's first. `sending` as `Paths::rank` has it: + /// for a message about to go, or only a look. + fn destination(&self, peer: &str, sending: bool) -> Option<String> { + self.proven_at(peer, sending) .into_iter() .next() - .or_else(|| self.hinted_at(peer).into_iter().next()) + .or_else(|| self.hinted_at(peer, sending).into_iter().next()) } /// The addresses `peer` proved itself at, best first (`Paths::rank`). - fn proven_at(&self, peer: &str) -> Vec<String> { + fn proven_at(&self, peer: &str, sending: bool) -> Vec<String> { let proven = self.lock().proven.get(peer).cloned().unwrap_or_default(); - self.paths.rank(&proven) + self.paths.rank(&proven, sending) } /// Where `peer` is said to be, best first. - fn hinted_at(&self, peer: &str) -> Vec<String> { + fn hinted_at(&self, peer: &str, sending: bool) -> Vec<String> { let hints = self.lock().hints.get(peer).cloned().unwrap_or_default(); - self.paths.rank(&hints) + self.paths.rank(&hints, sending) } } @@ -470,6 +495,10 @@ impl Transport for IdentityTransport { self.shared.paths.delivery_limit() } + fn sync_reply_timeout(&self) -> Duration { + self.shared.paths.sync_reply_timeout() + } + fn local_addresses(&self) -> Vec<String> { self.shared.paths.local_addresses() } @@ -533,7 +562,7 @@ impl Transport for IdentityTransport { } fn can_reach(&self, peer: &str) -> bool { - self.shared.destination(peer).is_some() + self.shared.destination(peer, false).is_some() } fn transports(&self) -> Vec<String> { @@ -548,16 +577,24 @@ impl Transport for IdentityTransport { /// The address `encode` picks, the transport's connection there, and every path /// to `peer` as paths measures it, best first. fn destination(&self, peer: &str) -> Option<Destination> { - let proven = self.shared.proven_at(peer); + let proven = self.shared.proven_at(peer, false); let (address, is_proven) = match proven.first() { Some(address) => (address.clone(), true), - None => (self.shared.hinted_at(peer).into_iter().next()?, false), + None => ( + self.shared.hinted_at(peer, false).into_iter().next()?, + false, + ), }; let connection = self.shared.paths.connection(&address); + let punching = match connection { + Some(Connection::Relayed) => self.shared.paths.punching(&address), + _ => None, + }; Some(Destination { address: self.shared.paths.location(&address), proven: is_proven, connection, + punching, paths: self.shared.paths.info(&proven), }) } @@ -569,15 +606,29 @@ impl Transport for IdentityTransport { /// The best path's, where messages to `peer` go. fn rtt(&self, peer: &str) -> Option<Duration> { - let address = self.shared.proven_at(peer).into_iter().next()?; + let address = self.shared.proven_at(peer, false).into_iter().next()?; self.shared.paths.rtt(&address) } + /// The narrowest of its proven paths'. + fn clock_offset(&self, peer: &str) -> Option<crate::transport::paths::ClockOffset> { + self.shared + .proven_at(peer, false) + .iter() + .filter_map(|address| self.shared.paths.clock_offset(address)) + .min_by_key(|offset| offset.within_ms) + } + /// The identity layer's, and each transport's. fn stats(&self) -> TransportStats { let mut stats = self.shared.paths.stats(); + let (mut identity, proven) = { + let books = self.shared.lock(); + (books.stats(), books.proven.clone()) + }; + identity.relayed = self.shared.relayed(&proven); stats.merge(TransportStats { - identity: Some(self.shared.lock().stats()), + identity: Some(identity), ..TransportStats::default() }); stats @@ -587,12 +638,20 @@ impl Transport for IdentityTransport { self.shared.paths.discovered_seeds() } + fn on_seeds_found(&self, found: std::sync::Arc<tokio::sync::Notify>) { + self.shared.paths.on_seeds_found(found) + } + + fn on_reachable(&self, reachable: std::sync::Arc<tokio::sync::Notify>) { + self.shared.paths.on_reachable(reachable) + } + /// A node ID resolves to itself if this node knows where it is; anything else is /// an address (`<scheme>:...`) or what one of the transports resolves to one (the /// first that does), asked WHO. async fn resolve_seed(&self, seed: &str) -> Result<String> { if identity::is_node_id(seed) { - if self.shared.destination(seed).is_some() { + if self.shared.destination(seed, false).is_some() { return Ok(seed.to_owned()); } bail!("a node ID alone doesn't say where that node is; give its address"); @@ -628,9 +687,10 @@ impl Transport for IdentityTransport { } /// For the address `destination` picks (a send there is refused until `peer` - /// proved itself there, `send_encoded`). + /// proved itself there, `send_encoded`). Only a look at the choice: the send makes + /// it (encoding again for another path if it changes). fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - let Some(address) = self.shared.destination(peer) else { + let Some(address) = self.shared.destination(peer, false) else { bail!("no address known for {peer} on this node's transports"); }; let mut encoded = self.shared.paths.encode(&address, message)?; @@ -641,7 +701,7 @@ impl Transport for IdentityTransport { /// To the addresses `peer` proved itself at, best first, the next when a send /// fails (`Paths::send_first`); refused if it proved itself nowhere yet. async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { - let proven = self.shared.proven_at(&message.peer); + let proven = self.shared.proven_at(&message.peer, true); if !proven.is_empty() { self.shared.ask_other_paths(&message.peer); } diff --git a/src/transport/libp2p.rs b/src/transport/libp2p.rs deleted file mode 100644 index 9217c79..0000000 --- a/src/transport/libp2p.rs +++ /dev/null @@ -1,2579 +0,0 @@ -//! libp2p as a transport (`--transports libp2p`, in builds with the `libp2p` feature): -//! direct connections over TCP (Noise, Yamux) and QUIC, to addresses this node knows. -//! -//! A node's transport identity is its libp2p peer ID; where it is, its listen addresses -//! (multiaddrs ending in the peer ID, `locations`). Seeds are such multiaddrs, or ones -//! without the peer ID, which a connection finds out. Messages go one per stream of the -//! `/continuum/1` protocol: their length (4 bytes, big endian), then the message as -//! `wire` encodes it. A send is done once the stream is written and closed; a connection -//! that can't be made, or a stream that can't be written, refuses it. A stream's protocol -//! goes with its first data, without waiting for the other side to agree -//! (`swarm_config`): opening a stream costs no round trip. -//! -//! Connected peers tell each other where they listen (libp2p's identify protocol), so a -//! node learns the addresses of peers that dialled it. A node publishes where it can be -//! reached (`local_addresses`): the addresses it announces (`--libp2p-announce`, e.g. a -//! public address forwarded to it), or else those it listens on, loopback left out. -//! -//! A node nobody can dial (behind NAT, or a closed port) can still be reached through a -//! relay: a node of the swarm that relays (`--libp2p-relay`), or one configured -//! (`--libp2p-relays`), on which it reserves a slot; its address through the relay -//! (`<relay>/p2p-circuit/p2p/<it>`) is published with the others. Over a relayed connection, both sides then try to -//! connect directly (libp2p's DCUtR, hole punching); the relay carries the traffic until -//! that works, or for good if it doesn't, within its limits (`RELAY_CIRCUIT_DURATION`, -//! `RELAY_CIRCUIT_BYTES`; a circuit that ends is made again on the next send). Hole -//! punching needs both sides to listen; where only one does, the other dials it -//! directly once it knows where (`upgrade`). -//! -//! Of several connections to one peer (both dialled at once, or AutoNAT dialled back), -//! all but one are closed a few seconds later (`DUPLICATE_GRACE`, `Book::duplicates`), -//! by the side with the smaller peer ID. -//! -//! A relay says so where it publishes its addresses: next to each, the same address -//! ending in `/p2p-circuit` (where to reserve a slot), which other nodes find in the -//! member list. A node that needs a relay picks up to `AUTO_RELAYS` relays -//! (`relay_check`): those configured first, then those listed in the document -//! (`learn_relays`), then members that relay. It needs one when -//! it has no address others could dial, or when AutoNAT says it can't be dialled: -//! AutoNAT asks connected peers (nodes of the swarm) to dial it back. Once AutoNAT says -//! it can be dialled, it lets the relays it picked go. -//! -//! With `--libp2p-mdns`, a node finds libp2p peers on its LAN (multicast DNS). Their -//! LAN addresses are noted, and those that turn out to speak continuum (identify says -//! so) are seeds while the node joins (`discovered_seeds`): nodes on one LAN find each -//! other without configured seeds. -//! -//! It uses public libp2p infrastructure only with `--libp2p-public`: then it joins the -//! IPFS DHT (Kademlia, through the IPFS bootstrap nodes), where it finds addresses of -//! peers the member list doesn't have, and public peers that relay; those are picked -//! after members that relay. Public relays keep circuits short (by default 2 minutes and -//! 128 KiB): enough to punch holes through NATs, not to carry a swarm's traffic. Without -//! it, relays are those configured, or members that say they relay. - -use std::collections::HashMap; -use std::path::Path; -use std::str::FromStr; -use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::Duration; - -use ::libp2p::multiaddr::Protocol; -use ::libp2p::swarm::dial_opts::{DialOpts, PeerCondition}; -use ::libp2p::swarm::{ConnectionId, SwarmEvent}; -use ::libp2p::{Multiaddr, PeerId, StreamProtocol, Swarm, identity::Keypair}; -use anyhow::{Context, Result, anyhow, bail}; -use async_trait::async_trait; -use futures::{AsyncReadExt, AsyncWriteExt, StreamExt}; -use tokio::sync::{Semaphore, mpsc, oneshot, watch}; -use tracing::{debug, info, trace, warn}; - -use crate::locks::MutexExt; -use crate::transport::stats::{ - Connections, DialFailures, HolePunches, Libp2pStats, Reachability, RelaySlots, TransportStats, -}; -use crate::transport::{ - Addressed, Connection, EncodedMessage, HandlerFn, MessageHandler, ParsedMessage, Transport, - hand_on_probe, is_probe, wire, -}; - -/// The scheme of this transport's addresses (`transport::paths`). -pub(crate) const SCHEME: &str = "libp2p"; - -const PROTOCOL: StreamProtocol = StreamProtocol::new("/continuum/1"); -/// Sends taking this long are logged with where the time went (at debug). -const SLOW_SEND: Duration = Duration::from_millis(5); -/// The largest message read: anyone who can connect can send anything, so this bounds -/// what one stream costs. Sync messages above 16 chunks travel as blobs. -const MAX_MESSAGE: usize = 128 << 20; -/// How long a connection may take to come up. -const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); -/// A connection nobody uses is closed after this; neighbours keep theirs busy. -const IDLE_TIMEOUT: Duration = Duration::from_secs(300); -/// How long a stream may take to open and to carry a message, before what its size adds -/// (`stream_timeout`). -const STREAM_TIMEOUT: Duration = Duration::from_secs(10); -/// The slowest a message is expected to go, in bytes per second. -const MIN_RATE: u64 = 1 << 20; -/// Inbound streams read at once. -const READS: usize = 32; -/// Addresses kept per peer, and peers with addresses at most. -const ADDRESSES_PER_PEER: usize = 8; -const MAX_PEERS: usize = 4096; - -/// A relayed connection (circuit) through this node lasts at most this long and carries -/// at most this much: generous, since the swarm's own relays may carry sync traffic for -/// good where hole punching fails (libp2p's defaults, 2 minutes and 128 KiB, are for -/// the punching only). -const RELAY_CIRCUIT_DURATION: Duration = Duration::from_secs(3600); -const RELAY_CIRCUIT_BYTES: u64 = 1 << 30; -/// Circuits through this node at once, and per peer (either end: a node behind the -/// relay is an end of every circuit to it). libp2p's rate limits on circuits and slots -/// (a burst, then one every minute or two, per peer and per IP address: every node -/// behind one NAT shares one) are left out: they're for public relays serving -/// strangers, and refused circuits used them up too. -const RELAY_CIRCUITS: usize = 1024; -const RELAY_CIRCUITS_PER_PEER: usize = 64; -/// A configured relay that couldn't be reached to find out its peer ID is tried again -/// after this. -const RELAY_RETRY: Duration = Duration::from_secs(30); -/// Relays picked among the members, at most, and how often a node checks whether it -/// needs them. -const AUTO_RELAYS: usize = 2; -const RELAY_CHECK: Duration = Duration::from_secs(30); -/// A relay that refused a slot (`relay_refused`: none left, not allowed, not a relay, -/// another peer at its address: a relay restarted without its libp2p key) isn't picked -/// again for this long. One that didn't grant it for another reason (it doesn't answer, -/// the connection closed) isn't for `RELAY_RETRY`; a slot lost is reserved again at -/// once. -const RELAY_PAUSE: Duration = Duration::from_secs(600); -/// A peer connected only through a relay is dialled directly where it can be reached -/// (`upgrade`): first at once, then after a failure no sooner than this, doubling up to -/// `RELAY_PAUSE`. -const UPGRADE_RETRY: Duration = Duration::from_secs(30); -/// A peer nobody mentioned for this long (the member list, identify, mDNS, a -/// connection) is forgotten, unless connected: mostly old peer IDs of nodes restarted -/// without their libp2p key, whose new ones the member list has since. Members are -/// mentioned every few seconds (`swarm::topology`). Checked every `PRUNE_EVERY`. -const STALE_AFTER: Duration = Duration::from_secs(600); -const PRUNE_EVERY: Duration = Duration::from_secs(60); -/// Several connections to one peer (both dialled at once, or AutoNAT dialled back to -/// see whether it can) are cut to one (`Book::duplicates`) this long after the last one -/// opened: long enough for what it was dialled for. Looked for every `DEDUPE_EVERY`. -const DUPLICATE_GRACE: Duration = Duration::from_secs(10); -const DEDUPE_EVERY: Duration = Duration::from_secs(5); - -/// How long a message of `bytes` may take on a stream. -fn stream_timeout(bytes: usize) -> Duration { - STREAM_TIMEOUT + Duration::from_millis(bytes as u64 * 1000 / MIN_RATE) -} - -/// What a libp2p transport is made with. -pub(crate) struct Libp2pConfig { - pub(crate) key: Keypair, - /// Multiaddrs to listen on. - pub(crate) listen: Vec<Multiaddr>, - /// Multiaddrs to publish instead of the listen addresses, if any. - pub(crate) announce: Vec<Multiaddr>, - /// Relays to reserve a slot on (multiaddrs; without the relay's peer ID, a connection - /// finds it out). - pub(crate) relays: Vec<Multiaddr>, - /// Relay for other nodes. - pub(crate) relay: bool, - /// Speak continuum (`PROTOCOL`): not on a node that only relays (`--relay-only`), - /// which nobody should take for a node. - pub(crate) continuum: bool, - /// Find peers on the LAN (mDNS). - pub(crate) mdns: bool, - /// Use public infrastructure: the IPFS DHT, public relays. - pub(crate) public: bool, - /// libp2p's in-memory transport instead of TCP and QUIC: for tests. - pub(crate) memory: bool, -} - -impl Libp2pConfig { - /// The key in `path` (32 raw bytes, an Ed25519 secret key), or a new one each start - /// without a file: the peer ID only says where a node is, so a new one costs nothing - /// but the WHO that finds it again (`transport::identity`). - pub(crate) fn load_key(path: Option<&Path>) -> Result<Keypair> { - let Some(path) = path else { - return Ok(Keypair::generate_ed25519()); - }; - let mut bytes = std::fs::read(path) - .with_context(|| format!("Reading the libp2p key from {}", path.display()))?; - crate::swarm::auth::warn_if_readable_by_others(path, "libp2p key"); - if bytes.len() != 32 { - bail!( - "The libp2p key in {} has {} bytes; it has 32", - path.display(), - bytes.len() - ); - } - Keypair::ed25519_from_bytes(&mut bytes) - .map_err(|error| anyhow!("The libp2p key in {}: {error}", path.display())) - } -} - -enum Command { - /// Connect to `peer` (at `addresses`) unless connected. - Connect { - peer: PeerId, - addresses: Vec<Multiaddr>, - done: oneshot::Sender<Result<(), String>>, - }, - /// Connect to `address` to find out which peer is there. - Identify { - address: Multiaddr, - done: oneshot::Sender<Result<PeerId, String>>, - }, - /// See whether relays are needed now (one was just learned of). - CheckRelays, - /// Find out which peer relays at `address`, named without the relay's peer ID - /// (`Book::finding`). - FindRelay { address: Multiaddr }, -} - -/// Who named a relay to pick, in the order they are picked. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Named { - /// `Libp2pConfig::relays`. - Configured, - /// The relays listed in the document (`Transport::learn_relays`). - Listed, -} - -impl Named { - fn as_str(self) -> &'static str { - match self { - Self::Configured => "configured", - Self::Listed => "listed", - } - } -} - -/// A relay named to pick. -#[derive(Clone, Debug, PartialEq, Eq)] -struct NamedRelay { - /// Where, with the relay's peer ID. - at: Multiaddr, - /// As named: without the peer ID, it is found out again when the relay doesn't - /// grant a slot (it may have a new one). - given: Multiaddr, - named: Named, -} - -/// Where peers are, and where we listen. -#[derive(Default)] -struct Book { - addresses: HashMap<PeerId, Vec<Multiaddr>>, - listening: Vec<Multiaddr>, - /// `Libp2pConfig::announce`. - announced: Vec<Multiaddr>, - /// `Libp2pConfig::relay`: this node relays, and says so. - serving: bool, - /// Relays named to pick (configured, listed), by peer: picked first. - named_relays: HashMap<PeerId, NamedRelay>, - /// Relays named without their peer ID, being found out (`Command::FindRelay`). - finding: HashMap<Multiaddr, Named>, - /// The relays listed in the document, as last learned (`learn_listed`). - listed: Vec<Multiaddr>, - /// Members that say they relay, and where (with the relay's peer ID). - relays: HashMap<PeerId, Multiaddr>, - /// With `Libp2pConfig::public`: other peers that relay (identify says so), and - /// where. - public_relays: HashMap<PeerId, Multiaddr>, - /// Relays not to be picked, until when (`RELAY_PAUSE`, `RELAY_RETRY`). - failed_relays: HashMap<PeerId, tokio::time::Instant>, - /// Relays that granted us a slot we hold. - slots: std::collections::HashSet<PeerId>, - /// Peers holding a slot on this node, as a relay. - slots_here: std::collections::HashSet<PeerId>, - /// Peers connected only through a relay, dialled directly (`upgrade`): when to again - /// at the soonest, and how long to wait after that. - upgrades: HashMap<PeerId, (tokio::time::Instant, Duration)>, - /// Open connections: to whom, whether through a relay, and the remote address. - connections: HashMap<ConnectionId, (PeerId, bool, Multiaddr)>, - /// What AutoNAT says, and the address it confirmed while public. - reachability: Reachability, - public_address: Option<Multiaddr>, - /// Since start (`stats`). - counts: Counts, - /// When each peer was last mentioned (`STALE_AFTER`). - seen: HashMap<PeerId, tokio::time::Instant>, - /// Peers found on the LAN (mDNS) not known yet to speak continuum. - lan: std::collections::HashSet<PeerId>, - /// Peers that speak continuum (identify says so). - continuum: std::collections::HashSet<PeerId>, - /// Peers found on the LAN that speak continuum, with their address: seeds not - /// handed out yet (`discovered_seeds`). - discovered: Vec<Multiaddr>, -} - -/// What `Libp2pStats` counts since start. -#[derive(Default)] -struct Counts { - slots_granted: u64, - slots_lost: u64, - slots_not_granted: u64, - hole_punches: HolePunches, - dial_failures: DialFailures, -} - -/// Counts a dial that failed with `error` (not one that wasn't needed). -/// A dial error, shortly: for each address tried (without the peer ID the caller names -/// already), the innermost cause, once. libp2p's own text nests each transport's errors -/// in the next ("Multiple dial errors occurred: ..."), repeating the cause at each level. -fn describe_dial_error(error: &::libp2p::swarm::DialError) -> String { - use ::libp2p::swarm::DialError; - let DialError::Transport(errors) = error else { - return error.to_string(); - }; - let described: Vec<String> = errors - .iter() - .map(|(address, error)| { - let mut address = address.clone(); - if let Some(Protocol::P2p(_)) = address.iter().last() { - address.pop(); - } - format!("{address}: {}", innermost_causes(error)) - }) - .collect(); - if described.is_empty() { - return error.to_string(); - } - described.join("; ") -} - -/// Why a slot on a relay ended, or was never granted, shortly: libp2p nests the -/// relay's answer several types deep (`Left(Left(Left(Reservation(...))))`), and the -/// common cases deserve a sentence. -fn relay_failure(reason: &std::io::Result<()>) -> String { - let Err(error) = reason else { - return "closed".to_owned(); - }; - relay_failure_text(&format!("{error:?}"), &innermost_causes(error)) -} - -/// `relay_failure` on the error's debug form, and its innermost cause as text. -fn relay_failure_text(debug: &str, innermost: &str) -> String { - let known = [ - ( - "NoAddressesInReservation", - "the relay gave no address to be reached through it", - ), - ("ResourceLimitExceeded", "the relay has no slot left"), - ("PermissionDenied", "the relay doesn't let us reserve"), - ("NoReservation", "the relay holds no slot for us"), - ( - "connection gated", - "the relay refused the connection (connection gated)", - ), - ( - "UnsupportedProtocol", - "the relay doesn't relay (no circuit relay protocol)", - ), - ("Timeout", "the relay didn't answer in time"), - ]; - if let Some((_, said)) = known.iter().find(|(pattern, _)| debug.contains(pattern)) { - return (*said).to_owned(); - } - // The protocol refused, but only after its first data (`swarm_config`). - if innermost == "Failed" { - return "the relay refused the circuit relay protocol (is it a relay?)".to_owned(); - } - innermost.to_owned() -} - -/// Whether a slot ended, or wasn't granted, because the relay refused it (its error's -/// debug form and innermost cause, as for `relay_failure_text`), rather than because it -/// didn't answer or the connection closed: only then is it worth not asking again for a -/// while. -fn relay_refused(debug: &str, innermost: &str) -> bool { - let refusals = [ - "NoAddressesInReservation", - "ResourceLimitExceeded", - "PermissionDenied", - "connection gated", - "UnsupportedProtocol", - "WrongPeerId", - ]; - refusals.iter().any(|pattern| debug.contains(pattern)) || innermost == "Failed" -} - -/// The innermost of `error`'s causes, each distinct one once: those libp2p lists in a -/// "Multiple dial errors occurred:" text (one per line, ` - <cause>: <its cause>...`), -/// or the last of the source chain. -fn innermost_causes(error: &(dyn std::error::Error + 'static)) -> String { - let mut innermost = error; - while let Some(source) = innermost.source() { - innermost = source; - } - let text = innermost.to_string(); - let mut causes: Vec<&str> = Vec::new(); - for line in text.lines() { - let Some(listed) = line.trim().strip_prefix("- ") else { - continue; - }; - // A line ends in the next level's heading when that level lists more. - let cause = listed - .split(": ") - .filter(|part| !part.starts_with("Multiple dial errors")) - .last() - .unwrap_or(listed) - .trim(); - if !cause.is_empty() && !causes.contains(&cause) { - causes.push(cause); - } - } - if causes.is_empty() { - let cause = text.rsplit(": ").next().unwrap_or(&text).trim(); - return cause.to_owned(); - } - causes.join(", ") -} - -fn count_dial_failure(failures: &mut DialFailures, error: &::libp2p::swarm::DialError) { - use ::libp2p::swarm::DialError; - match error { - DialError::DialPeerConditionFalse(_) => {} - DialError::Transport(_) => failures.unreachable += 1, - DialError::WrongPeerId { .. } => failures.wrong_peer_id += 1, - DialError::NoAddresses => failures.no_addresses += 1, - _ => failures.other += 1, - } -} - -impl Book { - /// How we are connected to `peer`: directly if any connection is direct. - fn connection(&self, peer: &PeerId) -> Option<Connection> { - let mut found = None; - for (connected, relayed, _) in self.connections.values() { - if connected == peer { - if !relayed { - return Some(Connection::Direct); - } - found = Some(Connection::Relayed); - } - } - found - } - - /// The connections to `peer` to close, of several: all but one, the oldest direct - /// one (the oldest relayed one if none is direct). - /// - /// None between a relay and a node with a slot on it, or reserving one (`reserving`): - /// a slot lives on one connection, which libp2p doesn't tell, and closing it would - /// end the slot, and with it every circuit to the node (the relay then answers - /// `NoReservation`). - fn duplicates(&self, peer: &PeerId, reserving: bool) -> Vec<ConnectionId> { - if reserving || self.slots.contains(peer) || self.slots_here.contains(peer) { - return Vec::new(); - } - let mut connections: Vec<(bool, ConnectionId)> = self - .connections - .iter() - .filter(|(_, (connected, _, _))| connected == peer) - .map(|(id, (_, relayed, _))| (*relayed, *id)) - .collect(); - // Direct (false) first, then the oldest. - connections.sort(); - connections.into_iter().skip(1).map(|(_, id)| id).collect() - } - - /// The remote address of a connection to `peer`: a direct one if there is one. - fn location(&self, peer: &PeerId) -> Option<&Multiaddr> { - let mut found = None; - for (connected, relayed, address) in self.connections.values() { - if connected == peer { - if !relayed { - return Some(address); - } - found = Some(address); - } - } - found - } - - fn stats(&self) -> Libp2pStats { - let mut connections = Connections::default(); - for (_, relayed, _) in self.connections.values() { - if *relayed { - connections.relayed += 1; - } else { - connections.direct += 1; - } - } - Libp2pStats { - reachability: self.reachability, - public_address: self.public_address.as_ref().map(Multiaddr::to_string), - connections, - relay_slots: RelaySlots { - held: self.slots.len(), - paused: self - .failed_relays - .values() - .filter(|until| tokio::time::Instant::now() < **until) - .count(), - granted: self.counts.slots_granted, - lost: self.counts.slots_lost, - not_granted: self.counts.slots_not_granted, - }, - hole_punches: self.counts.hole_punches, - dial_failures: self.counts.dial_failures, - } - } - - /// `peer` was mentioned just now. - fn touch(&mut self, peer: PeerId) { - self.seen.insert(peer, tokio::time::Instant::now()); - } - - /// Forgets the peers not mentioned within `STALE_AFTER` before `now`, but those - /// `connected`. Returns how many. - fn prune(&mut self, now: tokio::time::Instant, connected: impl Fn(&PeerId) -> bool) -> usize { - let stale: Vec<PeerId> = self - .addresses - .keys() - .chain(self.relays.keys()) - .chain(self.public_relays.keys()) - .chain(self.continuum.iter()) - .chain(self.lan.iter()) - .filter(|peer| { - !connected(peer) - && self - .seen - .get(peer) - .is_none_or(|at| now.saturating_duration_since(*at) >= STALE_AFTER) - }) - .copied() - .collect::<std::collections::HashSet<_>>() - .into_iter() - .collect(); - for peer in &stale { - self.addresses.remove(peer); - self.relays.remove(peer); - self.public_relays.remove(peer); - self.continuum.remove(peer); - self.lan.remove(peer); - self.failed_relays.remove(peer); - self.upgrades.remove(peer); - self.seen.remove(peer); - } - stale.len() - } - - /// `peer`, found on the LAN, speaks continuum: a seed, at the address we know first. - fn discover(&mut self, peer: PeerId) { - let Some(address) = self.addresses.get(&peer).and_then(|known| known.first()) else { - return; - }; - if let Ok(address) = address.clone().with_p2p(peer) - && !self.discovered.contains(&address) - && self.discovered.len() < MAX_PEERS - { - self.discovered.push(address); - } - } - - /// Where others could dial us directly: the announced addresses, or else those we - /// listen on (`published`), without the peer ID. - fn direct(&self) -> Vec<Multiaddr> { - if !self.announced.is_empty() { - return self.announced.clone(); - } - let direct: Vec<Multiaddr> = self - .listening - .iter() - .filter(|address| !is_relayed(address)) - .cloned() - .collect(); - published(&direct) - } -} - -/// Whether `address` only reaches this host (or this link): no use to other hosts, and -/// dialled by one, it reaches that host itself. -fn is_local(address: &Multiaddr) -> bool { - match address.iter().next() { - Some(Protocol::Ip4(ip)) => ip.is_loopback() || ip.is_link_local() || ip.is_unspecified(), - Some(Protocol::Ip6(ip)) => { - ip.is_loopback() || ip.is_unicast_link_local() || ip.is_unspecified() - } - _ => false, - } -} - -/// The addresses of `all` worth publishing: all but local ones, unless they're all -/// local (a test on one host). -fn published(all: &[Multiaddr]) -> Vec<Multiaddr> { - let reachable: Vec<Multiaddr> = all - .iter() - .filter(|address| !is_local(address)) - .cloned() - .collect(); - if reachable.is_empty() { - all.to_vec() - } else { - reachable - } -} - -impl Book { - /// `named` names the relay at `given`; returns it if its peer is to be found out. - /// `local`: this node, never its own relay. - fn name_relay(&mut self, given: Multiaddr, named: Named, local: PeerId) -> Option<Multiaddr> { - match split_peer(&given) { - Some(peer) if peer == local => None, - Some(peer) => { - // Configured wins over listed. - if self - .named_relays - .get(&peer) - .is_none_or(|known| known.named == Named::Listed || named == Named::Configured) - { - self.named_relays.insert( - peer, - NamedRelay { - at: given.clone(), - given, - named, - }, - ); - } - None - } - None => { - let known = self.finding.contains_key(&given) - || self.named_relays.values().any(|relay| relay.given == given); - if known { - return None; - } - self.finding.insert(given.clone(), named); - Some(given) - } - } - } - - /// The relay named at `given` (without a peer ID) is `peer`: unless it is no longer - /// named (or is this node). Returns whether that's news. - fn relay_found(&mut self, given: Multiaddr, peer: PeerId, local: PeerId) -> bool { - let Some(named) = self.finding.remove(&given) else { - return false; - }; - let Ok(at) = given.clone().with_p2p(peer) else { - return false; - }; - if peer == local - || (named == Named::Listed - && self - .named_relays - .get(&peer) - .is_some_and(|known| known.named == Named::Configured)) - { - return false; - } - self.named_relays - .insert(peer, NamedRelay { at, given, named }); - true - } - - /// `relay` didn't grant a slot: if it was named without its peer ID, its address is - /// to be found out again, returned. - fn find_again(&mut self, relay: PeerId) -> Option<Multiaddr> { - if self - .named_relays - .get(&relay) - .is_none_or(|known| split_peer(&known.given).is_some()) - { - return None; - } - let known = self.named_relays.remove(&relay)?; - self.finding.insert(known.given.clone(), known.named); - Some(known.given) - } - - /// The relays listed in the document are `listed` now: returns those to find out, - /// None if nothing changed. - fn learn_listed(&mut self, listed: Vec<Multiaddr>, local: PeerId) -> Option<Vec<Multiaddr>> { - if listed == self.listed { - return None; - } - self.named_relays - .retain(|_, relay| relay.named != Named::Listed || listed.contains(&relay.given)); - self.finding - .retain(|given, named| *named != Named::Listed || listed.contains(given)); - self.listed = listed.clone(); - Some( - listed - .into_iter() - .filter_map(|given| self.name_relay(given, Named::Listed, local)) - .collect(), - ) - } - - /// The named relays `named` names, by peer, where. - fn named(&self, named: Named) -> HashMap<PeerId, Multiaddr> { - self.named_relays - .iter() - .filter(|(_, relay)| relay.named == named) - .map(|(peer, relay)| (*peer, relay.at.clone())) - .collect() - } - - /// Whether all of `addresses` are known of `peer` (`remember`). - fn knows(&self, peer: PeerId, addresses: &[Multiaddr]) -> bool { - let Some(known) = self.addresses.get(&peer) else { - return false; - }; - addresses.iter().all(|address| { - let mut address = address.clone(); - if let Some(Protocol::P2p(_)) = address.iter().last() { - address.pop(); - } - known.contains(&address) - }) - } - - /// `peer` is at `address` (without its `/p2p` part): kept first. - fn remember(&mut self, peer: PeerId, mut address: Multiaddr) { - if let Some(Protocol::P2p(_)) = address.iter().last() { - address.pop(); - } - if !self.addresses.contains_key(&peer) && self.addresses.len() >= MAX_PEERS { - return; - } - self.touch(peer); - let addresses = self.addresses.entry(peer).or_default(); - addresses.retain(|known| *known != address); - addresses.insert(0, address); - addresses.truncate(ADDRESSES_PER_PEER); - } -} - -/// Whether `address` goes through a relay. -fn is_relayed(address: &Multiaddr) -> bool { - address - .iter() - .any(|protocol| matches!(protocol, Protocol::P2pCircuit)) -} - -/// The relay `address` goes through, if it goes through one: the peer before -/// `/p2p-circuit`. -fn relay_of(address: &Multiaddr) -> Option<PeerId> { - let mut relay = None; - for protocol in address.iter() { - match protocol { - Protocol::P2p(peer) => relay = Some(peer), - Protocol::P2pCircuit => return relay, - _ => {} - } - } - None -} - -/// The addresses of `known` this node can dial: not those through itself as the relay -/// (a relay reaches the nodes it relays for only while they are connected to it). -fn dialable(known: &[Multiaddr], local: PeerId) -> Vec<Multiaddr> { - known - .iter() - .filter(|address| relay_of(address) != Some(local)) - .cloned() - .collect() -} - -/// `address` with the peer ID it ends in, if it ends in one. -fn split_peer(address: &Multiaddr) -> Option<PeerId> { - match address.iter().last() { - Some(Protocol::P2p(peer)) => Some(peer), - _ => None, - } -} - -pub(crate) struct Libp2pTransport { - local_id: String, - peer_id: PeerId, - control: libp2p_stream::Control, - commands: mpsc::UnboundedSender<Command>, - book: Arc<Mutex<Book>>, - incoming: tokio::sync::Mutex<Option<libp2p_stream::IncomingStreams>>, - stopped: watch::Receiver<bool>, -} - -fn lock(book: &Mutex<Book>) -> MutexGuard<'_, Book> { - book.lock_or_recover() -} - -impl Libp2pTransport { - /// Builds the swarm, listens, and runs it in a task of its own. - pub(crate) fn start(config: Libp2pConfig) -> Result<Self> { - let peer_id = config.key.public().to_peer_id(); - let features = Features::of(&config); - let mut swarm = if config.memory { - memory_swarm(config.key, features)? - } else { - network_swarm(config.key, features)? - }; - for address in config.listen { - swarm - .listen_on(address.clone()) - .with_context(|| format!("Listening on {address}"))?; - } - for address in &config.announce { - // Identify tells peers these. - swarm.add_external_address(address.clone()); - } - let mut control = swarm.behaviour().stream.new_control(); - let incoming = if config.continuum { - Some( - control - .accept(PROTOCOL) - .map_err(|error| anyhow!("Accepting {PROTOCOL}: {error}"))?, - ) - } else { - None - }; - let (commands, receiver) = mpsc::unbounded_channel(); - let mut book = Book { - announced: config.announce, - serving: config.relay, - ..Book::default() - }; - for relay in config.relays { - if let Some(address) = book.name_relay(relay, Named::Configured, peer_id) { - let _ = commands.send(Command::FindRelay { address }); - } - } - let book = Arc::new(Mutex::new(book)); - let (stopping, stopped) = watch::channel(false); - tokio::spawn(run( - swarm, - receiver, - commands.clone(), - Arc::clone(&book), - Relaying { - serving: config.relay, - public: config.public, - local: peer_id, - }, - stopping, - )); - info!(peer_id = %peer_id, "Transport started"); - Ok(Self { - local_id: peer_id.to_string(), - peer_id, - control, - commands, - book, - incoming: tokio::sync::Mutex::new(incoming), - stopped, - }) - } - - /// Connected to `peer`, if it can be had in time. - async fn connect(&self, peer: PeerId) -> Result<()> { - let addresses = lock(&self.book) - .addresses - .get(&peer) - .map(|known| dialable(known, self.peer_id)) - .unwrap_or_default(); - let (done, connected) = oneshot::channel(); - self.commands - .send(Command::Connect { - peer, - addresses, - done, - }) - .map_err(|_| anyhow!("the libp2p swarm stopped"))?; - match tokio::time::timeout(CONNECT_TIMEOUT, connected).await { - Ok(Ok(Ok(()))) => Ok(()), - Ok(Ok(Err(error))) => bail!("couldn't connect to {peer}: {error}"), - Ok(Err(_)) => bail!("the libp2p swarm stopped"), - Err(_) => bail!("connecting to {peer} took longer than {CONNECT_TIMEOUT:?}"), - } - } -} - -/// The protocol version identify says (informational: nothing checks it). -const IDENTIFY_VERSION: &str = "/continuum/1"; - -#[derive(::libp2p::swarm::NetworkBehaviour)] -struct Behaviour { - stream: libp2p_stream::Behaviour, - identify: ::libp2p::identify::Behaviour, - relay_client: ::libp2p::relay::client::Behaviour, - /// Only with `Libp2pConfig::relay`. - relay: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::relay::Behaviour>, - dcutr: ::libp2p::dcutr::Behaviour, - autonat: ::libp2p::autonat::Behaviour, - /// Only with `Libp2pConfig::mdns`. - mdns: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::mdns::tokio::Behaviour>, - /// Only with `Libp2pConfig::public`: the IPFS DHT. - kad: ::libp2p::swarm::behaviour::toggle::Toggle< - ::libp2p::kad::Behaviour<::libp2p::kad::store::MemoryStore>, - >, -} - -/// What the behaviour has besides what every node has. -#[derive(Clone, Copy)] -struct Features { - relay: bool, - mdns: bool, - public: bool, -} - -impl Features { - fn of(config: &Libp2pConfig) -> Self { - Self { - relay: config.relay, - mdns: config.mdns, - public: config.public, - } - } -} - -/// The IPFS DHT's protocol, and its bootstrap nodes (all at `IPFS_BOOTSTRAP_ADDRESS`). -const IPFS_DHT: StreamProtocol = StreamProtocol::new("/ipfs/kad/1.0.0"); -const IPFS_BOOTSTRAP_ADDRESS: &str = "/dnsaddr/bootstrap.libp2p.io"; -const IPFS_BOOTSTRAP_PEERS: [&str; 4] = [ - "QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", - "QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", - "QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", - "QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", -]; -/// How often a node on the DHT refreshes its routing table. -const DHT_BOOTSTRAP: Duration = Duration::from_secs(300); - -impl Behaviour { - fn new( - key: &Keypair, - relay_client: ::libp2p::relay::client::Behaviour, - features: Features, - ) -> std::io::Result<Self> { - let Features { - relay, - mdns, - public, - } = features; - let peer_id = key.public().to_peer_id(); - let kad = public.then(|| { - let mut dht = ::libp2p::kad::Behaviour::with_config( - peer_id, - ::libp2p::kad::store::MemoryStore::new(peer_id), - ::libp2p::kad::Config::new(IPFS_DHT), - ); - let bootstrap: Multiaddr = IPFS_BOOTSTRAP_ADDRESS.parse().expect("a multiaddr"); - for peer in IPFS_BOOTSTRAP_PEERS { - let peer: PeerId = peer.parse().expect("a peer ID"); - let _ = dht.add_address(&peer, bootstrap.clone()); - } - dht - }); - let identify = ::libp2p::identify::Config::new(IDENTIFY_VERSION.to_owned(), key.public()) - .with_agent_version(format!("continuum/{}", env!("CARGO_PKG_VERSION"))); - let relay = relay.then(|| { - ::libp2p::relay::Behaviour::new( - peer_id, - ::libp2p::relay::Config { - max_circuits: RELAY_CIRCUITS, - max_circuits_per_peer: RELAY_CIRCUITS_PER_PEER, - max_circuit_duration: RELAY_CIRCUIT_DURATION, - max_circuit_bytes: RELAY_CIRCUIT_BYTES, - reservation_rate_limiters: Vec::new(), - circuit_src_rate_limiters: Vec::new(), - ..Default::default() - }, - ) - }); - let mdns = match mdns { - true => Some(::libp2p::mdns::tokio::Behaviour::new( - ::libp2p::mdns::Config::default(), - peer_id, - )?), - false => None, - }; - Ok(Self { - stream: libp2p_stream::Behaviour::new(), - identify: ::libp2p::identify::Behaviour::new(identify), - relay_client, - relay: relay.into(), - dcutr: ::libp2p::dcutr::Behaviour::new(peer_id), - autonat: ::libp2p::autonat::Behaviour::new( - peer_id, - ::libp2p::autonat::Config { - // Swarms on private networks (a LAN, WireGuard) count too. - only_global_ips: false, - ..Default::default() - }, - ), - mdns: mdns.into(), - kad: kad.into(), - }) - } -} - -fn network_swarm(key: Keypair, features: Features) -> Result<Swarm<Behaviour>> { - Ok(::libp2p::SwarmBuilder::with_existing_identity(key) - .with_tokio() - .with_tcp( - ::libp2p::tcp::Config::default(), - ::libp2p::noise::Config::new, - ::libp2p::yamux::Config::default, - )? - .with_quic() - .with_dns()? - .with_relay_client( - ::libp2p::noise::Config::new, - ::libp2p::yamux::Config::default, - )? - .with_behaviour(|key, relay_client| { - Behaviour::new(key, relay_client, features) - .map_err(|error| Box::new(error) as Box<dyn std::error::Error + Send + Sync>) - })? - .with_swarm_config(|config| swarm_config(config, features)) - .build()) -} - -/// The swarm's settings. Streams are opened without waiting for the other side to agree -/// on the protocol (`V1Lazy`): the protocol and the first data go together, saving a -/// round trip per message (one stream each, module docs). That is safe only towards -/// peers sure to speak the protocol, so not with public libp2p infrastructure -/// (`Libp2pConfig::public`), whose nodes speak only some: there a protocol they don't -/// would fail late, and unclearly. -fn swarm_config(config: ::libp2p::swarm::Config, features: Features) -> ::libp2p::swarm::Config { - let config = config.with_idle_connection_timeout(IDLE_TIMEOUT); - if features.public { - config - } else { - config.with_substream_upgrade_protocol_override(::libp2p::core::upgrade::Version::V1Lazy) - } -} - -fn memory_swarm(key: Keypair, features: Features) -> Result<Swarm<Behaviour>> { - use ::libp2p::Transport as _; - use ::libp2p::core::transport::{MemoryTransport, OrTransport}; - use ::libp2p::core::upgrade::Version; - - let peer_id = key.public().to_peer_id(); - let (relay_transport, relay_client) = ::libp2p::relay::client::new(peer_id); - let transport = OrTransport::new(relay_transport, MemoryTransport::default()) - .boxed() - .upgrade(Version::V1) - .authenticate(::libp2p::noise::Config::new(&key)?) - .multiplex(::libp2p::yamux::Config::default()) - .timeout(CONNECT_TIMEOUT) - .boxed(); - Ok(Swarm::new( - transport, - Behaviour::new(&key, relay_client, features)?, - peer_id, - swarm_config(::libp2p::swarm::Config::with_tokio_executor(), features), - )) -} - -/// How a node relates to relays. -struct Relaying { - /// It relays for others. - serving: bool, - /// Public relays may be picked, after members. - public: bool, - local: PeerId, -} - -/// The relays to pick among `offers`, if this node needs relays, keeping `using`: a -/// node needs them when it has no address others could dial (`direct`), or AutoNAT says -/// it can't be dialled; up to `AUTO_RELAYS`, in an order of its own (the lowest hash of -/// both peer IDs), so that not every node picks the same. -fn relay_check( - local: PeerId, - direct: bool, - nat: &::libp2p::autonat::NatStatus, - offers: &HashMap<PeerId, Multiaddr>, - using: &[PeerId], -) -> Vec<(PeerId, Multiaddr)> { - use ::libp2p::autonat::NatStatus; - let needed = match nat { - NatStatus::Private => true, - NatStatus::Public(_) => false, - NatStatus::Unknown => !direct, - }; - if !needed || using.len() >= AUTO_RELAYS { - return Vec::new(); - } - let score = |relay: &PeerId| { - use std::hash::{Hash, Hasher}; - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - (local, relay).hash(&mut hasher); - hasher.finish() - }; - let mut candidates: Vec<(PeerId, Multiaddr)> = offers - .iter() - .filter(|(relay, _)| **relay != local && !using.contains(relay)) - .map(|(relay, address)| (*relay, address.clone())) - .collect(); - candidates.sort_by_key(|(relay, _)| score(relay)); - candidates.truncate(AUTO_RELAYS - using.len()); - candidates -} - -/// Reserves slots on the relays `relay_check` picks (configured ones first, then listed -/// ones, members, public ones), and lets those picked go once AutoNAT says this node can -/// be dialled. -fn pick_relays( - swarm: &mut Swarm<Behaviour>, - book: &Mutex<Book>, - relaying: &Relaying, - nat: &::libp2p::autonat::NatStatus, - picked: &mut HashMap<::libp2p::core::transport::ListenerId, PeerId>, -) { - if nat.is_public() && !picked.is_empty() { - info!("Reachable directly (AutoNAT); letting the relays picked go"); - let mut book = lock(book); - for (listener, relay) in std::mem::take(picked) { - swarm.remove_listener(listener); - book.slots.remove(&relay); - } - return; - } - let (direct, configured, listed, members, public) = { - let mut book = lock(book); - let now = tokio::time::Instant::now(); - book.failed_relays.retain(|_, until| now < *until); - let failed = book.failed_relays.clone(); - let usable = |offers: &HashMap<PeerId, Multiaddr>| -> HashMap<PeerId, Multiaddr> { - offers - .iter() - .filter(|(relay, _)| !failed.contains_key(relay)) - .map(|(relay, address)| (*relay, address.clone())) - .collect() - }; - let public = if relaying.public { - usable(&book.public_relays) - } else { - HashMap::new() - }; - ( - !book.direct().is_empty(), - usable(&book.named(Named::Configured)), - usable(&book.named(Named::Listed)), - usable(&book.relays), - public, - ) - }; - let mut using: Vec<PeerId> = picked.values().copied().collect(); - // Each kind for what the ones before left. - for (kind, offers) in [ - (Named::Configured.as_str(), configured), - (Named::Listed.as_str(), listed), - ("member", members), - ("public", public), - ] { - for (relay, address) in relay_check(relaying.local, direct, nat, &offers, &using) { - using.push(relay); - match swarm.listen_on(address.clone().with(Protocol::P2pCircuit)) { - Ok(listener) => { - info!(relay = %Addressed(SCHEME, &relay), address = %address, kind, "Picked a relay"); - picked.insert(listener, relay); - } - Err(error) => { - debug!(relay = %Addressed(SCHEME, &relay), kind, error = %error, "Couldn't use a relay"); - } - } - } - } -} - -/// Dials `peer` directly if it's connected only through a relay and is known elsewhere -/// too (the member list, identify, mDNS), since the first connection, made through a -/// relay (a relayed seed), would otherwise be kept for good: a peer connected isn't -/// dialled again, and hole punching needs both sides to listen. The direct connection, -/// once open, wins over the relayed one (`Book::duplicates`). -fn upgrade(swarm: &mut Swarm<Behaviour>, book: &Mutex<Book>, peer: PeerId, local: PeerId) { - let direct: Vec<Multiaddr> = { - let mut book = lock(book); - if book.connection(&peer) != Some(Connection::Relayed) { - return; - } - let direct: Vec<Multiaddr> = book - .addresses - .get(&peer) - .map(|known| dialable(known, local)) - .unwrap_or_default() - .into_iter() - .filter(|address| !is_relayed(address)) - .collect(); - if direct.is_empty() { - return; - } - let now = tokio::time::Instant::now(); - let (due, wait) = book - .upgrades - .get(&peer) - .copied() - .unwrap_or((now, UPGRADE_RETRY)); - if now < due { - return; - } - book.upgrades - .insert(peer, (now + wait, (wait * 2).min(RELAY_PAUSE))); - direct - }; - debug!(peer = %Addressed(SCHEME, &peer), addresses = ?direct, "Connected only through a relay; dialling directly"); - let _ = swarm.dial( - DialOpts::peer_id(peer) - .addresses(direct) - .condition(PeerCondition::NotDialing) - .build(), - ); -} - -/// Asks the DHT (with `--libp2p-public`) where `peer` is, for the next attempt. -fn find_on_dht(swarm: &mut Swarm<Behaviour>, peer: PeerId) { - if let Some(dht) = swarm.behaviour_mut().kad.as_mut() { - dht.get_closest_peers(peer); - } -} - -/// Drives the swarm: dials what `commands` ask for and notes where we listen, until the -/// transport is dropped. `again`: to schedule commands. -async fn run( - mut swarm: Swarm<Behaviour>, - mut commands: mpsc::UnboundedReceiver<Command>, - again: mpsc::UnboundedSender<Command>, - book: Arc<Mutex<Book>>, - relaying: Relaying, - stopping: watch::Sender<bool>, -) { - let serving = relaying.serving; - let mut nat = ::libp2p::autonat::NatStatus::Unknown; - // Relays picked, by listener. - let mut picked: HashMap<::libp2p::core::transport::ListenerId, PeerId> = HashMap::new(); - let mut check = tokio::time::interval(RELAY_CHECK); - check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let mut dht_bootstrap = tokio::time::interval(DHT_BOOTSTRAP); - dht_bootstrap.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let mut connecting: HashMap<PeerId, Vec<oneshot::Sender<Result<(), String>>>> = HashMap::new(); - let mut identifying: HashMap< - ConnectionId, - (Multiaddr, oneshot::Sender<Result<PeerId, String>>), - > = HashMap::new(); - let mut prune = tokio::time::interval(PRUNE_EVERY); - prune.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - // Peers that may have several connections, and when to cut them to one. Only the - // side with the smaller peer ID does that, so the two never close different ones. - let mut dedupe_due: HashMap<PeerId, tokio::time::Instant> = HashMap::new(); - // Connections closed as duplicates, until they are: their "Connection closed" says so. - let mut closing_duplicates: std::collections::HashSet<ConnectionId> = - std::collections::HashSet::new(); - let mut dedupe = tokio::time::interval(DEDUPE_EVERY); - dedupe.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - tokio::select! { - _ = dedupe.tick(), if !dedupe_due.is_empty() => { - let now = tokio::time::Instant::now(); - let due: Vec<PeerId> = dedupe_due - .iter() - .filter(|(_, at)| **at <= now) - .map(|(peer, _)| *peer) - .collect(); - for peer in due { - dedupe_due.remove(&peer); - let reserving = picked.values().any(|relay| *relay == peer); - let duplicates = lock(&book).duplicates(&peer, reserving); - for connection in duplicates { - if swarm.close_connection(connection) { - closing_duplicates.insert(connection); - } - } - } - } - _ = prune.tick() => { - let forgotten = lock(&book).prune(tokio::time::Instant::now(), |peer| swarm.is_connected(peer)); - if forgotten > 0 { - debug!(forgotten, "Forgot peers nobody mentions any more"); - } - } - _ = dht_bootstrap.tick(), if relaying.public => { - if let Some(dht) = swarm.behaviour_mut().kad.as_mut() - && let Err(error) = dht.bootstrap() - { - warn!(error = %error, "Couldn't join the IPFS DHT"); - } - } - _ = check.tick() => { - pick_relays(&mut swarm, &book, &relaying, &nat, &mut picked); - } - command = commands.recv() => match command { - None => break, - Some(Command::CheckRelays) => { - pick_relays(&mut swarm, &book, &relaying, &nat, &mut picked); - } - Some(Command::Connect { peer, addresses, done }) => { - if swarm.is_connected(&peer) { - let _ = done.send(Ok(())); - upgrade(&mut swarm, &book, peer, relaying.local); - continue; - } - let waiting = connecting.entry(peer).or_default(); - waiting.push(done); - if waiting.len() > 1 { - continue; // Already dialing. - } - let mut opts = DialOpts::peer_id(peer) - .addresses(addresses) - .condition(PeerCondition::DisconnectedAndNotDialing); - if relaying.public { - // Addresses the DHT knows too. - opts = opts.extend_addresses_through_behaviour(); - } - if let Err(error) = swarm.dial(opts.build()) { - count_dial_failure(&mut lock(&book).counts.dial_failures, &error); - for done in connecting.remove(&peer).into_iter().flatten() { - let _ = done.send(Err(error.to_string())); - } - find_on_dht(&mut swarm, peer); - } - } - Some(Command::FindRelay { address }) => { - let again = again.clone(); - let local = relaying.local; - let book = Arc::clone(&book); - tokio::spawn(async move { - let (done, identified) = oneshot::channel(); - if again - .send(Command::Identify { address: address.clone(), done }) - .is_err() - { - return; - } - match tokio::time::timeout(CONNECT_TIMEOUT, identified).await { - Ok(Ok(Ok(peer))) => { - if lock(&book).relay_found(address.clone(), peer, local) { - debug!(relay = %Addressed(SCHEME, &address), peer = %Addressed(SCHEME, &peer), "Found a named relay"); - let _ = again.send(Command::CheckRelays); - } - } - _ => { - if !lock(&book).finding.contains_key(&address) { - return; // No longer named. - } - warn!(relay = %Addressed(SCHEME, &address), "Couldn't reach a named relay; trying again later"); - tokio::time::sleep(RELAY_RETRY).await; - let _ = again.send(Command::FindRelay { address }); - } - } - }); - } - Some(Command::Identify { address, done }) => { - let opts = DialOpts::unknown_peer_id().address(address.clone()).build(); - let id = opts.connection_id(); - match swarm.dial(opts) { - Ok(()) => { - identifying.insert(id, (address, done)); - } - Err(error) => { - let _ = done.send(Err(error.to_string())); - } - } - } - }, - event = swarm.select_next_some() => match event { - SwarmEvent::ConnectionEstablished { peer_id, connection_id, endpoint, .. } => { - { - let mut book = lock(&book); - book.touch(peer_id); - let relayed = endpoint.is_relayed(); - book.connections.insert( - connection_id, - (peer_id, relayed, endpoint.get_remote_address().clone()), - ); - if !relayed { - book.upgrades.remove(&peer_id); - } - if relaying.local < peer_id - && book.connections.values().filter(|(connected, _, _)| *connected == peer_id).count() > 1 - { - dedupe_due.insert(peer_id, tokio::time::Instant::now() + DUPLICATE_GRACE); - } - debug!( - peer = %Addressed(SCHEME, &peer_id), - connection = %connection_id, - dialled = endpoint.is_dialer(), - address = %endpoint.get_remote_address(), - relayed, - "Connection opened" - ); - } - for done in connecting.remove(&peer_id).into_iter().flatten() { - let _ = done.send(Ok(())); - } - if let Some((address, done)) = identifying.remove(&connection_id) { - lock(&book).remember(peer_id, address); - let _ = done.send(Ok(peer_id)); - } - } - SwarmEvent::ConnectionClosed { peer_id, connection_id, cause, .. } => { - lock(&book).connections.remove(&connection_id); - let cause = match cause { - _ if closing_duplicates.remove(&connection_id) => "a duplicate".to_owned(), - // By us, or idle: libp2p doesn't tell which. - None => "closed here".to_owned(), - Some(error) => error.to_string(), - }; - debug!( - peer = %Addressed(SCHEME, &peer_id), - connection = %connection_id, - cause = ?cause, - "Connection closed" - ); - } - SwarmEvent::OutgoingConnectionError { connection_id, peer_id, error } => { - count_dial_failure(&mut lock(&book).counts.dial_failures, &error); - if let Some((_, done)) = identifying.remove(&connection_id) { - let _ = done.send(Err(describe_dial_error(&error))); - } - if let Some(peer) = peer_id - && !swarm.is_connected(&peer) - { - if let Some(waiting) = connecting.remove(&peer) { - for done in waiting { - let _ = done.send(Err(describe_dial_error(&error))); - } - find_on_dht(&mut swarm, peer); - } - } - } - SwarmEvent::NewListenAddr { address, .. } => { - info!(address = %address, "Listening"); - // A relay tells those reserving where it is: its external addresses. - if serving && !is_local(&address) && !is_relayed(&address) { - swarm.add_external_address(address.clone()); - } - let mut book = lock(&book); - if !book.listening.contains(&address) { - book.listening.push(address); - } - } - SwarmEvent::ExpiredListenAddr { address, .. } => { - info!(address = %address, "No longer listening"); - if serving { - swarm.remove_external_address(&address); - } - lock(&book).listening.retain(|known| *known != address); - } - SwarmEvent::ListenerClosed { listener_id, addresses, reason } => { - lock(&book) - .listening - .retain(|known| !addresses.contains(known)); - // A relay picked: the next check picks again, another one if this one - // never granted the slot. - if let Some(relay) = picked.remove(&listener_id) { - let refused = match &reason { - Ok(()) => false, - Err(error) => relay_refused(&format!("{error:?}"), &innermost_causes(error)), - }; - let (lost, pause) = { - let mut book = lock(&book); - let lost = book.slots.remove(&relay); - let pause = match (refused, lost) { - (true, _) => Some(RELAY_PAUSE), - (false, true) => None, - (false, false) => Some(RELAY_RETRY), - }; - if lost { - book.counts.slots_lost += 1; - } else { - book.counts.slots_not_granted += 1; - // Named without its peer ID: maybe another one now. - if let Some(address) = book.find_again(relay) { - let _ = again.send(Command::FindRelay { address }); - } - } - if let Some(pause) = pause { - book.failed_relays.insert(relay, tokio::time::Instant::now() + pause); - } - (lost, pause) - }; - // Again at once, or others meanwhile. - let _ = again.send(Command::CheckRelays); - let reason = relay_failure(&reason); - match (lost, pause) { - (true, None) => { - info!(relay = %Addressed(SCHEME, &relay), reason = %reason, "Lost the slot on a relay; reserving again"); - } - (true, Some(pause)) => { - info!(relay = %Addressed(SCHEME, &relay), pause = ?pause, reason = %reason, "Lost the slot on a relay; picking others for a while"); - } - (false, pause) => { - info!( - relay = %Addressed(SCHEME, &relay), - pause = ?pause.unwrap_or_default(), - reason = %reason, - "A relay didn't grant a slot; picking others for a while" - ); - } - } - } - } - SwarmEvent::ListenerError { error, .. } => { - warn!(error = %error, "Listener failed"); - } - SwarmEvent::Behaviour(BehaviourEvent::RelayClient( - ::libp2p::relay::client::Event::ReservationReqAccepted { relay_peer_id, renewal, .. }, - )) => { - { - let mut book = lock(&book); - book.slots.insert(relay_peer_id); - book.failed_relays.remove(&relay_peer_id); - if !renewal { - book.counts.slots_granted += 1; - } - } - if !renewal { - info!(relay = %Addressed(SCHEME, &relay_peer_id), "Reachable through a relay"); - } - } - SwarmEvent::Behaviour(BehaviourEvent::Autonat( - ::libp2p::autonat::Event::StatusChanged { old, new }, - )) => { - info!(old = ?old, new = ?new, "Reachability (AutoNAT)"); - { - use ::libp2p::autonat::NatStatus; - let mut book = lock(&book); - (book.reachability, book.public_address) = match &new { - NatStatus::Unknown => (Reachability::Unknown, None), - NatStatus::Private => (Reachability::Private, None), - NatStatus::Public(address) => { - (Reachability::Public, Some(address.clone())) - } - }; - } - nat = new; - } - SwarmEvent::Behaviour(BehaviourEvent::Relay(event)) => { - { - use ::libp2p::relay::Event; - let mut book = lock(&book); - match &event { - Event::ReservationReqAccepted { src_peer_id, .. } => { - book.slots_here.insert(*src_peer_id); - } - Event::ReservationClosed { src_peer_id } - | Event::ReservationTimedOut { src_peer_id } => { - book.slots_here.remove(src_peer_id); - } - _ => {} - } - } - debug!(event = ?event, "Relay event"); - } - SwarmEvent::Behaviour(BehaviourEvent::Dcutr(event)) => match event.result { - Ok(_) => { - lock(&book).counts.hole_punches.succeeded += 1; - info!(peer = %Addressed(SCHEME, &event.remote_peer_id), "Connected directly to a peer behind a relay"); - } - Err(error) => { - lock(&book).counts.hole_punches.failed += 1; - debug!(peer = %Addressed(SCHEME, &event.remote_peer_id), error = %error, "Couldn't connect directly to a peer behind a relay; the relay carries on"); - } - }, - SwarmEvent::Behaviour(BehaviourEvent::Identify( - ::libp2p::identify::Event::Received { connection_id, peer_id, info }, - )) => { - { - let mut book = lock(&book); - let listen = published(&info.listen_addrs); - // Every new connection identifies again: logged when it says something new. - if book.knows(peer_id, &listen) { - trace!(peer = %Addressed(SCHEME, &peer_id), connection = %connection_id, "Peer identified again"); - } else { - debug!( - peer = %Addressed(SCHEME, &peer_id), - connection = %connection_id, - listen = ?info.listen_addrs, - observed = %info.observed_addr, - agent = %info.agent_version, - "Peer identified" - ); - } - // Last first: `remember` puts each in front. - for address in listen.into_iter().rev() { - book.remember(peer_id, address); - } - if relaying.public - && info.protocols.contains(&::libp2p::relay::HOP_PROTOCOL_NAME) - && (book.public_relays.len() < MAX_PEERS - || book.public_relays.contains_key(&peer_id)) - && let Some(address) = book - .addresses - .get(&peer_id) - .and_then(|known| known.first()) - .and_then(|address| address.clone().with_p2p(peer_id).ok()) - { - book.public_relays.insert(peer_id, address); - } - if info.protocols.contains(&PROTOCOL) { - book.continuum.insert(peer_id); - if book.lan.remove(&peer_id) { - book.discover(peer_id); - } - } - } - upgrade(&mut swarm, &book, peer_id, relaying.local); - } - SwarmEvent::Behaviour(BehaviourEvent::Mdns( - ::libp2p::mdns::Event::Discovered(found), - )) => { - for (peer, address) in found { - if peer == relaying.local { - continue; - } - debug!(peer = %Addressed(SCHEME, &peer), address = %address, "Peer on the LAN"); - let continuum = { - let mut book = lock(&book); - book.remember(peer, address.clone()); - book.continuum.contains(&peer) - }; - upgrade(&mut swarm, &book, peer, relaying.local); - if continuum { - lock(&book).discover(peer); - } else if lock(&book).lan.len() < MAX_PEERS { - // Connected, identify says whether it speaks continuum. - lock(&book).lan.insert(peer); - let _ = swarm.dial( - DialOpts::peer_id(peer) - .addresses(vec![address]) - .condition(PeerCondition::DisconnectedAndNotDialing) - .build(), - ); - } - } - } - _ => {} - }, - } - } - let _ = stopping.send(true); -} - -/// One message from an inbound stream. -async fn read_message(mut stream: ::libp2p::Stream) -> Result<Vec<u8>> { - let mut length = [0u8; 4]; - tokio::time::timeout(STREAM_TIMEOUT, stream.read_exact(&mut length)) - .await - .map_err(|_| anyhow!("no length within {STREAM_TIMEOUT:?}"))??; - let length = u32::from_be_bytes(length) as usize; - if length > MAX_MESSAGE { - bail!("a message of {length} bytes; at most {MAX_MESSAGE} are read"); - } - let mut body = vec![0u8; length]; - tokio::time::timeout(stream_timeout(length), stream.read_exact(&mut body)) - .await - .map_err(|_| { - anyhow!( - "{length} bytes took longer than {:?}", - stream_timeout(length) - ) - })??; - Ok(body) -} - -#[async_trait] -impl Transport for Libp2pTransport { - fn local_id(&self) -> &str { - &self.local_id - } - - fn is_peer_id(&self, id: &str) -> bool { - PeerId::from_str(id).is_ok() - } - - /// The longest a message may take on a stream. - fn delivery_limit(&self) -> Duration { - stream_timeout(MAX_MESSAGE) - } - - /// Where we can be reached, each with our peer ID: the announced addresses, or else - /// those we listen on (`published`), and those through relays. A relay adds each - /// direct one again, ending in `/p2p-circuit`: where to reserve a slot. - fn local_addresses(&self) -> Vec<String> { - let book = lock(&self.book); - let direct: Vec<Multiaddr> = book - .direct() - .into_iter() - .filter_map(|address| address.with_p2p(self.peer_id).ok()) - .collect(); - let relayed = book - .listening - .iter() - .filter(|address| is_relayed(address)) - .filter_map(|address| address.clone().with_p2p(self.peer_id).ok()); - let offers: Vec<Multiaddr> = if book.serving { - direct - .iter() - .map(|address| address.clone().with(Protocol::P2pCircuit)) - .collect() - } else { - Vec::new() - }; - direct - .into_iter() - .chain(relayed) - .chain(offers) - .map(|address| address.to_string()) - .collect() - } - - /// A peer ID, or a multiaddr ending in one (noted as where that peer is). - fn locate(&self, target: &str) -> Option<String> { - if let Ok(peer) = PeerId::from_str(target) { - return Some(peer.to_string()); - } - let mut address = Multiaddr::from_str(target).ok()?; - // A relay saying where to reserve a slot: `<address>/p2p/<relay>/p2p-circuit`. - if let Some(Protocol::P2pCircuit) = address.iter().last() { - address.pop(); - let relay = split_peer(&address)?; - if relay != self.peer_id { - let new = { - let mut book = lock(&self.book); - let new = book.relays.get(&relay) != Some(&address); - if new && (book.relays.len() < MAX_PEERS || book.relays.contains_key(&relay)) { - book.relays.insert(relay, address.clone()); - } - book.remember(relay, address); - new - }; - if new { - let _ = self.commands.send(Command::CheckRelays); - } - } - return Some(relay.to_string()); - } - let peer = split_peer(&address)?; - if peer != self.peer_id { - lock(&self.book).remember(peer, address); - } - Some(peer.to_string()) - } - - /// Multiaddrs, with or without the relay's peer ID; others are ignored. - fn learn_relays(&self, targets: &[String]) { - let listed = targets - .iter() - .filter_map(|target| Multiaddr::from_str(target).ok()) - .collect(); - let Some(found) = lock(&self.book).learn_listed(listed, self.peer_id) else { - return; - }; - for address in found { - let _ = self.commands.send(Command::FindRelay { address }); - } - let _ = self.commands.send(Command::CheckRelays); - } - - fn connection(&self, target: &str) -> Option<Connection> { - let peer = PeerId::from_str(target).ok()?; - lock(&self.book).connection(&peer) - } - - fn location(&self, target: &str) -> Option<String> { - let peer = PeerId::from_str(target).ok()?; - let address = lock(&self.book).location(&peer)?.clone(); - Some( - address - .with_p2p(peer) - .unwrap_or_else(|address| address) - .to_string(), - ) - } - - fn stats(&self) -> TransportStats { - TransportStats { - libp2p: Some(lock(&self.book).stats()), - ..TransportStats::default() - } - } - - /// Nodes found on the LAN (mDNS) that speak continuum. - fn discovered_seeds(&self) -> Vec<String> { - std::mem::take(&mut lock(&self.book).discovered) - .iter() - .map(Multiaddr::to_string) - .collect() - } - - /// Where `target` is, each address with its peer ID. - fn locations(&self, target: &str) -> Vec<String> { - let Ok(peer) = PeerId::from_str(target) else { - return Vec::new(); - }; - lock(&self.book) - .addresses - .get(&peer) - .into_iter() - .flatten() - .filter_map(|address| address.clone().with_p2p(peer).ok()) - .map(|address| address.to_string()) - .collect() - } - - /// A multiaddr ending in a peer ID is that peer; one without is connected to, to - /// find out. A bare peer ID resolves if this transport knows where it is. - async fn resolve_seed(&self, seed: &str) -> Result<String> { - if let Ok(peer) = PeerId::from_str(seed) { - if lock(&self.book).addresses.contains_key(&peer) { - return Ok(peer.to_string()); - } - bail!("no address known for libp2p peer {peer}; give a multiaddr"); - } - let address = - Multiaddr::from_str(seed).with_context(|| format!("{seed} is no multiaddr"))?; - if let Some(peer) = self.locate(seed) { - return Ok(peer); - } - let (done, identified) = oneshot::channel(); - self.commands - .send(Command::Identify { address, done }) - .map_err(|_| anyhow!("the libp2p swarm stopped"))?; - match tokio::time::timeout(CONNECT_TIMEOUT, identified).await { - Ok(Ok(Ok(peer))) => Ok(peer.to_string()), - Ok(Ok(Err(error))) => bail!("couldn't connect to {seed}: {error}"), - Ok(Err(_)) => bail!("the libp2p swarm stopped"), - Err(_) => bail!("connecting to {seed} took longer than {CONNECT_TIMEOUT:?}"), - } - } - - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - let body = wire::encode(message)?; - Ok(EncodedMessage { - peer: peer.to_owned(), - address: peer.to_owned(), - payload_len: body.len(), - body: body.into(), - kind: message.kind(), - message: None, - }) - } - - async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { - let peer = PeerId::from_str(&message.peer) - .with_context(|| format!("{} is no libp2p peer ID", message.peer))?; - let started = std::time::Instant::now(); - self.connect(peer).await?; - let connected = started.elapsed(); - let mut control = self.control.clone(); - let body = message.body.clone(); - let write = async move { - let mut stream = control.open_stream(peer, PROTOCOL).await?; - let opened = started.elapsed(); - stream.write_all(&(body.len() as u32).to_be_bytes()).await?; - stream.write_all(&body).await?; - stream.close().await?; - Ok::<_, anyhow::Error>(opened) - }; - let timeout = stream_timeout(message.body.len()); - let opened = tokio::time::timeout(timeout, write) - .await - .map_err(|_| anyhow!("sending to {peer} took longer than {timeout:?}"))? - .with_context(|| format!("Sending to {peer}"))?; - let took = started.elapsed(); - if took >= SLOW_SEND { - // Where a send's time goes: the swarm task (connecting), the stream's protocol - // negotiation (opening), or the connection (writing). - debug!( - peer = %Addressed(SCHEME, &peer), - kind = ?message.kind, - bytes = message.body.len(), - connect_ms = connected.as_secs_f64() * 1000.0, - open_ms = (opened - connected).as_secs_f64() * 1000.0, - write_ms = (took - opened).as_secs_f64() * 1000.0, - "Slow send" - ); - } - Ok(()) - } - - /// Reads inbound streams (`READS` at once) and handles their messages one at a - /// time, in the order they were read; PINGs and PONGs at once (`is_probe`). - async fn start_consumer(&self, handler: MessageHandler) -> Result<()> { - let Some(mut incoming) = self.incoming.lock().await.take() else { - bail!("libp2p consumer already started, or this node only relays"); - }; - let handler: Arc<HandlerFn> = Arc::from(handler); - let (sender, mut received) = mpsc::channel::<(PeerId, ParsedMessage, usize)>(READS); - let reads = Arc::new(Semaphore::new(READS)); - let probes = Arc::clone(&handler); - tokio::spawn(async move { - while let Some((peer, stream)) = incoming.next().await { - let Ok(permit) = Arc::clone(&reads).acquire_owned().await else { - break; - }; - let sender = sender.clone(); - let probes = Arc::clone(&probes); - tokio::spawn(async move { - match read_message(stream).await { - Ok(body) => match wire::decode(&body) { - Ok(message) if is_probe(&message) => { - hand_on_probe( - SCHEME, - &probes, - peer.to_string(), - message, - body.len(), - ); - } - Ok(message) => { - let _ = sender.send((peer, message, body.len())).await; - } - Err(error) => { - debug!(peer = %Addressed(SCHEME, &peer), error = %error, "Dropping a message that doesn't decode"); - } - }, - Err(error) => { - debug!(peer = %Addressed(SCHEME, &peer), error = %format!("{error:#}"), "Dropping an unreadable stream"); - } - } - drop(permit); - }); - } - }); - while let Some((peer, message, payload_len)) = received.recv().await { - if let Err(error) = handler(peer.to_string(), message, payload_len).await { - warn!(peer = %Addressed(SCHEME, &peer), error = %format!("{error:#}"), "Failed to process inbound message"); - } - } - Ok(()) - } - - async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> { - let mut stopped = self.stopped.clone(); - let _ = stopped.wait_for(|stopped| *stopped).await; - Err(std::io::Error::other("the libp2p swarm stopped")) - } -} - -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicU64, Ordering}; - - use super::*; - use crate::document::replica::Epoch; - use crate::transport::{HeadsSummary, Status}; - - /// Memory ports are process-wide. - static NEXT_PORT: AtomicU64 = AtomicU64::new(40_000); - - /// A node on the in-memory transport, listening on a port of its own (or nowhere), - /// and its address with its peer ID (or just its peer ID). - fn memory_node_with( - listen: bool, - relays: Vec<Multiaddr>, - relay: bool, - ) -> (Libp2pTransport, Multiaddr) { - let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); - let key = Keypair::generate_ed25519(); - let address: Multiaddr = format!("/memory/{port}").parse().unwrap(); - let peer = key.public().to_peer_id(); - let transport = Libp2pTransport::start(Libp2pConfig { - key, - listen: if listen { - vec![address.clone()] - } else { - Vec::new() - }, - announce: Vec::new(), - relays, - relay, - continuum: true, - mdns: false, - public: false, - memory: true, - }) - .unwrap(); - let address = if listen { - address.with_p2p(peer).unwrap() - } else { - Multiaddr::empty().with(Protocol::P2p(peer)) - }; - (transport, address) - } - - fn memory_node() -> (Libp2pTransport, Multiaddr) { - memory_node_with(true, Vec::new(), false) - } - - fn status() -> ParsedMessage { - ParsedMessage::Status(Status { - heads: HeadsSummary::of(Vec::new()), - epoch: Epoch::GENESIS, - epoch_seq: 3, - ready: true, - syncs_with_you: false, - }) - } - - /// Starts `transport`'s consumer, forwarding what it receives. - fn received( - transport: Arc<Libp2pTransport>, - ) -> mpsc::UnboundedReceiver<(String, ParsedMessage)> { - let (sender, receiver) = mpsc::unbounded_channel(); - tokio::spawn(async move { - transport - .start_consumer(Box::new(move |from, message, _| { - let sender = sender.clone(); - Box::pin(async move { - let _ = sender.send((from, message)); - Ok(()) - }) - })) - .await - }); - receiver - } - - #[tokio::test] - async fn messages_both_ways_over_one_connection() { - let (a, a_address) = memory_node(); - let (b, _) = memory_node(); - let (a, b) = (Arc::new(a), Arc::new(b)); - let mut at_a = received(Arc::clone(&a)); - let mut at_b = received(Arc::clone(&b)); - - assert_eq!( - b.resolve_seed(&a_address.to_string()).await.unwrap(), - a.local_id() - ); - b.send_message(a.local_id(), &status()).await.unwrap(); - let (from, message) = at_a.recv().await.unwrap(); - assert_eq!(from, b.local_id()); - assert!(matches!(message, ParsedMessage::Status(status) if status.epoch_seq == 3)); - - // a knows no address of b, but b's connection is there. - a.send_message(b.local_id(), &status()).await.unwrap(); - let (from, _) = at_b.recv().await.unwrap(); - assert_eq!(from, a.local_id()); - - assert_eq!(a.connection(b.local_id()), Some(Connection::Direct)); - assert_eq!(b.connection(a.local_id()), Some(Connection::Direct)); - let stats = a.stats().libp2p.unwrap(); - assert!(stats.connections.direct >= 1); - assert_eq!(stats.connections.relayed, 0); - } - - #[tokio::test] - async fn a_seed_without_a_peer_id_is_connected_to_to_find_out() { - let (a, a_address) = memory_node(); - let (b, _) = memory_node(); - let mut bare = a_address.clone(); - bare.pop(); - assert_eq!( - b.resolve_seed(&bare.to_string()).await.unwrap(), - a.local_id() - ); - // And b now knows where a is. - assert_eq!(b.locations(a.local_id()), [a_address.to_string()]); - } - - #[tokio::test] - async fn listen_addresses_carry_the_peer_id() { - let (a, a_address) = memory_node(); - let deadline = tokio::time::Instant::now() + Duration::from_secs(5); - while a.local_addresses().is_empty() && tokio::time::Instant::now() < deadline { - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert_eq!(a.local_addresses(), [a_address.to_string()]); - assert_eq!( - a.locate(&a_address.to_string()).as_deref(), - Some(a.local_id()) - ); - } - - /// Waits up to 5 s for `condition`. - async fn eventually(mut condition: impl FnMut() -> bool) -> bool { - let deadline = tokio::time::Instant::now() + Duration::from_secs(5); - while !condition() { - if tokio::time::Instant::now() >= deadline { - return false; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - true - } - - /// A node learns where a peer that dialled it listens (identify), and can then - /// pass it on. - #[tokio::test] - async fn a_dialled_node_learns_where_the_dialler_listens() { - let (a, a_address) = memory_node(); - let (b, b_address) = memory_node(); - let (a, b) = (Arc::new(a), Arc::new(b)); - let _at_a = received(Arc::clone(&a)); - b.resolve_seed(&a_address.to_string()).await.unwrap(); - b.send_message(a.local_id(), &status()).await.unwrap(); - let b_id = b.local_id().to_owned(); - assert!( - eventually(|| a.locations(&b_id) == [b_address.to_string()]).await, - "a knows {:?} of b", - a.locations(&b_id) - ); - } - - #[tokio::test] - async fn announced_addresses_are_published_instead() { - let key = Keypair::generate_ed25519(); - let peer = key.public().to_peer_id(); - let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); - let announced: Multiaddr = "/ip4/203.0.113.7/tcp/9661".parse().unwrap(); - let transport = Libp2pTransport::start(Libp2pConfig { - key, - listen: vec![format!("/memory/{port}").parse().unwrap()], - announce: vec![announced.clone()], - relays: Vec::new(), - relay: false, - continuum: true, - mdns: false, - public: false, - memory: true, - }) - .unwrap(); - assert_eq!( - transport.local_addresses(), - [announced.with_p2p(peer).unwrap().to_string()] - ); - } - - #[test] - fn local_addresses_are_left_out_unless_there_are_only_local_ones() { - let parse = |text: &str| text.parse::<Multiaddr>().unwrap(); - let loopback = parse("/ip4/127.0.0.1/tcp/1"); - let link = parse("/ip6/fe80::1/tcp/1"); - let lan = parse("/ip4/192.168.1.2/tcp/1"); - let wireguard = parse("/ip6/fd00::2/tcp/1"); - assert_eq!( - published(&[ - loopback.clone(), - link.clone(), - lan.clone(), - wireguard.clone() - ]), - [lan, wireguard] - ); - assert_eq!( - published(&[loopback.clone(), link.clone()]), - [loopback, link] - ); - } - - /// b listens nowhere but through the relay r; a reaches it there. - #[tokio::test] - async fn a_node_nobody_can_dial_is_reached_through_a_relay() { - let (_r, r_address) = memory_node_with(true, Vec::new(), true); - let (b, _) = memory_node_with(false, vec![r_address.clone()], false); - let b = Arc::new(b); - let mut at_b = received(Arc::clone(&b)); - assert!( - eventually(|| !b.local_addresses().is_empty()).await, - "b got no slot on the relay" - ); - let relayed = b.local_addresses()[0].clone(); - assert!( - relayed.starts_with(&format!("{r_address}/p2p-circuit/")), - "{relayed}" - ); - - assert!(eventually(|| b.stats().libp2p.unwrap().relay_slots.held == 1).await); - let slots = b.stats().libp2p.unwrap().relay_slots; - assert_eq!((slots.granted, slots.not_granted), (1, 0)); - - let (a, _) = memory_node(); - assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); - a.send_message(b.local_id(), &status()).await.unwrap(); - let (from, _) = at_b.recv().await.unwrap(); - assert_eq!(from, a.local_id()); - assert!(a.connection(b.local_id()).is_some()); - } - - /// b listens nowhere; a, which can be dialled, reaches it through the relay r: b - /// then dials a directly (identify told it where), and both keep that. - #[tokio::test] - async fn a_relayed_connection_is_upgraded_where_one_side_can_be_dialled() { - let (_r, r_address) = memory_node_with(true, Vec::new(), true); - let (b, _) = memory_node_with(false, vec![r_address.clone()], false); - let b = Arc::new(b); - let mut at_b = received(Arc::clone(&b)); - assert!( - eventually(|| !b.local_addresses().is_empty()).await, - "b got no slot on the relay" - ); - let (a, _) = memory_node(); - let relayed = b.local_addresses()[0].clone(); - assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); - a.send_message(b.local_id(), &status()).await.unwrap(); - at_b.recv().await.unwrap(); - assert!( - eventually(|| b.connection(a.local_id()) == Some(Connection::Direct)).await, - "b didn't dial a directly" - ); - assert_eq!(a.connection(b.local_id()), Some(Connection::Direct)); - // The relayed one goes (`DUPLICATE_GRACE`); the direct one stays. - let deadline = tokio::time::Instant::now() + DUPLICATE_GRACE * 2; - while a.stats().libp2p.unwrap().connections.relayed > 0 { - assert!( - tokio::time::Instant::now() < deadline, - "the relayed connection stayed" - ); - tokio::time::sleep(Duration::from_millis(100)).await; - } - b.send_message(a.local_id(), &status()).await.unwrap(); - assert_eq!(a.connection(b.local_id()), Some(Connection::Direct)); - } - - /// A relay that is no node (`--relay-only`) relays, and takes no messages itself. - #[tokio::test] - async fn a_relay_that_is_no_node() { - let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); - let key = Keypair::generate_ed25519(); - let r_peer = key.public().to_peer_id(); - let r_listen: Multiaddr = format!("/memory/{port}").parse().unwrap(); - let r = Libp2pTransport::start(Libp2pConfig { - key, - listen: vec![r_listen.clone()], - announce: Vec::new(), - relays: Vec::new(), - relay: true, - continuum: false, - mdns: false, - public: false, - memory: true, - }) - .unwrap(); - let r_address = r_listen.with_p2p(r_peer).unwrap(); - let (b, _) = memory_node_with(false, vec![r_address.clone()], false); - let b = Arc::new(b); - let mut at_b = received(Arc::clone(&b)); - assert!( - eventually(|| !b.local_addresses().is_empty()).await, - "b got no slot on the relay" - ); - let (a, _) = memory_node(); - let relayed = b.local_addresses()[0].clone(); - assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); - a.send_message(b.local_id(), &status()).await.unwrap(); - let (from, _) = at_b.recv().await.unwrap(); - assert_eq!(from, a.local_id()); - - // The relay itself doesn't speak continuum: it takes no messages (a send to it - // isn't refused, though: the protocol goes with the first data, `swarm_config`, - // so the relay's refusal comes too late; nodes never send to it, as it never - // proves itself a node). - assert_eq!( - a.resolve_seed(&r_address.to_string()).await.unwrap(), - r.local_id() - ); - assert!( - r.start_consumer(Box::new(|_, _, _| Box::pin(async { Ok(()) }))) - .await - .is_err() - ); - } - - /// A relay configured without its peer ID is found out, and used. - #[tokio::test] - async fn a_relay_configured_without_its_peer_id() { - let (_r, r_address) = memory_node_with(true, Vec::new(), true); - let mut bare = r_address.clone(); - bare.pop(); - let (b, _) = memory_node_with(false, vec![bare], false); - assert!( - eventually(|| !b.local_addresses().is_empty()).await, - "b got no slot on the relay" - ); - assert!(b.local_addresses()[0].starts_with(&format!("{r_address}/p2p-circuit/"))); - } - - fn peer() -> PeerId { - Keypair::generate_ed25519().public().to_peer_id() - } - - #[test] - fn relays_are_picked_when_needed() { - use ::libp2p::autonat::NatStatus; - let local = peer(); - let offers: HashMap<PeerId, Multiaddr> = (0..4) - .map(|port| (peer(), format!("/memory/{port}").parse().unwrap())) - .collect(); - let check = |direct, nat: &NatStatus, using: &[PeerId]| { - relay_check(local, direct, nat, &offers, using) - }; - // Dialable, or not known yet with an address to dial: none. - assert!(check(true, &NatStatus::Public(Multiaddr::empty()), &[]).is_empty()); - assert!(check(false, &NatStatus::Public(Multiaddr::empty()), &[]).is_empty()); - assert!(check(true, &NatStatus::Unknown, &[]).is_empty()); - // No address to dial, or AutoNAT says it can't be dialled: some. - assert_eq!(check(false, &NatStatus::Unknown, &[]).len(), AUTO_RELAYS); - let picked = check(true, &NatStatus::Private, &[]); - assert_eq!(picked.len(), AUTO_RELAYS); - // One in use: the next in the same order. - let more = check(true, &NatStatus::Private, &[picked[0].0]); - assert_eq!(more.len(), AUTO_RELAYS - 1); - assert_eq!(more[0].0, picked[1].0); - let all: Vec<PeerId> = picked.iter().map(|(relay, _)| *relay).collect(); - assert!(check(true, &NatStatus::Private, &all).is_empty()); - // Never itself. - let only_us = HashMap::from([(local, Multiaddr::empty())]); - assert!(relay_check(local, false, &NatStatus::Private, &only_us, &[]).is_empty()); - } - - /// A relay says so in its addresses; b, which nobody can dial, learns of it (as from - /// the member list) and becomes reachable through it. - #[tokio::test] - async fn a_member_that_relays_is_picked() { - let (r, _) = memory_node_with(true, Vec::new(), true); - assert!(eventually(|| !r.local_addresses().is_empty()).await); - let offer = r - .local_addresses() - .into_iter() - .find(|address| address.ends_with("/p2p-circuit")) - .expect("the relay offers itself"); - let (b, _) = memory_node_with(false, Vec::new(), false); - assert_eq!(b.locate(&offer).as_deref(), Some(r.local_id())); - assert!( - eventually(|| b - .local_addresses() - .iter() - .any(|address| address.contains("/p2p-circuit/"))) - .await, - "b isn't reachable through r: {:?}", - b.local_addresses() - ); - } - - /// A peer on the LAN that speaks continuum becomes a seed once, at its address. - #[test] - fn lan_peers_become_seeds_once() { - let mut book = Book::default(); - let lan = peer(); - let address: Multiaddr = "/ip4/192.168.1.5/tcp/9661".parse().unwrap(); - book.discover(lan); // No address known: nothing. - assert!(book.discovered.is_empty()); - book.remember(lan, address.clone()); - book.discover(lan); - book.discover(lan); - assert_eq!(book.discovered, [address.with_p2p(lan).unwrap()]); - } - - #[test] - fn named_relays() { - let (local, r1, r2) = (peer(), peer(), peer()); - let at = |port: u16| -> Multiaddr { format!("/ip6/fd00::1/tcp/{port}").parse().unwrap() }; - let mut book = Book::default(); - // With a peer ID: known at once; without: to be found out, once. - assert_eq!( - book.name_relay(at(1).with_p2p(r1).unwrap(), Named::Configured, local), - None - ); - assert_eq!( - book.name_relay(at(2), Named::Configured, local), - Some(at(2)) - ); - assert_eq!(book.name_relay(at(2), Named::Configured, local), None); - assert_eq!( - book.name_relay(at(3).with_p2p(local).unwrap(), Named::Configured, local), - None - ); - assert!(book.relay_found(at(2), r2, local)); - assert_eq!(book.named(Named::Configured).len(), 2); - // A relay named with its peer ID isn't found out again; one without is. - assert_eq!(book.find_again(r1), None); - assert_eq!(book.find_again(r2), Some(at(2))); - assert!(!book.named(Named::Configured).contains_key(&r2)); - - // Listed: configured wins; unlisted ones go, unless also configured. - let r3 = peer(); - let listed = vec![ - at(1).with_p2p(r1).unwrap(), - at(4).with_p2p(r3).unwrap(), - at(5), - ]; - assert_eq!(book.learn_listed(listed.clone(), local), Some(vec![at(5)])); - assert_eq!(book.learn_listed(listed, local), None); - assert_eq!(book.named_relays[&r1].named, Named::Configured); - assert_eq!(book.named(Named::Listed).keys().collect::<Vec<_>>(), [&r3]); - assert_eq!(book.learn_listed(Vec::new(), local), Some(Vec::new())); - assert!(book.named(Named::Listed).is_empty()); - assert!(!book.finding.contains_key(&at(5))); - assert!(!book.relay_found(at(5), peer(), local)); // No longer listed. - assert_eq!(book.named(Named::Configured).len(), 1); - } - - #[test] - fn known_addresses() { - let mut book = Book::default(); - let (relay, us) = (peer(), peer()); - let direct: Multiaddr = "/ip6/fd00::1/tcp/9661".parse().unwrap(); - let relayed: Multiaddr = format!("/ip6/fd00::2/tcp/9661/p2p/{relay}/p2p-circuit/p2p/{us}") - .parse() - .unwrap(); - assert!(!book.knows(us, &[direct.clone()])); - book.remember(us, relayed.clone()); - book.remember(us, direct.clone()); - // In any order, with or without the peer ID. - assert!(book.knows(us, &[relayed.clone(), direct.clone()])); - assert!(book.knows(us, &[direct.clone().with_p2p(us).unwrap()])); - assert!(!book.knows(us, &["/ip6/fd00::3/tcp/9661".parse().unwrap()])); - } - - /// Peers nobody mentions any more are forgotten, but connected ones. - #[test] - fn stale_peers_are_forgotten() { - let (old, current, connected) = (peer(), peer(), peer()); - let address: Multiaddr = "/ip6/fd00::1/tcp/9661".parse().unwrap(); - let mut book = Book::default(); - for peer in [old, current, connected] { - book.remember(peer, address.clone()); - } - book.relays.insert(old, address.clone()); - book.continuum.insert(old); - // From after the mentions above (taken at `Instant::now()`), however fast or - // slow the clock moved meanwhile. - let later = tokio::time::Instant::now() + STALE_AFTER; - // `current` was mentioned again since. - book.seen.insert(current, later); - assert_eq!(book.prune(later, |peer| *peer == connected), 1); - assert!(!book.addresses.contains_key(&old)); - assert!(!book.relays.contains_key(&old) && !book.continuum.contains(&old)); - assert!(book.addresses.contains_key(¤t)); - assert!(book.addresses.contains_key(&connected)); - assert_eq!(book.prune(later, |_| false), 1); // `connected`, once it isn't. - } - - #[test] - fn relay_failures_are_short() { - assert_eq!( - relay_failure_text( - "Custom { kind: Other, error: Other(Left(Left(Left(Reservation(Protocol(NoAddressesInReservation)))))) }", - "x" - ), - "the relay gave no address to be reached through it" - ); - assert_eq!( - relay_failure_text( - "Codec(Error(Custom { kind: Other, error: Failed }))", - "Failed" - ), - "the relay refused the circuit relay protocol (is it a relay?)" - ); - assert_eq!( - relay_failure_text("Io(…)", "connection reset"), - "connection reset" - ); - assert_eq!(relay_failure(&Ok(())), "closed"); - } - - /// Only refusals pause a relay; a slot that ended otherwise is asked for again. - #[test] - fn only_refusals_pause_relays() { - let refused = [ - ("Reservation(Protocol(NoAddressesInReservation))", "x"), - ("Reservation(Status(ResourceLimitExceeded))", "x"), - ("Reservation(Status(PermissionDenied))", "x"), - ("Denied { cause: \"connection gated\" }", "x"), - ("Upgrade(UnsupportedProtocol)", "x"), - ("Dial(WrongPeerId { obtained: … })", "x"), - ( - "Codec(Error(Custom { kind: Other, error: Failed }))", - "Failed", - ), - ]; - for (debug, innermost) in refused { - assert!(relay_refused(debug, innermost), "{debug}"); - } - let not = [ - ("Io(Custom { kind: Other, error: Timeout })", "Timeout"), - ( - "Io(Custom { kind: Other, error: Right(Closed) })", - "connection is closed", - ), - ("Io(…)", "connection reset"), - ]; - for (debug, innermost) in not { - assert!(!relay_refused(debug, innermost), "{debug}"); - } - } - - /// libp2p's nested dial errors come out as each address's cause, once. - #[test] - fn dial_errors_are_short() { - use ::libp2p::core::transport::TransportError; - let peer = peer(); - let address: Multiaddr = format!("/ip6/fd00::1/tcp/9661/p2p/{peer}").parse().unwrap(); - let nested = std::io::Error::other( - "Multiple dial errors occurred:\n - Network is unreachable (os error 101): Multiple dial errors occurred:\n - Network is unreachable (os error 101): Network is unreachable (os error 101)", - ); - let error = - ::libp2p::swarm::DialError::Transport(vec![(address, TransportError::Other(nested))]); - assert_eq!( - describe_dial_error(&error), - "/ip6/fd00::1/tcp/9661: Network is unreachable (os error 101)" - ); - let refused = ::libp2p::swarm::DialError::Transport(vec![( - "/ip4/10.0.0.1/tcp/1".parse().unwrap(), - TransportError::Other(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)), - )]); - assert_eq!( - describe_dial_error(&refused), - "/ip4/10.0.0.1/tcp/1: connection refused" - ); - } - - /// A relay doesn't dial the nodes it relays for through itself. - #[test] - fn no_circuits_through_ourselves() { - let (us, other_relay, target) = (peer(), peer(), peer()); - let through = |relay: PeerId| -> Multiaddr { - format!("/ip6/fd00::1/tcp/9661/p2p/{relay}/p2p-circuit") - .parse() - .unwrap() - }; - let direct: Multiaddr = "/ip6/fd00::2/tcp/9661".parse().unwrap(); - assert_eq!(relay_of(&through(us)), Some(us)); - assert_eq!(relay_of(&direct), None); - assert_eq!(relay_of(&direct.clone().with_p2p(target).unwrap()), None); - assert_eq!( - dialable(&[through(us), through(other_relay), direct.clone()], us), - [through(other_relay), direct] - ); - } - - #[tokio::test] - async fn unknown_peers_are_refused() { - let (b, _) = memory_node(); - let stranger = Keypair::generate_ed25519() - .public() - .to_peer_id() - .to_string(); - assert!(b.send_message(&stranger, &status()).await.is_err()); - assert!(b.resolve_seed(&stranger).await.is_err()); - assert_eq!(b.connection(&stranger), None); - } - - #[test] - fn a_direct_connection_wins_over_a_relayed_one() { - let mut book = Book::default(); - let (relayed_only, both) = (peer(), peer()); - let mut id = 0; - let mut connect = |book: &mut Book, peer: PeerId, relayed: bool| { - id += 1; - let address: Multiaddr = format!("/ip4/10.0.0.{id}/tcp/1").parse().unwrap(); - book.connections - .insert(ConnectionId::new_unchecked(id), (peer, relayed, address)); - }; - connect(&mut book, relayed_only, true); - connect(&mut book, both, true); - connect(&mut book, both, false); - assert_eq!(book.connection(&relayed_only), Some(Connection::Relayed)); - assert_eq!(book.connection(&both), Some(Connection::Direct)); - assert_eq!(book.connection(&peer()), None); - assert_eq!( - book.location(&both).map(|address| address.to_string()), - Some("/ip4/10.0.0.3/tcp/1".to_owned()) - ); - let connections = book.stats().connections; - assert_eq!((connections.direct, connections.relayed), (1, 2)); - // Of several, the oldest direct one stays; a relayed one only without a direct. - connect(&mut book, both, false); - assert_eq!( - book.duplicates(&both, false), - [ - ConnectionId::new_unchecked(4), - ConnectionId::new_unchecked(2) - ] - ); - assert!(book.duplicates(&relayed_only, false).is_empty()); - // Not while one of them carries a slot, ours or the peer's, or we reserve one. - assert!(book.duplicates(&both, true).is_empty()); - book.slots.insert(both); - assert!(book.duplicates(&both, false).is_empty()); - book.slots.clear(); - book.slots_here.insert(both); - assert!(book.duplicates(&both, false).is_empty()); - } -} diff --git a/src/transport/libp2p/behaviour.rs b/src/transport/libp2p/behaviour.rs new file mode 100644 index 0000000..0b9be6b --- /dev/null +++ b/src/transport/libp2p/behaviour.rs @@ -0,0 +1,375 @@ +//! The libp2p behaviours this transport combines (`Behaviour`), hole punching with LAN +//! candidates (`HolePunching`), and building the swarm (TCP and QUIC, or in memory). + +use super::*; + +/// The protocol version identify says (informational: nothing checks it). +pub(super) const IDENTIFY_VERSION: &str = "/continuum/1"; + +#[derive(::libp2p::swarm::NetworkBehaviour)] +pub(super) struct Behaviour { + /// First: connections it denies need nothing from the others. + pub(super) ignoring: Ignoring, + pub(super) stream: libp2p_stream::Behaviour, + pub(super) identify: Identifying, + pub(super) relay_client: ::libp2p::relay::client::Behaviour, + /// Only with `Libp2pConfig::relay`. + pub(super) relay: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::relay::Behaviour>, + pub(super) dcutr: HolePunching, + /// Each connection's round trip (`Book::rtts`): of several to one peer, the fastest + /// is kept. + pub(super) ping: ::libp2p::ping::Behaviour, + pub(super) autonat: ::libp2p::autonat::Behaviour, + /// Only with `Libp2pConfig::mdns`. + pub(super) mdns: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::mdns::tokio::Behaviour>, + /// Only with `Libp2pConfig::upnp`: maps our listen ports on the router, and confirms + /// the mapped addresses as ours (identify then tells them). + pub(super) upnp: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::upnp::tokio::Behaviour>, + /// Only with `Libp2pConfig::public`: the IPFS DHT. + pub(super) kad: ::libp2p::swarm::behaviour::toggle::Toggle< + ::libp2p::kad::Behaviour<::libp2p::kad::store::MemoryStore>, + >, +} + +/// libp2p's hole punching (DCUtR), offering our listen addresses too. DCUtR offers only +/// where others saw us (external address candidates, from identify): seen by a relay +/// out on the internet, two nodes on one LAN are both at their router's public address, +/// and punching through it needs the router to hairpin, which many don't. Their LAN +/// addresses work, the firewalls on the way being stateful. Only DCUtR is told: +/// AutoNAT would test every candidate, and fail on all of these. +pub(super) struct HolePunching { + pub(super) inner: ::libp2p::dcutr::Behaviour, + /// Where we listen (`offered`). + pub(super) listening: Vec<Multiaddr>, + /// `Libp2pConfig::ignore`: not offered. + pub(super) ignored: Vec<IpRange>, +} + +/// Whether DCUtR offers our listen address `address` too: neither loopback nor +/// link-local (as for publishing), nor through a relay. +pub(super) fn offered(address: &Multiaddr) -> bool { + !is_local(address) && !is_relayed(address) +} + +impl HolePunching { + pub(super) fn new(local: PeerId, ignored: Vec<IpRange>) -> Self { + Self { + inner: ::libp2p::dcutr::Behaviour::new(local), + listening: Vec::new(), + ignored, + } + } + + /// Tells DCUtR where we listen, as if others had seen us there. Again after every + /// other candidate, as DCUtR keeps the latest 20. + pub(super) fn offer_listening(&mut self) { + use ::libp2p::swarm::{FromSwarm, NetworkBehaviour, NewExternalAddrCandidate}; + for addr in &self.listening { + self.inner + .on_swarm_event(FromSwarm::NewExternalAddrCandidate( + NewExternalAddrCandidate { addr }, + )); + } + } +} + +impl ::libp2p::swarm::NetworkBehaviour for HolePunching { + type ConnectionHandler = + <::libp2p::dcutr::Behaviour as ::libp2p::swarm::NetworkBehaviour>::ConnectionHandler; + type ToSwarm = ::libp2p::dcutr::Event; + + fn handle_pending_inbound_connection( + &mut self, + connection_id: ConnectionId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ::libp2p::swarm::ConnectionDenied> { + self.inner + .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) + } + + fn handle_established_inbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + if is_relayed(local_addr) { + // DCUtR offers its candidates on this connection: where others saw us, and + // these. + debug!(peer = %Addressed(SCHEME, &peer), listening = ?self.listening, "Reached through a relay; hole punching offers where we listen too"); + } + self.inner.handle_established_inbound_connection( + connection_id, + peer, + local_addr, + remote_addr, + ) + } + + fn handle_pending_outbound_connection( + &mut self, + connection_id: ConnectionId, + maybe_peer: Option<PeerId>, + addresses: &[Multiaddr], + effective_role: ::libp2p::core::Endpoint, + ) -> Result<Vec<Multiaddr>, ::libp2p::swarm::ConnectionDenied> { + self.inner.handle_pending_outbound_connection( + connection_id, + maybe_peer, + addresses, + effective_role, + ) + } + + fn handle_established_outbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + addr: &Multiaddr, + role_override: ::libp2p::core::Endpoint, + port_use: ::libp2p::core::transport::PortUse, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + self.inner.handle_established_outbound_connection( + connection_id, + peer, + addr, + role_override, + port_use, + ) + } + + fn on_swarm_event(&mut self, event: ::libp2p::swarm::FromSwarm) { + use ::libp2p::swarm::FromSwarm; + self.inner.on_swarm_event(event); + match event { + FromSwarm::NewListenAddr(new) + if offered(new.addr) && !ignored(&self.ignored, new.addr) => + { + if !self.listening.contains(new.addr) { + debug!(address = %new.addr, "Offering where we listen for hole punching"); + self.listening.push(new.addr.clone()); + } + self.offer_listening(); + } + FromSwarm::ExpiredListenAddr(expired) => { + self.listening.retain(|address| address != expired.addr); + } + FromSwarm::NewExternalAddrCandidate(_) => self.offer_listening(), + _ => {} + } + } + + fn on_connection_handler_event( + &mut self, + peer_id: PeerId, + connection_id: ConnectionId, + event: ::libp2p::swarm::THandlerOutEvent<Self>, + ) { + self.inner + .on_connection_handler_event(peer_id, connection_id, event) + } + + fn poll( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll< + ::libp2p::swarm::ToSwarm<Self::ToSwarm, ::libp2p::swarm::THandlerInEvent<Self>>, + > { + self.inner.poll(cx) + } +} + +/// What the behaviour has besides what every node has. +#[derive(Clone, Copy)] +pub(super) struct Features { + pub(super) relay: bool, + pub(super) mdns: bool, + pub(super) upnp: bool, + pub(super) public: bool, + /// How long a connection may idle (`idle_timeout`). + pub(super) idle_timeout: Duration, +} + +impl Features { + pub(super) fn of(config: &Libp2pConfig) -> Self { + Self { + relay: config.relay, + mdns: config.mdns, + upnp: config.upnp, + public: config.public, + idle_timeout: idle_timeout(config.keepalive), + } + } +} + +/// The IPFS DHT's protocol, and its bootstrap nodes (all at `IPFS_BOOTSTRAP_ADDRESS`). +pub(super) const IPFS_DHT: StreamProtocol = StreamProtocol::new("/ipfs/kad/1.0.0"); + +pub(super) const IPFS_BOOTSTRAP_ADDRESS: &str = "/dnsaddr/bootstrap.libp2p.io"; + +pub(super) const IPFS_BOOTSTRAP_PEERS: [&str; 4] = [ + "QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", + "QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", + "QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", + "QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", +]; + +/// How often a node on the DHT refreshes its routing table. +pub(super) const DHT_BOOTSTRAP: Duration = Duration::from_secs(300); + +impl Behaviour { + pub(super) fn new( + key: &Keypair, + relay_client: ::libp2p::relay::client::Behaviour, + features: Features, + ignored: Vec<IpRange>, + ) -> std::io::Result<Self> { + let Features { + relay, + mdns, + upnp, + public, + idle_timeout: _, + } = features; + let peer_id = key.public().to_peer_id(); + let kad = public.then(|| { + let mut dht = ::libp2p::kad::Behaviour::with_config( + peer_id, + ::libp2p::kad::store::MemoryStore::new(peer_id), + ::libp2p::kad::Config::new(IPFS_DHT), + ); + let bootstrap: Multiaddr = IPFS_BOOTSTRAP_ADDRESS.parse().expect("a multiaddr"); + for peer in IPFS_BOOTSTRAP_PEERS { + let peer: PeerId = peer.parse().expect("a peer ID"); + let _ = dht.add_address(&peer, bootstrap.clone()); + } + dht + }); + let identify = ::libp2p::identify::Config::new(IDENTIFY_VERSION.to_owned(), key.public()) + .with_agent_version(format!("continuum/{}", env!("CARGO_PKG_VERSION"))) + // Connected peers hear of a new interface at once (`explore`). + .with_push_listen_addr_updates(true); + let relay = relay.then(|| { + ::libp2p::relay::Behaviour::new( + peer_id, + ::libp2p::relay::Config { + max_circuits: RELAY_CIRCUITS, + max_circuits_per_peer: RELAY_CIRCUITS_PER_PEER, + max_circuit_duration: RELAY_CIRCUIT_DURATION, + max_circuit_bytes: RELAY_CIRCUIT_BYTES, + reservation_rate_limiters: Vec::new(), + circuit_src_rate_limiters: Vec::new(), + ..Default::default() + }, + ) + }); + let mdns = match mdns { + true => Some(::libp2p::mdns::tokio::Behaviour::new( + ::libp2p::mdns::Config::default(), + peer_id, + )?), + false => None, + }; + Ok(Self { + ignoring: Ignoring { + ranges: ignored.clone(), + }, + stream: libp2p_stream::Behaviour::new(), + identify: Identifying { + inner: ::libp2p::identify::Behaviour::new(identify), + ranges: ignored.clone(), + // Set by `Libp2pTransport::start`. + announced: Vec::new(), + }, + relay_client, + relay: relay.into(), + dcutr: HolePunching::new(peer_id, ignored), + ping: ::libp2p::ping::Behaviour::new(::libp2p::ping::Config::new()), + autonat: ::libp2p::autonat::Behaviour::new( + peer_id, + ::libp2p::autonat::Config { + // Swarms on private networks (a LAN, WireGuard) count too. + only_global_ips: false, + ..Default::default() + }, + ), + mdns: mdns.into(), + upnp: upnp.then(::libp2p::upnp::tokio::Behaviour::default).into(), + kad: kad.into(), + }) + } +} + +pub(super) fn network_swarm( + key: Keypair, + features: Features, + ignored: Vec<IpRange>, +) -> Result<Swarm<Behaviour>> { + Ok(::libp2p::SwarmBuilder::with_existing_identity(key) + .with_tokio() + .with_tcp( + ::libp2p::tcp::Config::default(), + ::libp2p::noise::Config::new, + ::libp2p::yamux::Config::default, + )? + .with_quic() + .with_dns()? + .with_relay_client( + ::libp2p::noise::Config::new, + ::libp2p::yamux::Config::default, + )? + .with_behaviour(|key, relay_client| { + Behaviour::new(key, relay_client, features, ignored.clone()) + .map_err(|error| Box::new(error) as Box<dyn std::error::Error + Send + Sync>) + })? + .with_swarm_config(|config| swarm_config(config, features)) + .build()) +} + +/// The swarm's settings. Streams are opened without waiting for the other side to agree +/// on the protocol (`V1Lazy`): the protocol and the first data go together, saving a +/// round trip per message (one stream each, module docs). That is safe only towards +/// peers sure to speak the protocol, so not with public libp2p infrastructure +/// (`Libp2pConfig::public`), whose nodes speak only some: there a protocol they don't +/// would fail late, and unclearly. +pub(super) fn swarm_config( + config: ::libp2p::swarm::Config, + features: Features, +) -> ::libp2p::swarm::Config { + let config = config + .with_idle_connection_timeout(features.idle_timeout) + .with_dial_concurrency_factor(DIAL_CONCURRENCY); + if features.public { + config + } else { + config.with_substream_upgrade_protocol_override(::libp2p::core::upgrade::Version::V1Lazy) + } +} + +pub(super) fn memory_swarm( + key: Keypair, + features: Features, + ignored: Vec<IpRange>, +) -> Result<Swarm<Behaviour>> { + use ::libp2p::Transport as _; + use ::libp2p::core::transport::{MemoryTransport, OrTransport}; + use ::libp2p::core::upgrade::Version; + + let peer_id = key.public().to_peer_id(); + let (relay_transport, relay_client) = ::libp2p::relay::client::new(peer_id); + let transport = OrTransport::new(relay_transport, MemoryTransport::default()) + .boxed() + .upgrade(Version::V1) + .authenticate(::libp2p::noise::Config::new(&key)?) + .multiplex(::libp2p::yamux::Config::default()) + .timeout(CONNECT_TIMEOUT) + .boxed(); + Ok(Swarm::new( + transport, + Behaviour::new(&key, relay_client, features, ignored)?, + peer_id, + swarm_config(::libp2p::swarm::Config::with_tokio_executor(), features), + )) +} diff --git a/src/transport/libp2p/book.rs b/src/transport/libp2p/book.rs new file mode 100644 index 0000000..0c61714 --- /dev/null +++ b/src/transport/libp2p/book.rs @@ -0,0 +1,779 @@ +//! The book: who is where (peers' addresses, connections and their round trips), the +//! relays and their slots, hole punching and upgrades, and the counts the figures show. + +use super::*; + +/// Who named a relay to pick, in the order they are picked. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Named { + /// `Libp2pConfig::relays`. + Configured, + /// The relays listed in the document (`Transport::learn_relays`). + Listed, +} + +impl Named { + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Configured => "configured", + Self::Listed => "listed", + } + } +} + +/// A relay named to pick. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct NamedRelay { + /// Where, with the relay's peer ID. + pub(super) at: Multiaddr, + /// As named: without the peer ID, it is found out again when the relay doesn't + /// grant a slot (it may have a new one). + pub(super) given: Multiaddr, + pub(super) named: Named, +} + +/// Where peers are, and where we listen. +#[derive(Default)] +pub(super) struct Book { + pub(super) addresses: HashMap<PeerId, Vec<Multiaddr>>, + /// Where connected peers say they are, by their last identify (`identified`): while + /// connected, the only direct addresses of theirs kept. Forgotten when the last + /// connection to the peer closes, as a peer restarted elsewhere is found by what + /// the others say. + pub(super) identified: HashMap<PeerId, Vec<Multiaddr>>, + /// Addresses of directly connected peers dialled for a better path (`explore`): + /// not again while connected, unless our own interfaces change. + pub(super) explored: HashMap<PeerId, std::collections::HashSet<Multiaddr>>, + /// Addresses where a dial for a peer reached another (`wrongPeerId`), say a NATed + /// peer's LAN address in a range our LAN uses too: not that peer's, whatever + /// identify or the member list say, so not kept, dialled or explored again. As long + /// as the peer is known (`prune`), `WRONG_PER_PEER` each. + pub(super) wrong: HashMap<PeerId, std::collections::HashSet<Multiaddr>>, + pub(super) listening: Vec<Multiaddr>, + /// Where the router forwards to our listen addresses (UPnP, `Libp2pConfig::upnp`). + pub(super) mapped: Vec<Multiaddr>, + /// `Libp2pConfig::announce`. + pub(super) announced: Vec<Multiaddr>, + /// `Libp2pConfig::ignore`: not published (but announced), not dialled. + pub(super) ignored: Vec<IpRange>, + /// `Libp2pConfig::prefer`: addresses there first (`preference`). + pub(super) preferred: Vec<IpRange>, + /// Which address ranges have a route from here (`routes`): those without are left + /// out, as if ignored. + pub(super) routes: Routes, + /// `Libp2pConfig::relay`: this node relays, and says so. + pub(super) serving: bool, + /// Relays named to pick (configured, listed), by peer: picked first. + pub(super) named_relays: HashMap<PeerId, NamedRelay>, + /// Relays named without their peer ID, being found out (`Command::FindRelay`). + pub(super) finding: HashMap<Multiaddr, Named>, + /// The relays listed in the document, as last learned (`learn_listed`). + pub(super) listed: Vec<Multiaddr>, + /// Members that say they relay, and where (with the relay's peer ID). + pub(super) relays: HashMap<PeerId, Multiaddr>, + /// With `Libp2pConfig::public`: other peers that relay (identify says so), and + /// where. + pub(super) public_relays: HashMap<PeerId, Multiaddr>, + /// Relays not to be picked, until when (`RELAY_PAUSE`, `RELAY_RETRY`). + pub(super) failed_relays: HashMap<PeerId, tokio::time::Instant>, + /// Relays that granted us a slot we hold. + pub(super) slots: std::collections::HashSet<PeerId>, + /// Relays we hold a slot on whose duplicate connections we closed, and when + /// (`duplicates`): a slot lost then moves to the connection kept, it isn't lost. + pub(super) rehoming: HashMap<PeerId, tokio::time::Instant>, + /// Peers holding a slot on this node, as a relay. + pub(super) slots_here: std::collections::HashSet<PeerId>, + /// Peers connected only through a relay, dialled directly (`upgrade`): when to again + /// at the soonest, and how long to wait after that. + pub(super) upgrades: HashMap<PeerId, (tokio::time::Instant, Duration)>, + /// The shortest round trip measured on each open connection (libp2p's ping), since + /// the last connection to the same peer opened (they are compared then). + pub(super) rtts: HashMap<ConnectionId, Duration>, + /// Peers whose hole punching failed: how often in a row, and when to try again + /// (`PUNCH_RETRIES`). + pub(super) punching: HashMap<PeerId, (usize, tokio::time::Instant)>, + /// Open connections: to whom, whether through a relay, and the remote address. + pub(super) connections: HashMap<ConnectionId, (PeerId, bool, Multiaddr)>, + /// What AutoNAT says, and the address it confirmed while public. + pub(super) reachability: Reachability, + pub(super) public_address: Option<Multiaddr>, + /// Since start (`stats`). + pub(super) counts: Counts, + /// When each peer was last mentioned (`STALE_AFTER`). + pub(super) seen: HashMap<PeerId, tokio::time::Instant>, + /// Peers found on the LAN (mDNS) not known yet to speak continuum. + pub(super) lan: std::collections::HashSet<PeerId>, + /// Peers that speak continuum (identify says so). + pub(super) continuum: std::collections::HashSet<PeerId>, + /// Peers identify says don't speak continuum, and whether they relay: not sent to + /// (`send_encoded`), so that a seed that's no node says so. + pub(super) not_nodes: HashMap<PeerId, bool>, + /// Peers found on the LAN that speak continuum, with their address: seeds not + /// handed out yet (`discovered_seeds`). + pub(super) discovered: Vec<Multiaddr>, + /// Told when a seed is found (`Transport::on_seeds_found`). + pub(super) seeds_found: Option<Arc<tokio::sync::Notify>>, + /// Told when others may reach this node where they couldn't before + /// (`Transport::on_reachable`). + pub(super) reachable: Option<Arc<tokio::sync::Notify>>, +} + +/// What `Libp2pStats` counts since start. +#[derive(Default)] +pub(super) struct Counts { + pub(super) slots_granted: u64, + pub(super) slots_lost: u64, + pub(super) slots_not_granted: u64, + pub(super) hole_punches: HolePunches, + pub(super) dial_failures: DialFailures, +} + +impl Book { + /// How we are connected to `peer`: directly if any connection is direct. + pub(super) fn connection(&self, peer: &PeerId) -> Option<Connection> { + book_connection(&self.connections, peer) + } + + /// The connections to `peer` to close, of several: all but the one to keep + /// (`ranked`). + /// + /// Between a relay and a node with a slot on it too: messages go over any of the + /// connections (libp2p-stream picks one at random for each), so a slower one kept + /// would carry half of them. A slot lives on one connection, which libp2p doesn't + /// tell; if that one closes, the slot ends, and the node reserves it again at once, + /// over the connection kept (`Book::rehoming`, not counted as lost); circuits + /// through it end, and are made again with the next message. + pub(super) fn duplicates(&self, peer: &PeerId) -> Vec<ConnectionId> { + self.ranked(peer).into_iter().skip(1).collect() + } + + /// The connections to `peer`, the one to keep first: direct ones before relayed + /// ones, measured ones by their round trip (`rtts`, the shortest since the last of + /// them opened: over the LAN before a VPN's hub, whichever opened first), then the + /// oldest. + pub(super) fn ranked(&self, peer: &PeerId) -> Vec<ConnectionId> { + let mut connections: Vec<(bool, Duration, ConnectionId)> = self + .connections + .iter() + .filter(|(_, (connected, _, _))| connected == peer) + .map(|(id, (_, relayed, _))| { + let rtt = self.rtts.get(id).copied().unwrap_or(Duration::MAX); + (*relayed, rtt, *id) + }) + .collect(); + connections.sort(); + connections.into_iter().map(|(_, _, id)| id).collect() + } + + /// The remote address of the connection to `peer` that is kept (`ranked`). + pub(super) fn location(&self, peer: &PeerId) -> Option<&Multiaddr> { + let kept = *self.ranked(peer).first()?; + self.connections.get(&kept).map(|(_, _, address)| address) + } + + pub(super) fn stats(&self) -> Libp2pStats { + let mut connections = Connections::default(); + for (_, relayed, _) in self.connections.values() { + if *relayed { + connections.relayed += 1; + } else { + connections.direct += 1; + } + } + Libp2pStats { + reachability: self.reachability, + public_address: self.public_address.as_ref().map(Multiaddr::to_string), + connections, + relay_slots: RelaySlots { + held: self.slots.len(), + paused: self + .failed_relays + .values() + .filter(|until| tokio::time::Instant::now() < **until) + .count(), + granted: self.counts.slots_granted, + lost: self.counts.slots_lost, + not_granted: self.counts.slots_not_granted, + }, + hole_punches: self.counts.hole_punches, + dial_failures: self.counts.dial_failures, + } + } + + /// `peer` was mentioned just now. + pub(super) fn touch(&mut self, peer: PeerId) { + self.seen.insert(peer, tokio::time::Instant::now()); + } + + /// Forgets the peers not mentioned within `STALE_AFTER` before `now`, but those + /// `connected`. Returns how many. + pub(super) fn prune( + &mut self, + now: tokio::time::Instant, + connected: impl Fn(&PeerId) -> bool, + ) -> usize { + let stale: Vec<PeerId> = self + .addresses + .keys() + .chain(self.relays.keys()) + .chain(self.public_relays.keys()) + .chain(self.continuum.iter()) + .chain(self.lan.iter()) + .filter(|peer| { + !connected(peer) + && self + .seen + .get(peer) + .is_none_or(|at| now.saturating_duration_since(*at) >= STALE_AFTER) + }) + .copied() + .collect::<std::collections::HashSet<_>>() + .into_iter() + .collect(); + for peer in &stale { + self.addresses.remove(peer); + self.identified.remove(peer); + self.explored.remove(peer); + self.wrong.remove(peer); + self.relays.remove(peer); + self.public_relays.remove(peer); + self.continuum.remove(peer); + self.not_nodes.remove(peer); + self.lan.remove(peer); + self.failed_relays.remove(peer); + self.upgrades.remove(peer); + self.punching.remove(peer); + self.seen.remove(peer); + } + stale.len() + } + + /// `peer`, found on the LAN, speaks continuum: a seed, at the address we know first. + pub(super) fn discover(&mut self, peer: PeerId) { + let Some(address) = self.addresses.get(&peer).and_then(|known| known.first()) else { + return; + }; + if let Ok(address) = address.clone().with_p2p(peer) + && !self.discovered.contains(&address) + && self.discovered.len() < MAX_PEERS + { + self.discovered.push(address); + if let Some(found) = &self.seeds_found { + found.notify_one(); + } + } + } + + /// Where others could dial us directly: the announced addresses, or else those we + /// listen on (`published`), where the router forwards to them (`mapped`), and where + /// AutoNAT found others can dial us (`public_address`: a port forwarded by hand, say), + /// without the peer ID. + pub(super) fn direct(&self) -> Vec<Multiaddr> { + if !self.announced.is_empty() { + return self.announced.clone(); + } + let direct: Vec<Multiaddr> = self + .listening + .iter() + .filter(|address| !is_relayed(address)) + .cloned() + .collect(); + let mut direct = published(&direct); + let confirmed = self.public_address.iter().filter_map(|address| { + let address = without_peer(address.clone()); + (!address.is_empty() && !is_relayed(&address)).then_some(address) + }); + // First: only so many are published (`PUBLISHED_DIRECT`), and a node listening + // everywhere has many. + let mut first: Vec<Multiaddr> = Vec::new(); + for address in self.mapped.iter().cloned().chain(confirmed) { + if !first.contains(&address) { + first.push(address); + } + } + direct.retain(|address| !first.contains(address)); + direct.sort_by_key(|address| self.preferred_range(address)); + first.extend(direct); + first.retain(|address| { + !ignored(&self.ignored, address) + && ip_of(address).is_none_or(|ip| self.routes.publishable(ip)) + }); + first + } +} + +/// Whether `address` only reaches this host (or this link): no use to other hosts, and +/// dialled by one, it reaches that host itself. +pub(super) fn is_local(address: &Multiaddr) -> bool { + match address.iter().next() { + Some(Protocol::Ip4(ip)) => ip.is_loopback() || ip.is_link_local() || ip.is_unspecified(), + Some(Protocol::Ip6(ip)) => { + ip.is_loopback() || ip.is_unicast_link_local() || ip.is_unspecified() + } + _ => false, + } +} + +/// The addresses of `all` worth publishing: all but local ones, unless they're all +/// local (a test on one host). +pub(super) fn published(all: &[Multiaddr]) -> Vec<Multiaddr> { + let reachable: Vec<Multiaddr> = all + .iter() + .filter(|address| !is_local(address)) + .cloned() + .collect(); + if reachable.is_empty() { + all.to_vec() + } else { + reachable + } +} + +impl Book { + /// `named` names the relay at `given`; returns it if its peer is to be found out. + /// `local`: this node, never its own relay. + pub(super) fn name_relay( + &mut self, + given: Multiaddr, + named: Named, + local: PeerId, + ) -> Option<Multiaddr> { + match split_peer(&given) { + Some(peer) if peer == local => None, + Some(peer) => { + // Configured wins over listed. + if self + .named_relays + .get(&peer) + .is_none_or(|known| known.named == Named::Listed || named == Named::Configured) + { + self.named_relays.insert( + peer, + NamedRelay { + at: given.clone(), + given, + named, + }, + ); + } + None + } + None => { + let known = self.finding.contains_key(&given) + || self.named_relays.values().any(|relay| relay.given == given); + if known { + return None; + } + self.finding.insert(given.clone(), named); + Some(given) + } + } + } + + /// The relay named at `given` (without a peer ID) is `peer`: unless it is no longer + /// named (or is this node). Returns whether that's news. + pub(super) fn relay_found(&mut self, given: Multiaddr, peer: PeerId, local: PeerId) -> bool { + let Some(named) = self.finding.remove(&given) else { + return false; + }; + let Ok(at) = given.clone().with_p2p(peer) else { + return false; + }; + if peer == local + || (named == Named::Listed + && self + .named_relays + .get(&peer) + .is_some_and(|known| known.named == Named::Configured)) + { + return false; + } + self.named_relays + .insert(peer, NamedRelay { at, given, named }); + true + } + + /// `relay` didn't grant a slot: if it was named without its peer ID, its address is + /// to be found out again, returned. + pub(super) fn find_again(&mut self, relay: PeerId) -> Option<Multiaddr> { + if self + .named_relays + .get(&relay) + .is_none_or(|known| split_peer(&known.given).is_some()) + { + return None; + } + let known = self.named_relays.remove(&relay)?; + self.finding.insert(known.given.clone(), known.named); + Some(known.given) + } + + /// The relays listed in the document are `listed` now: returns those to find out, + /// None if nothing changed. + pub(super) fn learn_listed( + &mut self, + listed: Vec<Multiaddr>, + local: PeerId, + ) -> Option<Vec<Multiaddr>> { + if listed == self.listed { + return None; + } + self.named_relays + .retain(|_, relay| relay.named != Named::Listed || listed.contains(&relay.given)); + self.finding + .retain(|given, named| *named != Named::Listed || listed.contains(given)); + self.listed = listed.clone(); + Some( + listed + .into_iter() + .filter_map(|given| self.name_relay(given, Named::Listed, local)) + .collect(), + ) + } + + /// The named relays `named` names, by peer, where. + pub(super) fn named(&self, named: Named) -> HashMap<PeerId, Multiaddr> { + self.named_relays + .iter() + .filter(|(_, relay)| relay.named == named) + .map(|(peer, relay)| (*peer, relay.at.clone())) + .collect() + } + + /// Where `peer` may be reached through the relays we know (named, then members that + /// relay): for a peer known by its ID only (a seed `/p2p/<peer ID>`), which may hold + /// a slot on one of them. `local`: this node, never its own relay. + pub(super) fn through_relays(&mut self, peer: PeerId, local: PeerId) -> Vec<Multiaddr> { + let mut relays: Vec<(PeerId, Multiaddr)> = self + .named_relays + .iter() + .map(|(relay, named)| (*relay, named.at.clone())) + .collect(); + relays.sort_by_key(|(_, address)| address.to_string()); + let mut members: Vec<(PeerId, Multiaddr)> = self + .relays + .iter() + .filter(|(relay, _)| !self.named_relays.contains_key(relay)) + .map(|(relay, address)| (*relay, address.clone())) + .collect(); + members.sort_by_key(|(_, address)| address.to_string()); + relays + .into_iter() + .chain(members) + .filter(|(relay, address)| *relay != local && *relay != peer && !self.left_out(address)) + .map(|(_, address)| address.with(Protocol::P2pCircuit).with(Protocol::P2p(peer))) + .take(ADDRESSES_PER_PEER) + .collect() + } + + /// Whether all of `addresses` are known of `peer` (`remember`). + pub(super) fn knows(&self, peer: PeerId, addresses: &[Multiaddr]) -> bool { + let Some(known) = self.addresses.get(&peer) else { + return false; + }; + addresses + .iter() + .all(|address| known.contains(&without_peer(address.clone()))) + } + + /// `peer` is at `address` (without its `/p2p` part): kept first. Not a direct + /// address the peer's identify left out (`identified`): one it no longer has, or + /// one it doesn't publish (`Libp2pConfig::ignore`). Nor one in a range this node + /// leaves out, directly or through a relay there (`left_out`): never dialled, it + /// would only take a place, and be passed on. + /// A dial for `peer` at `address` reached another peer: `address` isn't `peer`'s. + /// Forgotten, and not taken again (`wrong`). + pub(super) fn not_theirs(&mut self, peer: PeerId, address: Multiaddr) { + let address = without_peer(address); + if let Some(known) = self.addresses.get_mut(&peer) { + known.retain(|known| *known != address); + } + let wrong = self.wrong.entry(peer).or_default(); + if wrong.len() < WRONG_PER_PEER { + wrong.insert(address); + } + } + + pub(super) fn remember(&mut self, peer: PeerId, address: Multiaddr) { + let address = without_peer(address); + if !self.addresses.contains_key(&peer) && self.addresses.len() >= MAX_PEERS { + return; + } + if self.left_out(&address) { + return; + } + if self + .wrong + .get(&peer) + .is_some_and(|wrong| wrong.contains(&address)) + { + return; + } + if self + .identified + .get(&peer) + .is_some_and(|said| is_ip(&address) && !said.contains(&address)) + { + return; + } + self.touch(peer); + let addresses = self.addresses.entry(peer).or_default(); + addresses.retain(|known| *known != address); + addresses.insert(0, address); + addresses.truncate(ADDRESSES_PER_PEER); + } + + /// Whether `address` can only be reached over a range left out: ignored + /// (`left_out`), or without a route from here (`routes`), directly or through a + /// relay there. + pub(super) fn left_out(&mut self, address: &Multiaddr) -> bool { + excluded(&self.ignored, &mut self.routes, address) + } + + /// Where `peer` can be dialled: its known addresses, but those through this node, + /// and those left out (`left_out`). What every dial of ours goes to (connecting, + /// upgrading a relayed connection, retrying a hole punch), so none dials into a + /// range left out. + pub(super) fn usable(&mut self, peer: &PeerId, local: PeerId) -> Vec<Multiaddr> { + let Some(known) = self.addresses.get(peer) else { + return Vec::new(); + }; + let mut usable = dialable(known, local); + usable.retain(|address| !excluded(&self.ignored, &mut self.routes, address)); + usable + } + + /// Asks again which ranges have a route (`Routes::recheck`); the addresses in those + /// that lost theirs go. + pub(super) fn recheck_routes(&mut self) { + let lost = self.routes.recheck(); + if lost.is_empty() { + return; + } + for known in self.addresses.values_mut() { + known.retain(|address| { + ip_of(address).is_none_or(|ip| !lost.iter().any(|range| range.contains(ip))) + }); + } + } + + /// The addresses `explore` dials `peer` at, noted as dialled: a node of the swarm + /// connected directly, at addresses of its not dialled so yet, and with `again` + /// not only those at an IP it isn't connected at (an inbound connection's remote + /// port isn't where the peer listens). + pub(super) fn to_explore(&mut self, peer: PeerId, again: bool) -> Vec<Multiaddr> { + if !self.continuum.contains(&peer) + || book_connection(&self.connections, &peer) != Some(Connection::Direct) + { + return Vec::new(); + } + let used: Vec<Option<std::net::IpAddr>> = self + .connections + .values() + .filter(|(connected, relayed, _)| *connected == peer && !relayed) + .map(|(_, _, address)| ip_of(address)) + .collect(); + let tried = self.explored.entry(peer).or_default(); + let candidates: Vec<Multiaddr> = self + .addresses + .get(&peer) + .into_iter() + .flatten() + .filter(|address| is_ip(address) && !left_out(&self.ignored, address)) + .filter(|address| again || !used.contains(&ip_of(address))) + .filter(|address| !tried.contains(*address)) + .cloned() + .collect(); + let candidates: Vec<Multiaddr> = candidates + .into_iter() + .filter(|address| !self.left_out(address)) + .take(EXPLORED_AT_ONCE) + .collect(); + self.explored + .entry(peer) + .or_default() + .extend(candidates.iter().cloned()); + candidates + } + + /// `peer`, connected, identified itself at `listen` (`published`): its direct + /// addresses are those from now on, while connected (`remember`); those through + /// relays and by name stay. Kept in order of `preference`: of a peer with many, + /// the ones that matter stay (`ADDRESSES_PER_PEER`). + pub(super) fn identified(&mut self, peer: PeerId, listen: Vec<Multiaddr>) { + let mut listen: Vec<Multiaddr> = listen.into_iter().map(without_peer).collect(); + listen.sort_by_key(|address| self.preference(address)); + if let Some(known) = self.addresses.get_mut(&peer) { + known.retain(|address| !is_ip(address) || listen.contains(address)); + } + self.identified.insert(peer, listen.clone()); + // Last first: `remember` puts each in front. + for address in listen.into_iter().rev() { + self.remember(peer, address); + } + } +} + +impl Book { + /// Where a peer's `address` comes, first first: in a range `libp2p-prefer` names + /// (by the range's place), then public, then through a relay or by name, then a + /// private one (another LAN, a container bridge). + fn preference(&self, address: &Multiaddr) -> (u8, usize) { + let range = self.preferred_range(address); + if range != usize::MAX { + return (0, range); + } + match ip_of(address).filter(|_| is_ip(address)) { + Some(ip) if is_public(ip) => (1, 0), + Some(_) => (3, 0), + None => (2, 0), + } + } + + /// The place of the first range of `libp2p-prefer` that has `address`; for an + /// address none has, `usize::MAX`: after them. + fn preferred_range(&self, address: &Multiaddr) -> usize { + ip_of(address) + .filter(|_| is_ip(address)) + .and_then(|ip| self.preferred.iter().position(|range| range.contains(ip))) + .unwrap_or(usize::MAX) + } +} + +/// The IP address `address` starts with, if any. +pub(super) fn ip_of(address: &Multiaddr) -> Option<std::net::IpAddr> { + match address.iter().next() { + Some(Protocol::Ip4(ip)) => Some(ip.into()), + Some(Protocol::Ip6(ip)) => Some(ip.into()), + _ => None, + } +} + +/// Whether `ip` can be reached from the internet, as far as its range tells. For IPv6, +/// only global unicast (`2000::/3`) is: other ranges in use are an overlay's +/// (Mycelium's `400::/7`), a translator's (`64:ff9b::/96`) or local, and, ranked as +/// public, would be preferred over the LAN (`preference`). +pub(super) fn is_public(ip: std::net::IpAddr) -> bool { + use std::net::IpAddr; + match ip { + IpAddr::V4(ip) => { + let [a, b, c, _] = ip.octets(); + // 100.64.0.0/10: carrier-grade NAT. + let shared = a == 100 && (b & 0xc0) == 64; + // 198.18.0.0/15: benchmarking; 240.0.0.0/4: reserved (and broadcast). + let benchmarking = a == 198 && (b & 0xfe) == 18; + let reserved = a >= 240; + // 192.0.0.0/24: protocol assignments. + let protocol = a == 192 && b == 0 && c == 0; + !(ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_unspecified() + || ip.is_documentation() + || ip.is_multicast() + || shared + || benchmarking + || reserved + || protocol) + } + IpAddr::V6(ip) => { + let [first, second, ..] = ip.segments(); + // 2001:db8::/32: documentation. + let documentation = first == 0x2001 && second == 0x0db8; + (first & 0xe000) == 0x2000 && !documentation + } + } +} + +/// Whether `address` is a direct one at an IP address (not through a relay, nor a name). +pub(super) fn is_ip(address: &Multiaddr) -> bool { + matches!( + address.iter().next(), + Some(Protocol::Ip4(_) | Protocol::Ip6(_)) + ) && !is_relayed(address) +} + +/// Whether `address` goes through a relay. +pub(super) fn is_relayed(address: &Multiaddr) -> bool { + address + .iter() + .any(|protocol| matches!(protocol, Protocol::P2pCircuit)) +} + +/// The relay `address` goes through, if it goes through one: the peer before +/// `/p2p-circuit`. +pub(super) fn relay_of(address: &Multiaddr) -> Option<PeerId> { + let mut relay = None; + for protocol in address.iter() { + match protocol { + Protocol::P2p(peer) => relay = Some(peer), + Protocol::P2pCircuit => return relay, + _ => {} + } + } + None +} + +/// How `peer` is connected, of `connections` (`Book::connections`): directly if one is +/// direct. +pub(super) fn book_connection( + connections: &HashMap<ConnectionId, (PeerId, bool, Multiaddr)>, + peer: &PeerId, +) -> Option<Connection> { + let mut found = None; + for (connected, relayed, _) in connections.values() { + if connected == peer { + if !relayed { + return Some(Connection::Direct); + } + found = Some(Connection::Relayed); + } + } + found +} + +/// The addresses of `known` this node can dial: not those through itself as the relay +/// (a relay reaches the nodes it relays for only while they are connected to it). +/// `Book::left_out`, by the fields it needs, for where the book is borrowed in parts. +pub(super) fn excluded(ignored: &[IpRange], routes: &mut Routes, address: &Multiaddr) -> bool { + if left_out(ignored, address) { + return true; + } + let at = if is_relayed(address) { + address + .iter() + .take_while(|protocol| !matches!(protocol, Protocol::P2pCircuit)) + .collect::<Multiaddr>() + } else { + address.clone() + }; + ip_of(&at).is_some_and(|ip| !routes.routable(ip)) +} + +pub(super) fn dialable(known: &[Multiaddr], local: PeerId) -> Vec<Multiaddr> { + known + .iter() + .filter(|address| relay_of(address) != Some(local)) + .cloned() + .collect() +} + +/// `address` without the peer ID (`/p2p` part) it ends in, if it ends in one. +pub(super) fn without_peer(mut address: Multiaddr) -> Multiaddr { + if let Some(Protocol::P2p(_)) = address.iter().last() { + address.pop(); + } + address +} + +/// `address` with the peer ID it ends in, if it ends in one. +pub(super) fn split_peer(address: &Multiaddr) -> Option<PeerId> { + match address.iter().last() { + Some(Protocol::P2p(peer)) => Some(peer), + _ => None, + } +} + +pub(super) fn lock(book: &Mutex<Book>) -> MutexGuard<'_, Book> { + book.lock_or_recover() +} diff --git a/src/transport/libp2p/config.rs b/src/transport/libp2p/config.rs new file mode 100644 index 0000000..712f2e2 --- /dev/null +++ b/src/transport/libp2p/config.rs @@ -0,0 +1,177 @@ +//! What the options make of the libp2p transport (`Libp2pConfig`), and their checks. + +use super::*; + +/// What a libp2p transport is made with. +pub(crate) struct Libp2pConfig { + pub(crate) key: Keypair, + /// Multiaddrs to listen on. + pub(crate) listen: Vec<Multiaddr>, + /// Multiaddrs to publish instead of the listen addresses, if any. + pub(crate) announce: Vec<Multiaddr>, + /// Relays to reserve a slot on (multiaddrs; without the relay's peer ID, a connection + /// finds it out). + pub(crate) relays: Vec<Multiaddr>, + /// Relay for other nodes. + pub(crate) relay: bool, + /// Speak continuum (`PROTOCOL`): not on a node that only relays (`--relay-only`), + /// which nobody should take for a node. + pub(crate) continuum: bool, + /// Find peers on the LAN (mDNS). + pub(crate) mdns: bool, + /// Ask the router to forward our listen ports (UPnP), and publish where. + pub(crate) upnp: bool, + /// Address ranges neither published nor dialled, nor connected over (`Ignoring`). + pub(crate) ignore: Vec<IpRange>, + /// Address ranges that matter most, best first (`Book::preference`). + pub(crate) prefer: Vec<IpRange>, + /// Hosts on the internet (names or addresses) whose routes tell whether there is + /// one to the internet at all (`Routes::internet`). + pub(crate) internet_hosts: Vec<String>, + /// Use public infrastructure: the IPFS DHT, public relays. + pub(crate) public: bool, + /// libp2p's in-memory transport instead of TCP and QUIC: for tests. + pub(crate) memory: bool, + /// The node's keepalive interval (`--keepalive`): connections idle for a few of + /// them are closed (`idle_timeout`). + pub(crate) keepalive: Duration, +} + +/// The multiaddrs of option `option`. +pub(super) fn parse_addresses(option: &str, addresses: &[String]) -> Result<Vec<Multiaddr>> { + addresses + .iter() + .map(|address| { + address + .parse::<Multiaddr>() + .with_context(|| format!("{option} {address:?}")) + }) + .collect() +} + +/// The ranges of option `option`. +pub(super) fn parse_ranges(option: &str, ranges: &[String]) -> Result<Vec<IpRange>> { + ranges + .iter() + .map(|range| range.parse().with_context(|| option.to_owned())) + .collect() +} + +/// This transport's options, checked when the configuration is loaded +/// (`config::Settings::check`), so `--print-config` tells a wrong one too: the +/// multiaddrs and ranges parse, and a relay that is no node (`--relay-only`) can be +/// dialled and keeps its peer ID. +pub(crate) fn check(settings: &crate::config::Settings) -> Result<()> { + parse_addresses("libp2p-listen", &settings.libp2p_listen)?; + parse_addresses("libp2p-announce", &settings.libp2p_announce)?; + parse_addresses("libp2p-relays", &settings.libp2p_relays)?; + let ignore = parse_ranges("libp2p-ignore", &settings.libp2p_ignore)?; + let prefer = parse_ranges("libp2p-prefer", &settings.libp2p_prefer)?; + check_preferred( + &ignore, + &prefer, + &settings.libp2p_ignore, + &settings.libp2p_prefer, + )?; + if settings.libp2p_key_create && settings.libp2p_key_file.is_none() { + bail!("libp2p-key-create creates the key at libp2p-key-file; set that too"); + } + if settings.relay_only { + if settings.libp2p_listen.is_empty() { + bail!("relay-only needs libp2p-listen: where nodes reach the relay"); + } + if settings.libp2p_key_file.is_none() { + bail!( + "relay-only needs libp2p-key-file: without it, the relay's peer ID changes with every start and nodes lose it" + ); + } + } + Ok(()) +} + +/// A range both ignored and preferred: refused where all of the preferred one is +/// ignored (it would name nothing usable), a warning where only part of it is (the +/// ignored part stays left out). `ignore_given` and `prefer_given`: as written, for +/// the messages. Ranges nest or are apart, so there's nothing in between. +pub(super) fn check_preferred( + ignore: &[IpRange], + prefer: &[IpRange], + ignore_given: &[String], + prefer_given: &[String], +) -> Result<()> { + for (preferred, preferred_given) in prefer.iter().zip(prefer_given) { + for (ignored, ignored_given) in ignore.iter().zip(ignore_given) { + if ignored.covers(preferred) { + bail!( + "libp2p-prefer {preferred_given:?} is all in libp2p-ignore {ignored_given:?}: nothing there is used" + ); + } + if preferred.covers(ignored) { + warn!( + "libp2p-prefer {preferred_given:?} has libp2p-ignore {ignored_given:?} in it: addresses there stay left out" + ); + } + } + } + Ok(()) +} + +impl Libp2pConfig { + /// The configuration `settings` give; `relay_only`: a relay that is no node + /// (`--relay-only`). + pub(crate) fn from_settings( + settings: &crate::config::Settings, + relay_only: bool, + ) -> Result<Self> { + let parse = parse_addresses; + Ok(Self { + key: Self::load_key( + settings.libp2p_key_file.as_deref(), + settings.libp2p_key_create, + )?, + listen: parse("libp2p-listen", &settings.libp2p_listen)?, + announce: parse("libp2p-announce", &settings.libp2p_announce)?, + relays: parse("libp2p-relays", &settings.libp2p_relays)?, + relay: settings.libp2p_relay || relay_only, + continuum: !relay_only, + mdns: settings.libp2p_mdns, + upnp: settings.libp2p_upnp, + ignore: parse_ranges("libp2p-ignore", &settings.libp2p_ignore)?, + prefer: parse_ranges("libp2p-prefer", &settings.libp2p_prefer)?, + internet_hosts: settings.libp2p_internet_hosts.clone(), + public: settings.libp2p_public, + memory: false, + keepalive: settings.keepalive, + }) + } + + /// The key in `path` (32 raw bytes, an Ed25519 secret key), created there with + /// `create` if there is no file; or a new one each start without a path: the peer + /// ID only says where a node is, so a new one costs nothing but the WHO that finds + /// it again (`transport::identity`). + pub(crate) fn load_key(path: Option<&Path>, create: bool) -> Result<Keypair> { + let Some(path) = path else { + return Ok(Keypair::generate_ed25519()); + }; + let (mut bytes, created) = crate::files::key_file( + path, + "libp2p key", + create, + "libp2p-key-create", + "a new peer ID", + 32, + )?; + if created { + info!(path = %path.display(), "Created a new libp2p key"); + } + if bytes.len() != 32 { + bail!( + "The libp2p key in {} has {} bytes; it has 32", + path.display(), + bytes.len() + ); + } + Keypair::ed25519_from_bytes(&mut bytes) + .map_err(|error| anyhow!("The libp2p key in {}: {error}", path.display())) + } +} diff --git a/src/transport/libp2p/driver.rs b/src/transport/libp2p/driver.rs new file mode 100644 index 0000000..6ef04e7 --- /dev/null +++ b/src/transport/libp2p/driver.rs @@ -0,0 +1,624 @@ +//! The swarm's task (`run`): it owns the swarm, takes `Command`s from the transport, and +//! answers the swarm's events (connections, identify, relays, hole punching, AutoNAT, +//! mDNS, UPnP, the DHT), keeping the book. + +use super::*; + +pub(super) enum Command { + /// Connect to `peer` (at `addresses`) unless connected. + Connect { + peer: PeerId, + addresses: Vec<Multiaddr>, + done: oneshot::Sender<Result<(), String>>, + }, + /// Connect to `address` to find out which peer is there. + Identify { + address: Multiaddr, + done: oneshot::Sender<Result<PeerId, String>>, + }, + /// See whether relays are needed now (one was just learned of). + CheckRelays, + /// Find out which peer relays at `address`, named without the relay's peer ID + /// (`Book::finding`). + FindRelay { address: Multiaddr }, + /// Where `--libp2p-internet-hosts` are now (`Routes::internet`). + Internet(Vec<std::net::IpAddr>), +} + +/// Drives the swarm: dials what `commands` ask for and notes where we listen, until the +/// transport is dropped. `again`: to schedule commands. +pub(super) async fn run( + mut swarm: Swarm<Behaviour>, + mut commands: mpsc::UnboundedReceiver<Command>, + again: mpsc::UnboundedSender<Command>, + book: Arc<Mutex<Book>>, + relaying: Relaying, + stopping: watch::Sender<bool>, +) { + let serving = relaying.serving; + let mut nat = ::libp2p::autonat::NatStatus::Unknown; + // Relays picked, by listener. + let mut picked: HashMap<::libp2p::core::transport::ListenerId, PeerId> = HashMap::new(); + let mut check = tokio::time::interval(RELAY_CHECK); + check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut dht_bootstrap = tokio::time::interval(DHT_BOOTSTRAP); + dht_bootstrap.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut connecting: HashMap<PeerId, Vec<oneshot::Sender<Result<(), String>>>> = HashMap::new(); + let mut identifying: HashMap< + ConnectionId, + (Multiaddr, oneshot::Sender<Result<PeerId, String>>), + > = HashMap::new(); + let mut prune = tokio::time::interval(PRUNE_EVERY); + prune.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // Peers that may have several connections, and when to cut them to one. Only the + // side with the smaller peer ID does that, so the two never close different ones. + let mut dedupe_due: HashMap<PeerId, tokio::time::Instant> = HashMap::new(); + // Connections closed as duplicates, until they are: their "Connection closed" says so. + let mut closing_duplicates: std::collections::HashSet<ConnectionId> = + std::collections::HashSet::new(); + let mut dedupe = tokio::time::interval(DEDUPE_EVERY); + dedupe.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + _ = dedupe.tick(), if !dedupe_due.is_empty() => { + let now = tokio::time::Instant::now(); + let due: Vec<PeerId> = dedupe_due + .iter() + .filter(|(_, at)| **at <= now) + .map(|(peer, _)| *peer) + .collect(); + for peer in due { + dedupe_due.remove(&peer); + let reserving = picked.values().any(|relay| *relay == peer); + let duplicates = { + let mut book = lock(&book); + let duplicates = book.duplicates(&peer); + if !duplicates.is_empty() && (reserving || book.slots.contains(&peer)) { + book.rehoming.insert(peer, now); + } + duplicates + }; + for connection in duplicates { + if swarm.close_connection(connection) { + closing_duplicates.insert(connection); + } + } + } + } + _ = prune.tick() => { + let forgotten = lock(&book).prune(tokio::time::Instant::now(), |peer| swarm.is_connected(peer)); + if forgotten > 0 { + debug!(forgotten, "Forgot peers nobody mentions any more"); + } + } + _ = dht_bootstrap.tick(), if relaying.public => { + if let Some(dht) = swarm.behaviour_mut().kad.as_mut() + && let Err(error) = dht.bootstrap() + { + warn!(error = %error, "Couldn't join the IPFS DHT"); + } + } + _ = check.tick() => { + let hosts = { + let mut book = lock(&book); + book.recheck_routes(); + book.routes.internet_hosts.clone() + }; + // Names are resolved aside, not to hold up the swarm. + if !hosts.is_empty() { + let again = again.clone(); + tokio::spawn(async move { + let _ = again.send(Command::Internet(resolve_internet_hosts(&hosts).await)); + }); + } + pick_relays(&mut swarm, &book, &relaying, &nat, &mut picked); + retry_punches(&mut swarm, &book, relaying.local); + } + command = commands.recv() => match command { + None => break, + Some(Command::CheckRelays) => { + pick_relays(&mut swarm, &book, &relaying, &nat, &mut picked); + } + Some(Command::Internet(hosts)) => lock(&book).routes.internet(&hosts), + Some(Command::Connect { peer, addresses, done }) => { + if swarm.is_connected(&peer) { + let _ = done.send(Ok(())); + upgrade(&mut swarm, &book, peer, relaying.local); + continue; + } + let waiting = connecting.entry(peer).or_default(); + waiting.push(done); + if waiting.len() > 1 { + continue; // Already dialing. + } + let mut opts = DialOpts::peer_id(peer) + .addresses(addresses) + .condition(PeerCondition::DisconnectedAndNotDialing); + if relaying.public { + // Addresses the DHT knows too. + opts = opts.extend_addresses_through_behaviour(); + } + if let Err(error) = swarm.dial(opts.build()) { + count_dial_failure(&mut lock(&book).counts.dial_failures, &error); + for done in connecting.remove(&peer).into_iter().flatten() { + let _ = done.send(Err(error.to_string())); + } + find_on_dht(&mut swarm, peer); + } + } + Some(Command::FindRelay { address }) => { + let again = again.clone(); + let local = relaying.local; + let book = Arc::clone(&book); + tokio::spawn(async move { + let (done, identified) = oneshot::channel(); + if again + .send(Command::Identify { address: address.clone(), done }) + .is_err() + { + return; + } + match tokio::time::timeout(CONNECT_TIMEOUT, identified).await { + Ok(Ok(Ok(peer))) => { + if lock(&book).relay_found(address.clone(), peer, local) { + debug!(relay = %Addressed(SCHEME, &address), peer = %Addressed(SCHEME, &peer), "Found a named relay"); + let _ = again.send(Command::CheckRelays); + } + } + failed => { + if !lock(&book).finding.contains_key(&address) { + return; // No longer named. + } + let error = match failed { + Ok(Ok(Err(error))) => error, + Ok(_) => "the libp2p swarm stopped".to_owned(), + Err(_) => format!("no answer within {CONNECT_TIMEOUT:?}"), + }; + warn!(relay = %Addressed(SCHEME, &address), error = %error, retry = ?RELAY_RETRY, "Couldn't reach a named relay; trying again later"); + tokio::time::sleep(RELAY_RETRY).await; + let _ = again.send(Command::FindRelay { address }); + } + } + }); + } + Some(Command::Identify { address, done }) => { + let opts = DialOpts::unknown_peer_id().address(address.clone()).build(); + let id = opts.connection_id(); + match swarm.dial(opts) { + Ok(()) => { + identifying.insert(id, (address, done)); + } + Err(error) => { + let _ = done.send(Err(error.to_string())); + } + } + } + }, + event = swarm.select_next_some() => match event { + SwarmEvent::ConnectionEstablished { peer_id, connection_id, endpoint, .. } => { + { + let mut book = lock(&book); + book.touch(peer_id); + let relayed = endpoint.is_relayed(); + book.connections.insert( + connection_id, + (peer_id, relayed, endpoint.get_remote_address().clone()), + ); + if !relayed { + book.upgrades.remove(&peer_id); + book.punching.remove(&peer_id); + } + let to_peer: Vec<ConnectionId> = book + .connections + .iter() + .filter(|(_, (connected, _, _))| *connected == peer_id) + .map(|(id, _)| *id) + .collect(); + if to_peer.len() > 1 { + // A new path to compare: each is judged by its round trips + // from now on, not an old one by its best over hours + // (`Book::ranked`). + for connection in &to_peer { + book.rtts.remove(connection); + } + } + if relaying.local < peer_id && to_peer.len() > 1 { + // Only through relays: maybe a punch under way. + let grace = match book.connection(&peer_id) { + Some(Connection::Relayed) => PUNCH_GRACE, + _ => DUPLICATE_GRACE, + }; + dedupe_due.insert(peer_id, tokio::time::Instant::now() + grace); + } + debug!( + peer = %Addressed(SCHEME, &peer_id), + connection = %connection_id, + dialled = endpoint.is_dialer(), + address = %endpoint.get_remote_address(), + relayed, + "Connection opened" + ); + } + for done in connecting.remove(&peer_id).into_iter().flatten() { + let _ = done.send(Ok(())); + } + if let Some((address, done)) = identifying.remove(&connection_id) { + lock(&book).remember(peer_id, address); + let _ = done.send(Ok(peer_id)); + } + } + SwarmEvent::ConnectionClosed { peer_id, connection_id, cause, .. } => { + { + let mut book = lock(&book); + book.connections.remove(&connection_id); + book.rtts.remove(&connection_id); + if book.connection(&peer_id).is_none() { + book.identified.remove(&peer_id); + book.explored.remove(&peer_id); + } + } + let cause = match cause { + _ if closing_duplicates.remove(&connection_id) => "a duplicate".to_owned(), + // By us, or idle: libp2p doesn't tell which. + None => "closed here".to_owned(), + Some(error) => error.to_string(), + }; + debug!( + peer = %Addressed(SCHEME, &peer_id), + connection = %connection_id, + cause = %cause, + "Connection closed" + ); + } + SwarmEvent::OutgoingConnectionError { connection_id, peer_id, error } => { + count_dial_failure(&mut lock(&book).counts.dial_failures, &error); + // Another peer there: not this one's address, though it said so (a + // NATed peer's LAN address, the same range as ours). + if let (Some(peer), ::libp2p::swarm::DialError::WrongPeerId { obtained, address }) = + (peer_id, &error) + { + debug!( + peer = %Addressed(SCHEME, &peer), + address = %address, + obtained = %obtained, + "Another peer at an address of this one; not dialling it there again" + ); + lock(&book).not_theirs(peer, address.clone()); + } + if let Some((_, done)) = identifying.remove(&connection_id) { + let _ = done.send(Err(describe_dial_error(&error))); + } + if let Some(peer) = peer_id + && !swarm.is_connected(&peer) + && let Some(waiting) = connecting.remove(&peer) + { + for done in waiting { + let _ = done.send(Err(describe_dial_error(&error))); + } + find_on_dht(&mut swarm, peer); + } + // A dial that kept ours from going (`upgrade`) is over. + if let Some(peer) = peer_id { + upgrade(&mut swarm, &book, peer, relaying.local); + } + } + SwarmEvent::NewListenAddr { address, .. } => { + // libp2p listens on every interface it's told to; the ranges left + // out are only kept out of what's published, told and connected. + let left_out = ignored(&lock(&book).ignored, &address); + if left_out { + debug!(address = %address, "Listening, but left out (libp2p-ignore): neither published nor connected over"); + } else { + info!(address = %address, "Listening"); + } + // A relay tells those reserving where it is: its external addresses. + if serving && !left_out && !is_local(&address) && !is_relayed(&address) { + swarm.add_external_address(address.clone()); + } + let new_path = !left_out && !is_local(&address) && !is_relayed(&address); + { + let mut book = lock(&book); + if !book.listening.contains(&address) { + book.listening.push(address); + } + // A new interface (ethernet besides the WLAN, say): every + // address may be reached better now. + if new_path { + book.explored.clear(); + } + } + if new_path { + let peers: Vec<PeerId> = swarm.connected_peers().copied().collect(); + for peer in peers { + explore(&mut swarm, &book, peer, true); + } + } + } + SwarmEvent::ExpiredListenAddr { address, .. } => { + info!(address = %address, "No longer listening"); + if serving { + swarm.remove_external_address(&address); + } + lock(&book).listening.retain(|known| *known != address); + } + SwarmEvent::ListenerClosed { listener_id, addresses, reason } => { + lock(&book) + .listening + .retain(|known| !addresses.contains(known)); + // A relay picked: the next check picks again, another one if this one + // never granted the slot. + if let Some(relay) = picked.remove(&listener_id) { + let refused = match &reason { + Ok(()) => false, + Err(error) => relay_refused(&format!("{error:?}"), &innermost_causes(error)), + }; + let (lost, pause, rehomed) = { + let mut book = lock(&book); + let lost = book.slots.remove(&relay); + // Its connection closed as a duplicate (`duplicates`): again + // at once, over the one kept. + let rehomed = lost + && book.rehoming.remove(&relay).is_some_and(|at| { + tokio::time::Instant::now().saturating_duration_since(at) + < REHOMING_WITHIN + }); + let pause = match (refused, lost) { + (true, _) => Some(RELAY_PAUSE), + (false, true) => None, + (false, false) => Some(RELAY_RETRY), + }; + if lost { + if !rehomed { + book.counts.slots_lost += 1; + } + } else { + book.counts.slots_not_granted += 1; + // Named without its peer ID: maybe another one now. + if let Some(address) = book.find_again(relay) { + let _ = again.send(Command::FindRelay { address }); + } + } + if let Some(pause) = pause { + book.failed_relays.insert(relay, tokio::time::Instant::now() + pause); + } + (lost, pause, rehomed) + }; + // Again at once, or others meanwhile. + let _ = again.send(Command::CheckRelays); + let reason = relay_failure(&reason); + if rehomed { + debug!(relay = %Addressed(SCHEME, &relay), "A duplicate connection to a relay closed with our slot on it; reserving again over the one kept"); + continue; + } + match (lost, pause) { + (true, None) => { + info!(relay = %Addressed(SCHEME, &relay), reason = %reason, "Lost the slot on a relay; reserving again"); + } + (true, Some(pause)) => { + info!(relay = %Addressed(SCHEME, &relay), pause = ?pause, reason = %reason, "Lost the slot on a relay; picking others for a while"); + } + (false, pause) => { + info!( + relay = %Addressed(SCHEME, &relay), + pause = ?pause.unwrap_or_default(), + reason = %reason, + "A relay didn't grant a slot; picking others for a while" + ); + } + } + } + } + SwarmEvent::ListenerError { error, .. } => { + warn!(error = %error, "Listener failed"); + } + SwarmEvent::Behaviour(BehaviourEvent::RelayClient( + ::libp2p::relay::client::Event::ReservationReqAccepted { relay_peer_id, renewal, .. }, + )) => { + { + let mut book = lock(&book); + book.slots.insert(relay_peer_id); + book.failed_relays.remove(&relay_peer_id); + if !renewal { + book.counts.slots_granted += 1; + // Peers that couldn't reach us may now, through it. + if let Some(reachable) = &book.reachable { + reachable.notify_one(); + } + } + } + if !renewal { + info!(relay = %Addressed(SCHEME, &relay_peer_id), "Reachable through a relay"); + } + } + SwarmEvent::Behaviour(BehaviourEvent::Autonat( + ::libp2p::autonat::Event::StatusChanged { old, new }, + )) => { + info!(old = ?old, new = ?new, "Reachability (AutoNAT)"); + { + use ::libp2p::autonat::NatStatus; + let mut book = lock(&book); + (book.reachability, book.public_address) = match &new { + NatStatus::Unknown => (Reachability::Unknown, None), + NatStatus::Private => (Reachability::Private, None), + NatStatus::Public(address) => { + (Reachability::Public, Some(address.clone())) + } + }; + if matches!(new, NatStatus::Public(_)) + && !matches!(old, NatStatus::Public(_)) + && let Some(reachable) = &book.reachable + { + reachable.notify_one(); + } + } + nat = new; + } + SwarmEvent::Behaviour(BehaviourEvent::Relay(event)) => { + { + use ::libp2p::relay::Event; + let mut book = lock(&book); + match &event { + Event::ReservationReqAccepted { src_peer_id, .. } => { + book.slots_here.insert(*src_peer_id); + } + Event::ReservationClosed { src_peer_id } + | Event::ReservationTimedOut { src_peer_id } => { + book.slots_here.remove(src_peer_id); + } + _ => {} + } + } + debug!(event = ?event, "Relay event"); + } + SwarmEvent::Behaviour(BehaviourEvent::Ping(::libp2p::ping::Event { + peer, + connection, + result, + })) => match result { + Ok(rtt) => { + let mut book = lock(&book); + if book.connections.contains_key(&connection) { + let shortest = book.rtts.entry(connection).or_insert(rtt); + *shortest = (*shortest).min(rtt); + } + trace!(peer = %Addressed(SCHEME, &peer), connection = %connection, rtt = ?rtt, "Connection's round trip"); + } + Err(error) => { + trace!(peer = %Addressed(SCHEME, &peer), connection = %connection, error = %error, "Connection's ping failed"); + } + }, + SwarmEvent::Behaviour(BehaviourEvent::Upnp(event)) => match event { + ::libp2p::upnp::Event::NewExternalAddr { + local_addr, + external_addr: address, + } => { + info!(address = %address, to = %local_addr, "The router forwards to us (UPnP)"); + let mut book = lock(&book); + if !book.mapped.contains(&address) { + book.mapped.push(address); + } + } + ::libp2p::upnp::Event::ExpiredExternalAddr { + local_addr, + external_addr: address, + } => { + info!(address = %address, to = %local_addr, "The router no longer forwards to us (UPnP)"); + lock(&book).mapped.retain(|mapped| *mapped != address); + } + ::libp2p::upnp::Event::GatewayNotFound => { + info!("No router that maps ports (UPnP) answered"); + } + ::libp2p::upnp::Event::NonRoutableGateway => { + warn!("The router that maps ports (UPnP) is behind another NAT itself; its mappings don't reach us from outside"); + } + }, + SwarmEvent::Behaviour(BehaviourEvent::Dcutr(event)) => match event.result { + Ok(_) => { + let mut book = lock(&book); + book.counts.hole_punches.succeeded += 1; + book.punching.remove(&event.remote_peer_id); + drop(book); + info!(peer = %Addressed(SCHEME, &event.remote_peer_id), "Connected directly to a peer behind a relay"); + } + Err(error) => { + let retry = { + let mut book = lock(&book); + book.counts.hole_punches.failed += 1; + let failed = book + .punching + .get(&event.remote_peer_id) + .map_or(0, |(failed, _)| *failed); + let retry = PUNCH_RETRIES[failed.min(PUNCH_RETRIES.len() - 1)]; + book.punching.insert( + event.remote_peer_id, + (failed + 1, tokio::time::Instant::now() + retry), + ); + retry + }; + debug!(peer = %Addressed(SCHEME, &event.remote_peer_id), error = %error, retry = ?retry, "Couldn't connect directly to a peer behind a relay; the relay carries on"); + } + }, + SwarmEvent::Behaviour(BehaviourEvent::Identify( + ::libp2p::identify::Event::Received { connection_id, peer_id, info }, + )) => { + { + let mut book = lock(&book); + let listen = published(&info.listen_addrs); + // Every new connection identifies again: logged when it says something new. + if book.knows(peer_id, &listen) { + trace!(peer = %Addressed(SCHEME, &peer_id), connection = %connection_id, "Peer identified again"); + } else { + debug!( + peer = %Addressed(SCHEME, &peer_id), + connection = %connection_id, + listen = ?info.listen_addrs, + observed = %info.observed_addr, + agent = %info.agent_version, + "Peer identified" + ); + } + book.identified(peer_id, listen); + if relaying.public + && info.protocols.contains(&::libp2p::relay::HOP_PROTOCOL_NAME) + && (book.public_relays.len() < MAX_PEERS + || book.public_relays.contains_key(&peer_id)) + && let Some(address) = book + .addresses + .get(&peer_id) + .and_then(|known| known.first()) + .and_then(|address| address.clone().with_p2p(peer_id).ok()) + { + book.public_relays.insert(peer_id, address); + } + if !info.protocols.contains(&PROTOCOL) + && (book.not_nodes.len() < MAX_PEERS + || book.not_nodes.contains_key(&peer_id)) + { + let relays = info.protocols.contains(&::libp2p::relay::HOP_PROTOCOL_NAME); + book.not_nodes.insert(peer_id, relays); + } + if info.protocols.contains(&PROTOCOL) { + book.not_nodes.remove(&peer_id); + book.continuum.insert(peer_id); + if book.lan.remove(&peer_id) { + book.discover(peer_id); + } + } + } + upgrade(&mut swarm, &book, peer_id, relaying.local); + explore(&mut swarm, &book, peer_id, false); + } + SwarmEvent::Behaviour(BehaviourEvent::Mdns( + ::libp2p::mdns::Event::Discovered(found), + )) => { + for (peer, address) in found { + if peer == relaying.local { + continue; + } + debug!(peer = %Addressed(SCHEME, &peer), address = %address, "Peer on the LAN"); + let continuum = { + let mut book = lock(&book); + book.remember(peer, address.clone()); + book.continuum.contains(&peer) + }; + upgrade(&mut swarm, &book, peer, relaying.local); + if continuum { + lock(&book).discover(peer); + } else if lock(&book).lan.len() < MAX_PEERS { + // Connected, identify says whether it speaks continuum. + lock(&book).lan.insert(peer); + let _ = swarm.dial( + DialOpts::peer_id(peer) + .addresses(vec![address]) + .condition(PeerCondition::DisconnectedAndNotDialing) + .build(), + ); + } + } + } + _ => {} + }, + } + } + let _ = stopping.send(true); +} diff --git a/src/transport/libp2p/errors.rs b/src/transport/libp2p/errors.rs new file mode 100644 index 0000000..bb6b090 --- /dev/null +++ b/src/transport/libp2p/errors.rs @@ -0,0 +1,127 @@ +//! What libp2p's errors say, for the log and the figures: dial failures, relay +//! reservations refused or lost. + +use super::*; + +/// A dial error, shortly: for each address tried (without the peer ID the caller names +/// already), the innermost cause, once. libp2p's own text nests each transport's errors +/// in the next ("Multiple dial errors occurred: ..."), repeating the cause at each level. +pub(super) fn describe_dial_error(error: &::libp2p::swarm::DialError) -> String { + use ::libp2p::swarm::DialError; + let DialError::Transport(errors) = error else { + return error.to_string(); + }; + let described: Vec<String> = errors + .iter() + .map(|(address, error)| { + let address = without_peer(address.clone()); + format!("{address}: {}", innermost_causes(error)) + }) + .collect(); + if described.is_empty() { + return error.to_string(); + } + described.join("; ") +} + +/// Why a slot on a relay ended, or was never granted, shortly: libp2p nests the +/// relay's answer several types deep (`Left(Left(Left(Reservation(...))))`), and the +/// common cases deserve a sentence. +pub(super) fn relay_failure(reason: &std::io::Result<()>) -> String { + let Err(error) = reason else { + return "closed".to_owned(); + }; + relay_failure_text(&format!("{error:?}"), &innermost_causes(error)) +} + +/// `relay_failure` on the error's debug form, and its innermost cause as text. +pub(super) fn relay_failure_text(debug: &str, innermost: &str) -> String { + let known = [ + ( + "NoAddressesInReservation", + "the relay gave no address to be reached through it", + ), + ("ResourceLimitExceeded", "the relay has no slot left"), + ("PermissionDenied", "the relay doesn't let us reserve"), + ("NoReservation", "the relay holds no slot for us"), + ( + "connection gated", + "the relay refused the connection (connection gated)", + ), + ( + "UnsupportedProtocol", + "the relay doesn't relay (no circuit relay protocol)", + ), + ("Timeout", "the relay didn't answer in time"), + ]; + if let Some((_, said)) = known.iter().find(|(pattern, _)| debug.contains(pattern)) { + return (*said).to_owned(); + } + // The protocol refused, but only after its first data (`swarm_config`). + if innermost == "Failed" { + return "the relay refused the circuit relay protocol (is it a relay?)".to_owned(); + } + innermost.to_owned() +} + +/// Whether a slot ended, or wasn't granted, because the relay refused it (its error's +/// debug form and innermost cause, as for `relay_failure_text`), rather than because it +/// didn't answer or the connection closed: only then is it worth not asking again for a +/// while. +pub(super) fn relay_refused(debug: &str, innermost: &str) -> bool { + let refusals = [ + "NoAddressesInReservation", + "ResourceLimitExceeded", + "PermissionDenied", + "connection gated", + "UnsupportedProtocol", + "WrongPeerId", + // `Ignoring`: in a range left out, whoever dialled. + "IgnoredAddress", + ]; + refusals.iter().any(|pattern| debug.contains(pattern)) || innermost == "Failed" +} + +/// The innermost of `error`'s causes, each distinct one once: those libp2p lists in a +/// "Multiple dial errors occurred:" text (one per line, ` - <cause>: <its cause>...`), +/// or the last of the source chain. +pub(super) fn innermost_causes(error: &(dyn std::error::Error + 'static)) -> String { + let mut innermost = error; + while let Some(source) = innermost.source() { + innermost = source; + } + let text = innermost.to_string(); + let mut causes: Vec<&str> = Vec::new(); + for line in text.lines() { + let Some(listed) = line.trim().strip_prefix("- ") else { + continue; + }; + // A line ends in the next level's heading when that level lists more. + let cause = listed + .split(": ") + .filter(|part| !part.starts_with("Multiple dial errors")) + .last() + .unwrap_or(listed) + .trim(); + if !cause.is_empty() && !causes.contains(&cause) { + causes.push(cause); + } + } + if causes.is_empty() { + let cause = text.rsplit(": ").next().unwrap_or(&text).trim(); + return cause.to_owned(); + } + causes.join(", ") +} + +/// Counts a dial that failed with `error` (not one that wasn't needed). +pub(super) fn count_dial_failure(failures: &mut DialFailures, error: &::libp2p::swarm::DialError) { + use ::libp2p::swarm::DialError; + match error { + DialError::DialPeerConditionFalse(_) => {} + DialError::Transport(_) => failures.unreachable += 1, + DialError::WrongPeerId { .. } => failures.wrong_peer_id += 1, + DialError::NoAddresses => failures.no_addresses += 1, + _ => failures.other += 1, + } +} diff --git a/src/transport/libp2p/ignore.rs b/src/transport/libp2p/ignore.rs new file mode 100644 index 0000000..eca9342 --- /dev/null +++ b/src/transport/libp2p/ignore.rs @@ -0,0 +1,326 @@ +//! `--libp2p-ignore`: address ranges neither published nor dialled, nor connected over +//! (`Ignoring`, a behaviour that denies connections there), nor told peers by identify +//! (`Identifying`). + +use std::fmt; + +use super::*; + +/// An address range (`10.244.0.0/16`, `42::/16`; a bare address is a range of one), for +/// `Libp2pConfig::ignore`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct IpRange { + pub(super) network: std::net::IpAddr, + pub(super) prefix: u8, +} + +impl FromStr for IpRange { + type Err = anyhow::Error; + + fn from_str(text: &str) -> Result<Self> { + let (address, prefix) = match text.split_once('/') { + Some((address, prefix)) => (address, Some(prefix)), + None => (text, None), + }; + let network: std::net::IpAddr = address + .parse() + .with_context(|| format!("{text:?} is no address range (e.g. 10.244.0.0/16)"))?; + let bits = if network.is_ipv4() { 32 } else { 128 }; + let prefix = match prefix { + Some(prefix) => prefix + .parse::<u8>() + .ok() + .filter(|prefix| *prefix <= bits) + .with_context(|| format!("{text:?}: the prefix is 0 to {bits}"))?, + None => bits, + }; + Ok(Self { network, prefix }) + } +} + +impl fmt::Display for IpRange { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.network, self.prefix) + } +} + +impl IpRange { + /// The range of the `prefix` bits of `ip` (the rest zero). + pub(super) fn masked(ip: std::net::IpAddr, prefix: u8) -> Self { + use std::net::IpAddr; + let network = match ip { + IpAddr::V4(ip) => { + let mask = u32::MAX.checked_shl(32 - u32::from(prefix)).unwrap_or(0); + IpAddr::V4((u32::from(ip) & mask).into()) + } + IpAddr::V6(ip) => { + let mask = u128::MAX.checked_shl(128 - u32::from(prefix)).unwrap_or(0); + IpAddr::V6((u128::from(ip) & mask).into()) + } + }; + Self { network, prefix } + } + + /// Whether every address of `other` is in this range too. + pub(super) fn covers(&self, other: &IpRange) -> bool { + self.network.is_ipv4() == other.network.is_ipv4() + && self.prefix <= other.prefix + && self.contains(other.network) + } + + pub(super) fn contains(&self, ip: std::net::IpAddr) -> bool { + use std::net::IpAddr; + match (self.network, ip) { + (IpAddr::V4(network), IpAddr::V4(ip)) => { + let mask = u32::MAX + .checked_shl(32 - u32::from(self.prefix)) + .unwrap_or(0); + u32::from(network) & mask == u32::from(ip) & mask + } + (IpAddr::V6(network), IpAddr::V6(ip)) => { + let mask = u128::MAX + .checked_shl(128 - u32::from(self.prefix)) + .unwrap_or(0); + u128::from(network) & mask == u128::from(ip) & mask + } + _ => false, + } + } +} + +/// Whether `address` is in one of `ranges`: a direct address there (one through a relay +/// is judged by the connection to the relay). +pub(super) fn ignored(ranges: &[IpRange], address: &Multiaddr) -> bool { + if ranges.is_empty() || is_relayed(address) { + return false; + } + let ip = match address.iter().next() { + Some(Protocol::Ip4(ip)) => std::net::IpAddr::V4(ip), + Some(Protocol::Ip6(ip)) => std::net::IpAddr::V6(ip), + _ => return false, + }; + ranges.iter().any(|range| range.contains(ip)) +} + +/// Whether `address` can only be reached over a range of `ranges`: a direct address +/// there, or one through a relay whose own address is there (dialling the relay is +/// refused, `Ignoring`). For what's kept and dialled; connections are judged by +/// `ignored`. +pub(super) fn left_out(ranges: &[IpRange], address: &Multiaddr) -> bool { + if !is_relayed(address) { + return ignored(ranges, address); + } + let relay: Multiaddr = address + .iter() + .take_while(|protocol| !matches!(protocol, Protocol::P2pCircuit)) + .collect(); + ignored(ranges, &without_peer(relay)) +} + +/// Keeps connections off the address ranges `libp2p-ignore` names, whoever dialled: +/// hole punching and identify bring addresses there too, and a peer may dial us there. +pub(super) struct Ignoring { + pub(super) ranges: Vec<IpRange>, +} + +#[derive(Debug, thiserror::Error)] +#[error("{0} is in a range left out (libp2p-ignore)")] +pub(super) struct IgnoredAddress(Multiaddr); + +impl ::libp2p::swarm::NetworkBehaviour for Ignoring { + type ConnectionHandler = ::libp2p::swarm::dummy::ConnectionHandler; + type ToSwarm = std::convert::Infallible; + + fn handle_pending_inbound_connection( + &mut self, + _connection_id: ConnectionId, + _local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ::libp2p::swarm::ConnectionDenied> { + if ignored(&self.ranges, remote_addr) { + return Err(::libp2p::swarm::ConnectionDenied::new(IgnoredAddress( + remote_addr.clone(), + ))); + } + Ok(()) + } + + fn handle_established_inbound_connection( + &mut self, + _connection_id: ConnectionId, + _peer: PeerId, + _local_addr: &Multiaddr, + _remote_addr: &Multiaddr, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + Ok(::libp2p::swarm::dummy::ConnectionHandler) + } + + /// Dials libp2p makes itself (hole punching, AutoNAT, the DHT) to addresses all in + /// ranges left out are refused before they go out; ours never go there + /// (`Book::usable`). A dial with an address elsewhere too goes, and a connection it + /// makes in a range left out is refused once established (below). + fn handle_pending_outbound_connection( + &mut self, + _connection_id: ConnectionId, + _maybe_peer: Option<PeerId>, + addresses: &[Multiaddr], + _effective_role: ::libp2p::core::Endpoint, + ) -> Result<Vec<Multiaddr>, ::libp2p::swarm::ConnectionDenied> { + if let Some(first) = addresses.first() + && addresses + .iter() + .all(|address| ignored(&self.ranges, address)) + { + return Err(::libp2p::swarm::ConnectionDenied::new(IgnoredAddress( + first.clone(), + ))); + } + Ok(Vec::new()) + } + + fn handle_established_outbound_connection( + &mut self, + _connection_id: ConnectionId, + _peer: PeerId, + addr: &Multiaddr, + _role_override: ::libp2p::core::Endpoint, + _port_use: ::libp2p::core::transport::PortUse, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + if ignored(&self.ranges, addr) { + return Err(::libp2p::swarm::ConnectionDenied::new(IgnoredAddress( + addr.clone(), + ))); + } + Ok(::libp2p::swarm::dummy::ConnectionHandler) + } + + fn on_swarm_event(&mut self, _event: ::libp2p::swarm::FromSwarm) {} + + fn on_connection_handler_event( + &mut self, + _peer_id: PeerId, + _connection_id: ConnectionId, + event: ::libp2p::swarm::THandlerOutEvent<Self>, + ) { + match event {} + } + + fn poll( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll< + ::libp2p::swarm::ToSwarm<Self::ToSwarm, ::libp2p::swarm::THandlerInEvent<Self>>, + > { + std::task::Poll::Pending + } +} + +/// libp2p's identify, not told our listen addresses, nor external ones (found by UPnP, +/// AutoNAT, a relay), in the ranges `libp2p-ignore` names: peers would remember them, +/// show them, and dial them in vain. Announced addresses (`--libp2p-announce`) are told +/// whatever range they are in. +pub(super) struct Identifying { + pub(super) inner: ::libp2p::identify::Behaviour, + pub(super) ranges: Vec<IpRange>, + pub(super) announced: Vec<Multiaddr>, +} + +impl Identifying { + /// An external address identify isn't told: in an ignored range, and not announced. + fn left_out(&self, address: &Multiaddr) -> bool { + ignored(&self.ranges, address) && !self.announced.contains(address) + } +} + +impl ::libp2p::swarm::NetworkBehaviour for Identifying { + type ConnectionHandler = + <::libp2p::identify::Behaviour as ::libp2p::swarm::NetworkBehaviour>::ConnectionHandler; + type ToSwarm = ::libp2p::identify::Event; + + fn handle_pending_inbound_connection( + &mut self, + connection_id: ConnectionId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<(), ::libp2p::swarm::ConnectionDenied> { + self.inner + .handle_pending_inbound_connection(connection_id, local_addr, remote_addr) + } + + fn handle_established_inbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + local_addr: &Multiaddr, + remote_addr: &Multiaddr, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + self.inner.handle_established_inbound_connection( + connection_id, + peer, + local_addr, + remote_addr, + ) + } + + fn handle_pending_outbound_connection( + &mut self, + connection_id: ConnectionId, + maybe_peer: Option<PeerId>, + addresses: &[Multiaddr], + effective_role: ::libp2p::core::Endpoint, + ) -> Result<Vec<Multiaddr>, ::libp2p::swarm::ConnectionDenied> { + self.inner.handle_pending_outbound_connection( + connection_id, + maybe_peer, + addresses, + effective_role, + ) + } + + fn handle_established_outbound_connection( + &mut self, + connection_id: ConnectionId, + peer: PeerId, + addr: &Multiaddr, + role_override: ::libp2p::core::Endpoint, + port_use: ::libp2p::core::transport::PortUse, + ) -> Result<::libp2p::swarm::THandler<Self>, ::libp2p::swarm::ConnectionDenied> { + self.inner.handle_established_outbound_connection( + connection_id, + peer, + addr, + role_override, + port_use, + ) + } + + fn on_swarm_event(&mut self, event: ::libp2p::swarm::FromSwarm) { + use ::libp2p::swarm::FromSwarm; + match event { + FromSwarm::NewListenAddr(new) if ignored(&self.ranges, new.addr) => {} + FromSwarm::ExpiredListenAddr(expired) if ignored(&self.ranges, expired.addr) => {} + FromSwarm::ExternalAddrConfirmed(confirmed) if self.left_out(confirmed.addr) => {} + FromSwarm::ExternalAddrExpired(expired) if self.left_out(expired.addr) => {} + _ => self.inner.on_swarm_event(event), + } + } + + fn on_connection_handler_event( + &mut self, + peer_id: PeerId, + connection_id: ConnectionId, + event: ::libp2p::swarm::THandlerOutEvent<Self>, + ) { + self.inner + .on_connection_handler_event(peer_id, connection_id, event) + } + + fn poll( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll< + ::libp2p::swarm::ToSwarm<Self::ToSwarm, ::libp2p::swarm::THandlerInEvent<Self>>, + > { + self.inner.poll(cx) + } +} diff --git a/src/transport/libp2p/mod.rs b/src/transport/libp2p/mod.rs new file mode 100644 index 0000000..056b1b2 --- /dev/null +++ b/src/transport/libp2p/mod.rs @@ -0,0 +1,698 @@ +//! libp2p as a transport (`--libp2p-enable`, in builds with the `libp2p` feature): +//! direct connections over TCP (Noise, Yamux) and QUIC, to addresses this node knows. +//! +//! A node's transport identity is its libp2p peer ID; where it is, its listen addresses +//! (multiaddrs ending in the peer ID, `locations`). Seeds are such multiaddrs, or ones +//! without the peer ID, which a connection finds out. Messages go one per stream of the +//! `/continuum/1` protocol: their length (4 bytes, big endian), then the message as +//! `wire` encodes it. A send is done once the stream is written and closed; a connection +//! that can't be made, or a stream that can't be written, refuses it. A stream's protocol +//! goes with its first data, without waiting for the other side to agree +//! (`swarm_config`): opening a stream costs no round trip. +//! +//! Connected peers tell each other where they listen (libp2p's identify protocol), so a +//! node learns the addresses of peers that dialled it. While connected, a peer's +//! identify is all a node goes by for its direct addresses (`Book::identified`): those +//! it no longer has, or no longer publishes, are forgotten, whoever still names them. A node publishes where it can be +//! reached (`local_addresses`): the addresses it announces (`--libp2p-announce`, e.g. a +//! public address forwarded to it), or else those it listens on, loopback left out. +//! +//! A node nobody can dial (behind NAT, or a closed port) can still be reached through a +//! relay: a node of the swarm that relays (`--libp2p-relay`), or one configured +//! (`--libp2p-relays`), on which it reserves a slot; its address through the relay +//! (`<relay>/p2p-circuit/p2p/<it>`) is published with the others. Over a relayed connection, both sides then try to +//! connect directly (libp2p's DCUtR, hole punching); the relay carries the traffic until +//! that works, or for good if it doesn't (punched again now and then, `PUNCH_RETRIES`), +//! within its limits (`RELAY_CIRCUIT_DURATION`, +//! `RELAY_CIRCUIT_BYTES`; a circuit that ends is made again on the next send). Hole +//! punching needs both sides to listen (`HolePunching`: at their LAN addresses too); +//! where only one does, the other dials it directly once it knows where (`upgrade`). +//! +//! Of several connections to one peer (both dialled at once, or AutoNAT dialled back), +//! all but one are closed a few seconds later (`DUPLICATE_GRACE`, `Book::duplicates`), +//! by the side with the smaller peer ID: it keeps the direct one with the shortest round +//! trip (libp2p's ping, on each connection), so a path over the LAN wins over one +//! through a VPN, whichever opened first; between a relay and a node with a slot on it +//! too, where a slot on a connection closed is reserved again over the one kept. A +//! peer connected directly is dialled again where it might be reached better +//! (`explore`): at addresses it tells of later (a new interface: identify pushes them +//! at once), and at all of them when this node gains an interface. Otherwise a +//! connection made over the WLAN would stay there once ethernet comes. +//! +//! A relay says so where it publishes its addresses: next to each, the same address +//! ending in `/p2p-circuit` (where to reserve a slot), which other nodes find in the +//! member list. A node that needs a relay picks up to `AUTO_RELAYS` relays +//! (`relay_check`): those configured first, then those listed in the document +//! (`learn_relays`), then members that relay. It needs one when +//! it has no address others could dial, or when AutoNAT says it can't be dialled: +//! AutoNAT asks connected peers (nodes of the swarm) to dial it back. Once AutoNAT says +//! it can be dialled, it lets the relays it picked go. +//! +//! With `--libp2p-ignore`, address ranges are left out (VPNs): not published, not told +//! peers by identify (`Identifying`), not offered for hole punching, not dialled by us +//! (`Book::usable`) nor by libp2p unless with an address elsewhere too, and connections +//! there refused (`Ignoring`). Ranges this node has no route to are left +//! out alike, as long as it has none (`routes`): without IPv6 to the internet, peers' +//! public IPv6 addresses; and with `--libp2p-internet-hosts`, its own aren't published +//! then. With `--libp2p-prefer`, +//! addresses in the ranges named come first, where only so many are kept +//! (`Book::preference`). +//! +//! With `--libp2p-upnp`, a node asks its router to forward its listen ports (UPnP), and +//! publishes the router's public address for them with its own (`Book::direct`): then +//! it can be dialled from elsewhere, whatever the other side's NAT. +//! +//! With `--libp2p-mdns`, a node finds libp2p peers on its LAN (multicast DNS). Their +//! LAN addresses are noted, and those that turn out to speak continuum (identify says +//! so) are seeds while the node joins (`discovered_seeds`): nodes on one LAN find each +//! other without configured seeds. +//! +//! It uses public libp2p infrastructure only with `--libp2p-public`: then it joins the +//! IPFS DHT (Kademlia, through the IPFS bootstrap nodes), where it finds addresses of +//! peers the member list doesn't have, and public peers that relay; those are picked +//! after members that relay. Public relays keep circuits short (by default 2 minutes and +//! 128 KiB): enough to punch holes through NATs, not to carry a swarm's traffic. Without +//! it, relays are those configured, or members that say they relay. + +use std::collections::HashMap; +use std::path::Path; +use std::str::FromStr; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; + +use ::libp2p::multiaddr::Protocol; +use ::libp2p::swarm::dial_opts::{DialOpts, PeerCondition}; +use ::libp2p::swarm::{ConnectionId, SwarmEvent}; +use ::libp2p::{Multiaddr, PeerId, StreamProtocol, Swarm, identity::Keypair}; +use anyhow::{Context, Result, anyhow, bail}; +use async_trait::async_trait; +use futures::{AsyncReadExt, AsyncWriteExt, StreamExt}; +use tokio::sync::{Semaphore, mpsc, oneshot, watch}; +use tracing::{debug, info, trace, warn}; + +use crate::locks::MutexExt; +use crate::transport::stats::{ + Connections, DialFailures, HolePunches, Libp2pStats, Reachability, RelaySlots, TransportStats, +}; +use crate::transport::{ + Addressed, Connection, EncodedMessage, HandlerFn, MessageHandler, ParsedMessage, Transport, + hand_on_probe, is_probe, wire, +}; + +mod behaviour; +mod book; +mod config; +mod driver; +mod errors; +mod ignore; +mod relays; +mod routes; + +use behaviour::*; +use book::*; +pub(crate) use config::{Libp2pConfig, check}; +use driver::*; +use errors::*; +use ignore::*; +use relays::*; +use routes::*; + +/// The scheme of this transport's addresses (`transport::paths`). +pub(crate) const SCHEME: &str = "libp2p"; + +const PROTOCOL: StreamProtocol = StreamProtocol::new("/continuum/1"); +/// Sends taking this long are logged with where the time went (at debug). +const SLOW_SEND: Duration = Duration::from_millis(5); +/// The largest message read: anyone who can connect can send anything, so this bounds +/// what one stream costs. Sync messages above 16 chunks travel as blobs. +const MAX_MESSAGE: usize = 128 << 20; +/// How long a connection may take to come up. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// A connection nobody uses is closed after this at least; neighbours keep theirs busy, +/// with a keepalive within one and a half intervals (`PeerState::take_keepalive_due`), +/// so a long interval lengthens it (`idle_timeout`). +const IDLE_TIMEOUT: Duration = Duration::from_secs(300); + +/// How long a connection may idle before it is closed: `IDLE_TIMEOUT`, or three +/// keepalive intervals if that is longer. +fn idle_timeout(keepalive: Duration) -> Duration { + IDLE_TIMEOUT.max(keepalive * 3) +} +/// Addresses of one dial tried at once (libp2p's default: 8). Hole punching (`HolePunching`) +/// dials every address both sides offer, where they listen (LAN, VPNs, public IPv6) and +/// where others saw them, often more than 8, while the other side punches towards all of +/// them at the same moment: an address dialled only after others failed (each after +/// QUIC's 5 s handshake timeout) misses its punch, and the LAN's is often among the +/// last. Dials of the member list's addresses (`ADDRESSES_PER_PEER`) are unchanged. +const DIAL_CONCURRENCY: std::num::NonZeroU8 = std::num::NonZeroU8::new(32).unwrap(); +/// How long a stream may take to open and to carry a message, before what its size adds +/// (`stream_timeout`). +const STREAM_TIMEOUT: Duration = Duration::from_secs(10); +/// The slowest a message is expected to go, in bytes per second. +const MIN_RATE: u64 = 1 << 20; +/// Inbound streams read at once. +const READS: usize = 32; +/// Addresses kept per peer, and peers with addresses at most. +const ADDRESSES_PER_PEER: usize = 8; +/// Addresses known not to be a peer's (`Book::wrong`), per peer at most. +const WRONG_PER_PEER: usize = 32; +/// Of this node's own, the direct ones (the best first) and those through relays it +/// publishes at most (`local_addresses`): a node listening everywhere has many direct +/// ones, and those through a relay are what reaches it from behind a NAT. +const PUBLISHED_DIRECT: usize = 4; +const PUBLISHED_RELAYED: usize = 2; +const MAX_PEERS: usize = 4096; + +/// A relayed connection (circuit) through this node lasts at most this long and carries +/// at most this much: generous, since the swarm's own relays may carry sync traffic for +/// good where hole punching fails (libp2p's defaults, 2 minutes and 128 KiB, are for +/// the punching only). +const RELAY_CIRCUIT_DURATION: Duration = Duration::from_secs(3600); +const RELAY_CIRCUIT_BYTES: u64 = 1 << 30; +/// Circuits through this node at once, and per peer (either end: a node behind the +/// relay is an end of every circuit to it). libp2p's rate limits on circuits and slots +/// (a burst, then one every minute or two, per peer and per IP address: every node +/// behind one NAT shares one) are left out: they're for public relays serving +/// strangers, and refused circuits used them up too. +const RELAY_CIRCUITS: usize = 1024; +const RELAY_CIRCUITS_PER_PEER: usize = 64; +/// A configured relay that couldn't be reached to find out its peer ID is tried again +/// after this. +const RELAY_RETRY: Duration = Duration::from_secs(30); +/// Relays picked among the members, at most, and how often a node checks whether it +/// needs them. +const AUTO_RELAYS: usize = 2; +const RELAY_CHECK: Duration = Duration::from_secs(30); +/// A relay that refused a slot (`relay_refused`: none left, not allowed, not a relay, +/// another peer at its address: a relay restarted without its libp2p key) isn't picked +/// again for this long. One that didn't grant it for another reason (it doesn't answer, +/// the connection closed) isn't for `RELAY_RETRY`; a slot lost is reserved again at +/// once. +const RELAY_PAUSE: Duration = Duration::from_secs(600); +/// A peer connected only through a relay is dialled directly where it can be reached +/// (`upgrade`): first at once, then after a failure no sooner than this, doubling up to +/// `RELAY_PAUSE`. +const UPGRADE_RETRY: Duration = Duration::from_secs(30); +/// A directly connected peer's addresses dialled at once for a better path (`explore`). +const EXPLORED_AT_ONCE: usize = 4; +/// A peer nobody mentioned for this long (the member list, identify, mDNS, a +/// connection) is forgotten, unless connected: mostly old peer IDs of nodes restarted +/// without their libp2p key, whose new ones the member list has since. Members are +/// mentioned every few seconds (`swarm::topology`). Checked every `PRUNE_EVERY`. +const STALE_AFTER: Duration = Duration::from_secs(600); +const PRUNE_EVERY: Duration = Duration::from_secs(60); +/// Several connections to one peer (both dialled at once, AutoNAT dialling back to see +/// whether it can, or a better path explored) are cut to one, the fastest +/// (`Book::duplicates`), this long after the last one opened: long enough for what it +/// was dialled for, and for each to be pinged since (a new one at once, as it opens; +/// the others within libp2p's ping interval, 15 s). Looked for every `DEDUPE_EVERY`. +const DUPLICATE_GRACE: Duration = Duration::from_secs(20); +/// The same while a peer is connected only through relays: a new relayed connection is +/// a hole punch under way (`PUNCH_RETRIES`), which takes its tries (up to a QUIC +/// handshake's 5 s each, three times) on that connection. +const PUNCH_GRACE: Duration = Duration::from_secs(60); +/// A slot lost this soon after its relay's duplicate connections were closed moved to +/// the one kept (`Book::rehoming`). +const REHOMING_WITHIN: Duration = Duration::from_secs(10); +/// After libp2p's hole punching gave up on a peer (three tries, as the relayed +/// connection opened), it tries again after these, the last one over and over: what +/// failed may have been passing (a NAT's mapping, a busy link). Each time through a new +/// connection through the relay, on which the peer starts the punch (`retry_punches`). +const PUNCH_RETRIES: [Duration; 3] = [ + Duration::from_secs(60), + Duration::from_secs(300), + Duration::from_secs(1800), +]; +const DEDUPE_EVERY: Duration = Duration::from_secs(5); + +/// How long a message of `bytes` may take on a stream. +fn stream_timeout(bytes: usize) -> Duration { + STREAM_TIMEOUT + Duration::from_millis(bytes as u64 * 1000 / MIN_RATE) +} + +pub(crate) struct Libp2pTransport { + local_id: String, + peer_id: PeerId, + control: libp2p_stream::Control, + commands: mpsc::UnboundedSender<Command>, + book: Arc<Mutex<Book>>, + incoming: tokio::sync::Mutex<Option<libp2p_stream::IncomingStreams>>, + stopped: watch::Receiver<bool>, +} + +impl Libp2pTransport { + /// Builds the swarm, listens, and runs it in a task of its own. + pub(crate) fn start(config: Libp2pConfig) -> Result<Self> { + let peer_id = config.key.public().to_peer_id(); + let features = Features::of(&config); + let mut swarm = if config.memory { + memory_swarm(config.key, features, config.ignore.clone())? + } else { + network_swarm(config.key, features, config.ignore.clone())? + }; + for address in config.listen { + swarm + .listen_on(address.clone()) + .with_context(|| format!("Listening on {address}"))?; + } + // Told peers whatever range they are in (`Identifying`). + swarm.behaviour_mut().identify.announced = config.announce.clone(); + for address in &config.announce { + // Identify tells peers these. + swarm.add_external_address(address.clone()); + } + let mut control = swarm.behaviour().stream.new_control(); + let incoming = if config.continuum { + Some( + control + .accept(PROTOCOL) + .map_err(|error| anyhow!("Accepting {PROTOCOL}: {error}"))?, + ) + } else { + None + }; + let (commands, receiver) = mpsc::unbounded_channel(); + let mut book = Book { + announced: config.announce, + ignored: config.ignore.clone(), + preferred: config.prefer.clone(), + routes: Routes::asking(config.internet_hosts.clone()), + serving: config.relay, + ..Book::default() + }; + for relay in config.relays { + if let Some(address) = book.name_relay(relay, Named::Configured, peer_id) { + let _ = commands.send(Command::FindRelay { address }); + } + } + let book = Arc::new(Mutex::new(book)); + let (stopping, stopped) = watch::channel(false); + tokio::spawn(run( + swarm, + receiver, + commands.clone(), + Arc::clone(&book), + Relaying { + serving: config.relay, + public: config.public, + local: peer_id, + }, + stopping, + )); + info!(peer_id = %peer_id, "Transport started"); + Ok(Self { + local_id: peer_id.to_string(), + peer_id, + control, + commands, + book, + incoming: tokio::sync::Mutex::new(incoming), + stopped, + }) + } + + /// Connected to `peer`, if it can be had in time. + async fn connect(&self, peer: PeerId) -> Result<()> { + let addresses = { + let mut book = lock(&self.book); + let known = book.usable(&peer, self.peer_id); + if known.is_empty() { + book.through_relays(peer, self.peer_id) + } else { + known + } + }; + let (done, connected) = oneshot::channel(); + self.commands + .send(Command::Connect { + peer, + addresses, + done, + }) + .map_err(|_| anyhow!("the libp2p swarm stopped"))?; + match tokio::time::timeout(CONNECT_TIMEOUT, connected).await { + Ok(Ok(Ok(()))) => Ok(()), + Ok(Ok(Err(error))) => bail!("couldn't connect to {peer}: {error}"), + Ok(Err(_)) => bail!("the libp2p swarm stopped"), + Err(_) => bail!("connecting to {peer} took longer than {CONNECT_TIMEOUT:?}"), + } + } +} + +/// One message from an inbound stream. +async fn read_message(mut stream: ::libp2p::Stream) -> Result<Vec<u8>> { + let mut length = [0u8; 4]; + tokio::time::timeout(STREAM_TIMEOUT, stream.read_exact(&mut length)) + .await + .map_err(|_| anyhow!("no length within {STREAM_TIMEOUT:?}"))??; + let length = u32::from_be_bytes(length) as usize; + if length > MAX_MESSAGE { + bail!("a message of {length} bytes; at most {MAX_MESSAGE} are read"); + } + let mut body = vec![0u8; length]; + tokio::time::timeout(stream_timeout(length), stream.read_exact(&mut body)) + .await + .map_err(|_| { + anyhow!( + "{length} bytes took longer than {:?}", + stream_timeout(length) + ) + })??; + Ok(body) +} + +#[async_trait] +impl Transport for Libp2pTransport { + fn local_id(&self) -> &str { + &self.local_id + } + + fn is_peer_id(&self, id: &str) -> bool { + PeerId::from_str(id).is_ok() + } + + /// The longest a message may take on a stream. + fn delivery_limit(&self) -> Duration { + stream_timeout(MAX_MESSAGE) + } + + /// Where we can be reached, each with our peer ID: the announced addresses, or else + /// those we listen on (`published`), and those through relays; at most + /// `PUBLISHED_DIRECT` and `PUBLISHED_RELAYED` of them, so the direct ones never + /// crowd out the relayed ones. A relay adds each direct one again, ending in + /// `/p2p-circuit`: where to reserve a slot. + fn local_addresses(&self) -> Vec<String> { + let book = lock(&self.book); + let direct: Vec<Multiaddr> = book + .direct() + .into_iter() + .filter_map(|address| address.with_p2p(self.peer_id).ok()) + .take(PUBLISHED_DIRECT) + .collect(); + let relayed = book + .listening + .iter() + .filter(|address| is_relayed(address)) + .filter_map(|address| address.clone().with_p2p(self.peer_id).ok()) + .take(PUBLISHED_RELAYED); + let offers: Vec<Multiaddr> = if book.serving { + direct + .iter() + .map(|address| address.clone().with(Protocol::P2pCircuit)) + .collect() + } else { + Vec::new() + }; + direct + .into_iter() + .chain(relayed) + .chain(offers) + .map(|address| address.to_string()) + .collect() + } + + /// A peer ID, or a multiaddr ending in one (noted as where that peer is). + fn locate(&self, target: &str) -> Option<String> { + if let Ok(peer) = PeerId::from_str(target) { + return Some(peer.to_string()); + } + let mut address = Multiaddr::from_str(target).ok()?; + // A relay saying where to reserve a slot: `<address>/p2p/<relay>/p2p-circuit`. + if let Some(Protocol::P2pCircuit) = address.iter().last() { + address.pop(); + let relay = split_peer(&address)?; + if relay != self.peer_id { + let new = { + let mut book = lock(&self.book); + // One in a range left out is no use (`Book::left_out`). + let new = book.relays.get(&relay) != Some(&address) && !book.left_out(&address); + if new && (book.relays.len() < MAX_PEERS || book.relays.contains_key(&relay)) { + book.relays.insert(relay, address.clone()); + } + book.remember(relay, address); + new + }; + if new { + let _ = self.commands.send(Command::CheckRelays); + } + } + return Some(relay.to_string()); + } + let peer = split_peer(&address)?; + address.pop(); + // A peer ID only says nothing about where. + if peer != self.peer_id && !address.is_empty() { + lock(&self.book).remember(peer, address); + } + Some(peer.to_string()) + } + + /// Multiaddrs, with or without the relay's peer ID; others are ignored. + fn learn_relays(&self, targets: &[String]) { + let listed = targets + .iter() + .filter_map(|target| Multiaddr::from_str(target).ok()) + .collect(); + let Some(found) = lock(&self.book).learn_listed(listed, self.peer_id) else { + return; + }; + for address in found { + let _ = self.commands.send(Command::FindRelay { address }); + } + let _ = self.commands.send(Command::CheckRelays); + } + + fn connection(&self, target: &str) -> Option<Connection> { + let peer = PeerId::from_str(target).ok()?; + lock(&self.book).connection(&peer) + } + + fn punching(&self, target: &str) -> Option<crate::transport::Punching> { + let peer = PeerId::from_str(target).ok()?; + let (failed, next) = *lock(&self.book).punching.get(&peer)?; + Some(crate::transport::Punching { + failed, + next_in_secs: next + .saturating_duration_since(tokio::time::Instant::now()) + .as_secs(), + }) + } + + fn location(&self, target: &str) -> Option<String> { + let peer = PeerId::from_str(target).ok()?; + let address = lock(&self.book).location(&peer)?.clone(); + Some( + address + .with_p2p(peer) + .unwrap_or_else(|address| address) + .to_string(), + ) + } + + fn stats(&self) -> TransportStats { + TransportStats { + libp2p: Some(lock(&self.book).stats()), + ..TransportStats::default() + } + } + + fn on_seeds_found(&self, found: Arc<tokio::sync::Notify>) { + lock(&self.book).seeds_found = Some(found); + } + + fn on_reachable(&self, reachable: Arc<tokio::sync::Notify>) { + lock(&self.book).reachable = Some(reachable); + } + + /// Nodes found on the LAN (mDNS) that speak continuum. + fn discovered_seeds(&self) -> Vec<String> { + std::mem::take(&mut lock(&self.book).discovered) + .iter() + .map(Multiaddr::to_string) + .collect() + } + + /// Where `target` is, each address with its peer ID. + fn locations(&self, target: &str) -> Vec<String> { + let Ok(peer) = PeerId::from_str(target) else { + return Vec::new(); + }; + lock(&self.book) + .addresses + .get(&peer) + .into_iter() + .flatten() + .filter_map(|address| address.clone().with_p2p(peer).ok()) + .map(|address| address.to_string()) + .collect() + } + + /// A multiaddr ending in a peer ID is that peer; one without is connected to, to + /// find out. A bare peer ID resolves if this transport knows where it is. + async fn resolve_seed(&self, seed: &str) -> Result<String> { + let address = match PeerId::from_str(seed) { + Ok(peer) => Multiaddr::empty().with(Protocol::P2p(peer)), + Err(_) => { + Multiaddr::from_str(seed).with_context(|| format!("{seed} is no multiaddr"))? + } + }; + // A peer ID only (`/p2p/<peer ID>`): reached where it's known, or through the + // relays we know. + if let [Protocol::P2p(peer)] = address.iter().collect::<Vec<_>>()[..] { + let mut book = lock(&self.book); + if book.addresses.contains_key(&peer) + || !book.through_relays(peer, self.peer_id).is_empty() + { + return Ok(peer.to_string()); + } + bail!( + "no address known for libp2p peer {peer}, and no relay to reach it through; \ + give a multiaddr, or a relay (libp2p-relays)" + ); + } + if let Some(peer) = self.locate(seed) { + return Ok(peer); + } + let (done, identified) = oneshot::channel(); + self.commands + .send(Command::Identify { address, done }) + .map_err(|_| anyhow!("the libp2p swarm stopped"))?; + match tokio::time::timeout(CONNECT_TIMEOUT, identified).await { + Ok(Ok(Ok(peer))) => Ok(peer.to_string()), + Ok(Ok(Err(error))) => bail!("couldn't connect to {seed}: {error}"), + Ok(Err(_)) => bail!("the libp2p swarm stopped"), + Err(_) => bail!("connecting to {seed} took longer than {CONNECT_TIMEOUT:?}"), + } + } + + fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { + wire::encoded(peer, message) + } + + async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { + let peer = PeerId::from_str(&message.peer) + .with_context(|| format!("{} is no libp2p peer ID", message.peer))?; + let started = std::time::Instant::now(); + self.connect(peer).await?; + // The protocol goes with the first data (`swarm_config`), so a peer that doesn't + // speak it seems to take the message: once identify has told, say so. + if let Some(relays) = lock(&self.book).not_nodes.get(&peer).copied() { + if relays { + bail!( + "{peer} is a relay, not a node: give a node behind it as the seed \ + (<relay>/p2p-circuit/p2p/<peer ID>, or /p2p/<peer ID> with libp2p-relays)" + ); + } + bail!("{peer} doesn't speak continuum"); + } + let connected = started.elapsed(); + let mut control = self.control.clone(); + let body = message.body.clone(); + let write = async move { + let mut stream = control.open_stream(peer, PROTOCOL).await?; + let opened = started.elapsed(); + stream.write_all(&(body.len() as u32).to_be_bytes()).await?; + stream.write_all(&body).await?; + stream.close().await?; + // Kept until the peer has closed its side too. Over QUIC, dropping a stream + // stops its reading half (STOP_SENDING), and the peer's answer to the + // protocol (sent with the first data, `swarm_config`) then fails, and with it + // the stream, before the peer has read the message: identify, which reads + // its answers, works, while every message is lost. Yamux (TCP) doesn't mind. + // Off the send, which is done. + tokio::spawn(async move { + let mut rest = [0u8; 64]; + let _ = tokio::time::timeout(STREAM_TIMEOUT, async { + while let Ok(read) = stream.read(&mut rest).await { + if read == 0 { + break; + } + } + }) + .await; + }); + Ok::<_, anyhow::Error>(opened) + }; + let timeout = stream_timeout(message.body.len()); + let opened = tokio::time::timeout(timeout, write) + .await + .map_err(|_| anyhow!("sending to {peer} took longer than {timeout:?}"))? + .with_context(|| format!("Sending to {peer}"))?; + let took = started.elapsed(); + if took >= SLOW_SEND { + // Where a send's time goes: the swarm task (connecting), the stream's protocol + // negotiation (opening), or the connection (writing). + debug!( + peer = %Addressed(SCHEME, &peer), + kind = ?message.kind, + bytes = message.body.len(), + connect_ms = connected.as_secs_f64() * 1000.0, + open_ms = (opened - connected).as_secs_f64() * 1000.0, + write_ms = (took - opened).as_secs_f64() * 1000.0, + "Slow send" + ); + } + Ok(()) + } + + /// Reads inbound streams (`READS` at once) and handles their messages one at a + /// time, in the order they were read; PINGs and PONGs at once (`is_probe`). + async fn start_consumer(&self, handler: MessageHandler) -> Result<()> { + let Some(mut incoming) = self.incoming.lock().await.take() else { + bail!("libp2p consumer already started, or this node only relays"); + }; + let handler: Arc<HandlerFn> = Arc::from(handler); + let (sender, mut received) = mpsc::channel::<(PeerId, ParsedMessage, usize)>(READS); + let reads = Arc::new(Semaphore::new(READS)); + let probes = Arc::clone(&handler); + tokio::spawn(async move { + while let Some((peer, stream)) = incoming.next().await { + let Ok(permit) = Arc::clone(&reads).acquire_owned().await else { + break; + }; + let sender = sender.clone(); + let probes = Arc::clone(&probes); + tokio::spawn(async move { + match read_message(stream).await { + Ok(body) => match wire::decode(&body) { + Ok(message) if is_probe(&message) => { + hand_on_probe( + SCHEME, + &probes, + peer.to_string(), + message, + body.len(), + ); + } + Ok(message) => { + let _ = sender.send((peer, message, body.len())).await; + } + Err(error) => { + debug!(peer = %Addressed(SCHEME, &peer), error = %error, "Dropping a message that doesn't decode"); + } + }, + Err(error) => { + debug!(peer = %Addressed(SCHEME, &peer), error = %format!("{error:#}"), "Dropping an unreadable stream"); + } + } + drop(permit); + }); + } + }); + while let Some((peer, message, payload_len)) = received.recv().await { + if let Err(error) = handler(peer.to_string(), message, payload_len).await { + warn!(peer = %Addressed(SCHEME, &peer), error = %format!("{error:#}"), "Failed to process inbound message"); + } + } + Ok(()) + } + + async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> { + let mut stopped = self.stopped.clone(); + let _ = stopped.wait_for(|stopped| *stopped).await; + Err(std::io::Error::other("the libp2p swarm stopped")) + } +} + +#[cfg(test)] +mod tests; diff --git a/src/transport/libp2p/relays.rs b/src/transport/libp2p/relays.rs new file mode 100644 index 0000000..1a57635 --- /dev/null +++ b/src/transport/libp2p/relays.rs @@ -0,0 +1,271 @@ +//! Reaching peers past NATs: picking relays and keeping slots on them, upgrading +//! relayed connections to direct ones, retrying hole punches, and finding peers on the +//! DHT. + +use super::*; + +/// How a node relates to relays. +pub(super) struct Relaying { + /// It relays for others. + pub(super) serving: bool, + /// Public relays may be picked, after members. + pub(super) public: bool, + pub(super) local: PeerId, +} + +/// The relays to pick among `offers`, if this node needs relays, keeping `using`: a +/// node needs them when it has no address others could dial (`direct`), or AutoNAT says +/// it can't be dialled; up to `AUTO_RELAYS`, in an order of its own (the lowest hash of +/// both peer IDs), so that not every node picks the same. +pub(super) fn relay_check( + local: PeerId, + direct: bool, + nat: &::libp2p::autonat::NatStatus, + offers: &HashMap<PeerId, Multiaddr>, + using: &[PeerId], +) -> Vec<(PeerId, Multiaddr)> { + use ::libp2p::autonat::NatStatus; + let needed = match nat { + NatStatus::Private => true, + NatStatus::Public(_) => false, + NatStatus::Unknown => !direct, + }; + if !needed || using.len() >= AUTO_RELAYS { + return Vec::new(); + } + let score = |relay: &PeerId| { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + (local, relay).hash(&mut hasher); + hasher.finish() + }; + let mut candidates: Vec<(PeerId, Multiaddr)> = offers + .iter() + .filter(|(relay, _)| **relay != local && !using.contains(relay)) + .map(|(relay, address)| (*relay, address.clone())) + .collect(); + candidates.sort_by_key(|(relay, _)| score(relay)); + candidates.truncate(AUTO_RELAYS - using.len()); + candidates +} + +/// Reserves slots on the relays `relay_check` picks (configured ones first, then listed +/// ones, members, public ones), and lets those picked go once AutoNAT says this node can +/// be dialled. +pub(super) fn pick_relays( + swarm: &mut Swarm<Behaviour>, + book: &Mutex<Book>, + relaying: &Relaying, + nat: &::libp2p::autonat::NatStatus, + picked: &mut HashMap<::libp2p::core::transport::ListenerId, PeerId>, +) { + if nat.is_public() && !picked.is_empty() { + info!("Reachable directly (AutoNAT); letting the relays picked go"); + let mut book = lock(book); + for (listener, relay) in std::mem::take(picked) { + swarm.remove_listener(listener); + book.slots.remove(&relay); + } + return; + } + let (direct, configured, listed, members, public) = { + let mut book = lock(book); + let now = tokio::time::Instant::now(); + book.failed_relays.retain(|_, until| now < *until); + let failed = book.failed_relays.clone(); + // Not paused, and not in a range left out (ignored, or without a route): those + // would only be denied or fail, again and again. + let usable = |book: &mut Book, offers: HashMap<PeerId, Multiaddr>| { + offers + .into_iter() + .filter(|(relay, address)| !failed.contains_key(relay) && !book.left_out(address)) + .collect::<HashMap<PeerId, Multiaddr>>() + }; + let public = if relaying.public { + let offers = book.public_relays.clone(); + usable(&mut book, offers) + } else { + HashMap::new() + }; + let (configured, listed, members) = ( + book.named(Named::Configured), + book.named(Named::Listed), + book.relays.clone(), + ); + ( + !book.direct().is_empty(), + usable(&mut book, configured), + usable(&mut book, listed), + usable(&mut book, members), + public, + ) + }; + let mut using: Vec<PeerId> = picked.values().copied().collect(); + // Each kind for what the ones before left. + for (kind, offers) in [ + (Named::Configured.as_str(), configured), + (Named::Listed.as_str(), listed), + ("member", members), + ("public", public), + ] { + for (relay, address) in relay_check(relaying.local, direct, nat, &offers, &using) { + using.push(relay); + match swarm.listen_on(address.clone().with(Protocol::P2pCircuit)) { + Ok(listener) => { + info!(relay = %Addressed(SCHEME, &relay), address = %address, kind, "Picked a relay"); + picked.insert(listener, relay); + } + Err(error) => { + debug!(relay = %Addressed(SCHEME, &relay), kind, error = %error, "Couldn't use a relay"); + } + } + } + } +} + +/// Dials `peer` directly if it's connected only through a relay and is known elsewhere +/// too (the member list, identify, mDNS), since the first connection, made through a +/// relay (a relayed seed), would otherwise be kept for good: a peer connected isn't +/// dialled again, and hole punching needs both sides to listen. The direct connection, +/// once open, wins over the relayed one (`Book::duplicates`). A dial refused because +/// another to the peer is under way (hole punching's, say) doesn't count as a try: the +/// next comes when that one fails (`driver`), not `UPGRADE_RETRY` later. +pub(super) fn upgrade( + swarm: &mut Swarm<Behaviour>, + book: &Mutex<Book>, + peer: PeerId, + local: PeerId, +) { + let (direct, now, wait) = { + let mut book = lock(book); + if book.connection(&peer) != Some(Connection::Relayed) { + return; + } + let mut direct = book.usable(&peer, local); + direct.retain(|address| !is_relayed(address)); + if direct.is_empty() { + return; + } + let now = tokio::time::Instant::now(); + let (due, wait) = book + .upgrades + .get(&peer) + .copied() + .unwrap_or((now, UPGRADE_RETRY)); + if now < due { + return; + } + (direct, now, wait) + }; + let dialled = swarm.dial( + DialOpts::peer_id(peer) + .addresses(direct.clone()) + .condition(PeerCondition::NotDialing) + .build(), + ); + match dialled { + Ok(()) => { + debug!(peer = %Addressed(SCHEME, &peer), addresses = ?direct, "Connected only through a relay; dialling directly"); + lock(book) + .upgrades + .insert(peer, (now + wait, (wait * 2).min(RELAY_PAUSE))); + } + Err(error) => { + debug!(peer = %Addressed(SCHEME, &peer), error = %error, "Connected only through a relay; can't dial directly yet"); + } + } +} + +/// Dials `peer`, connected directly, at its other addresses, in case one is a better +/// path: one that came since (a new interface of its, identify tells), or, `again` +/// (one of ours came), every one, as our side may reach them better now, the one in +/// use too. Each once while connected (`Book::explored`); a connection that opens is +/// pinged at once, and of several the fastest is kept (`Book::duplicates`). Without +/// it, a connection stays on the path it was made on (the WLAN) when a better one comes +/// (ethernet): a connected peer isn't dialled again. +pub(super) fn explore(swarm: &mut Swarm<Behaviour>, book: &Mutex<Book>, peer: PeerId, again: bool) { + let candidates = lock(book).to_explore(peer, again); + if candidates.is_empty() { + return; + } + debug!(peer = %Addressed(SCHEME, &peer), addresses = ?candidates, "Connected directly; dialling other addresses, for a better path"); + if let Err(error) = swarm.dial( + DialOpts::peer_id(peer) + .addresses(candidates) + .condition(PeerCondition::Always) + .build(), + ) { + debug!(peer = %Addressed(SCHEME, &peer), error = %error, "Couldn't dial for a better path"); + } +} + +/// Tries hole punching again with the peers it failed with that are due (`punching`). +/// libp2p punches as a relayed connection opens, started by the side that was dialled +/// through the relay; that side is where a punch's failure is told, so it is this one. +/// A new relayed connection starts a new punch: we dial the peer through a relay if it +/// has a slot on one; else we close our relayed connections to it, and its next message +/// (a probe, within a minute) dials us through ours again. Peers connected directly by +/// now are dropped; those not connected now are kept, failures and all. +pub(super) fn retry_punches(swarm: &mut Swarm<Behaviour>, book: &Mutex<Book>, local: PeerId) { + let now = tokio::time::Instant::now(); + let mut close: Vec<ConnectionId> = Vec::new(); + let due: Vec<(PeerId, usize, Vec<Multiaddr>)> = { + let mut book = lock(book); + let book = &mut *book; + let mut due = Vec::new(); + book.punching.retain(|peer, (failed, next)| { + match book_connection(&book.connections, peer) { + Some(Connection::Direct) => return false, + // Kept, failures and all: one we closed to have it dial anew comes back. + None => return true, + Some(Connection::Relayed) if now < *next => return true, + Some(Connection::Relayed) => {} + } + // As `Book::usable`, by parts: `punching` is borrowed. + let through: Vec<Multiaddr> = book + .addresses + .get(peer) + .map(|known| dialable(known, local)) + .unwrap_or_default() + .into_iter() + .filter(|address| { + is_relayed(address) && !excluded(&book.ignored, &mut book.routes, address) + }) + .collect(); + // Until its punch fails again (or the peer tries from its side). + *next = now + PUNCH_RETRIES[(*failed).min(PUNCH_RETRIES.len() - 1)]; + if through.is_empty() { + close.extend( + book.connections + .iter() + .filter(|(_, (connected, relayed, _))| connected == peer && *relayed) + .map(|(id, _)| *id), + ); + } else { + due.push((*peer, *failed, through)); + } + true + }); + due + }; + for connection in close { + debug!(connection = %connection, "Hole punching again: closing a relayed connection, for the peer to dial anew"); + swarm.close_connection(connection); + } + for (peer, failed, through) in due { + debug!(peer = %Addressed(SCHEME, &peer), failed, "Hole punching again, through a new relayed connection"); + let _ = swarm.dial( + DialOpts::peer_id(peer) + .addresses(through) + .condition(PeerCondition::Always) + .build(), + ); + } +} + +/// Asks the DHT (with `--libp2p-public`) where `peer` is, for the next attempt. +pub(super) fn find_on_dht(swarm: &mut Swarm<Behaviour>, peer: PeerId) { + if let Some(dht) = swarm.behaviour_mut().kad.as_mut() { + dht.get_closest_peers(peer); + } +} diff --git a/src/transport/libp2p/routes.rs b/src/transport/libp2p/routes.rs new file mode 100644 index 0000000..3108f0c --- /dev/null +++ b/src/transport/libp2p/routes.rs @@ -0,0 +1,203 @@ +//! Addresses without a route from here, left out as if `--libp2p-ignore` named them: a +//! node without IPv6 to the internet doesn't keep, show or dial peers' public IPv6 +//! addresses; one without a route to another LAN, that LAN's. And with +//! `--libp2p-internet-hosts`, whether there's a route to the internet at all, per +//! family: a node without one doesn't publish its own public addresses of that family. +//! +//! Whether there is a route is asked of the operating system: a UDP socket "connected" +//! to the address (no packet is sent) fails where the routing table has none. Asked +//! per range (`ROUTE_PREFIX_V4`, `ROUTE_PREFIX_V6`) of the addresses heard of, and +//! asked again every `ROUTES_RECHECK`: a range gets a route (IPv6 comes up), and its +//! addresses are taken again as they're heard of next (the member list, identify); +//! one loses its route, and the addresses there are dropped. + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket}; + +use super::*; + +/// The ranges routes are asked for: an IPv4 /24 or an IPv6 /64, as far as a route +/// usually differs within them. +const ROUTE_PREFIX_V4: u8 = 24; +const ROUTE_PREFIX_V6: u8 = 64; +/// How long an answer is taken for one. +const ROUTES_RECHECK: Duration = Duration::from_secs(30); +/// Ranges remembered at most. +const MAX_RANGES: usize = 1024; +/// How long resolving `--libp2p-internet-hosts` may take, each time. +const RESOLVE_WITHIN: Duration = Duration::from_secs(5); + +/// Whether the routing table has a route to `ip`. +pub(super) fn has_route(ip: IpAddr) -> bool { + let any: SocketAddr = match ip { + IpAddr::V4(_) => (Ipv4Addr::UNSPECIFIED, 0).into(), + IpAddr::V6(_) => (Ipv6Addr::UNSPECIFIED, 0).into(), + }; + UdpSocket::bind(any) + .and_then(|socket| socket.connect((ip, 9))) + .is_ok() +} + +/// The answers, by range, and the question asked (`has_route`, or a stand-in in tests). +pub(super) struct Routes { + known: HashMap<IpRange, (bool, tokio::time::Instant)>, + /// Whether there is a route to the internet, per family (IPv4, IPv6), as last + /// asked (`internet`): own public addresses are published only then. True where + /// nobody asks (no `--libp2p-internet-hosts` of that family); false until the first + /// answer where someone does (resolving takes up to `RESOLVE_WITHIN`), so the first + /// member list entry doesn't list addresses nobody can reach. + internet: [bool; 2], + /// `--libp2p-internet-hosts`: names or addresses on the internet, whose routes + /// tell whether there is one to the internet at all. + pub(super) internet_hosts: Vec<String>, + probe: fn(IpAddr) -> bool, +} + +impl Default for Routes { + fn default() -> Self { + // Tests don't depend on the host's routes; those that care set `probe`. + #[cfg(test)] + let probe: fn(IpAddr) -> bool = |_| true; + #[cfg(not(test))] + let probe: fn(IpAddr) -> bool = has_route; + Self::with_probe(probe) + } +} + +/// The range of `ip` routes are asked for. +fn range_of(ip: IpAddr) -> IpRange { + let prefix = if ip.is_ipv4() { + ROUTE_PREFIX_V4 + } else { + ROUTE_PREFIX_V6 + }; + IpRange::masked(ip, prefix) +} + +impl Routes { + /// With `--libp2p-internet-hosts` `internet_hosts`. + pub(super) fn asking(internet_hosts: Vec<String>) -> Self { + let unknown = !internet_hosts.is_empty(); + Self { + internet: [!unknown, !unknown], + internet_hosts, + ..Self::default() + } + } + + pub(super) fn with_probe(probe: fn(IpAddr) -> bool) -> Self { + Self { + known: HashMap::new(), + internet: [true, true], + internet_hosts: Vec::new(), + probe, + } + } + + /// Whether `ip` has a route from here: asked if not known lately, told when the + /// answer is news (debug). + pub(super) fn routable(&mut self, ip: IpAddr) -> bool { + let range = range_of(ip); + let now = tokio::time::Instant::now(); + if let Some((routable, at)) = self.known.get(&range) + && now.saturating_duration_since(*at) < ROUTES_RECHECK + { + return *routable; + } + let routable = (self.probe)(ip); + let before = self.known.get(&range).map(|(routable, _)| *routable); + if before.is_none() && self.known.len() >= MAX_RANGES { + self.known.clear(); + } + self.known.insert(range, (routable, now)); + announce(range, before, routable); + routable + } + + /// Asks again for the ranges known; returns those that lost their route, whose + /// addresses are to go. + pub(super) fn recheck(&mut self) -> Vec<IpRange> { + let now = tokio::time::Instant::now(); + let mut lost = Vec::new(); + for (range, (routable, at)) in &mut self.known { + let now_routable = (self.probe)(range.network); + if *routable != now_routable { + announce(*range, Some(*routable), now_routable); + if !now_routable { + lost.push(*range); + } + } + *routable = now_routable; + *at = now; + } + lost + } + + /// Whether there's a route to the internet, per family, by the addresses + /// `--libp2p-internet-hosts` resolved to (`resolve_internet_hosts`): one of them + /// with a route is enough. A family none of them has isn't judged (taken as having + /// one). + pub(super) fn internet(&mut self, hosts: &[IpAddr]) { + for (family, name) in ["IPv4", "IPv6"].into_iter().enumerate() { + let of_family: Vec<IpAddr> = hosts + .iter() + .copied() + .filter(|ip| usize::from(ip.is_ipv6()) == family) + .collect(); + let routable = of_family.is_empty() || of_family.iter().any(|ip| (self.probe)(*ip)); + if routable != self.internet[family] { + if routable { + debug!( + family = name, + "A route to the internet again: own public addresses published again" + ); + } else { + debug!( + family = name, + "No route to the internet: own public addresses not published" + ); + } + } + self.internet[family] = routable; + } + } + + /// Whether an own address `ip` is worth publishing: not a public one of a family + /// with no route to the internet. + pub(super) fn publishable(&self, ip: IpAddr) -> bool { + !is_public(ip) || self.internet[usize::from(ip.is_ipv6())] + } +} + +fn announce(range: IpRange, before: Option<bool>, routable: bool) { + match (before, routable) { + (None | Some(true), false) => { + debug!(range = %range, "No route to an address range: its addresses left out until there is"); + } + (Some(false), true) => { + debug!(range = %range, "A route to an address range again: its addresses taken again"); + } + _ => {} + } +} + +/// The addresses `hosts` (names or addresses) resolve to, within `RESOLVE_WITHIN`; a +/// name that doesn't resolve is left out (told at debug level). +pub(super) async fn resolve_internet_hosts(hosts: &[String]) -> Vec<IpAddr> { + let mut addresses = Vec::new(); + for host in hosts { + if let Ok(ip) = host.parse::<IpAddr>() { + addresses.push(ip); + continue; + } + match tokio::time::timeout(RESOLVE_WITHIN, tokio::net::lookup_host((host.as_str(), 0))) + .await + { + Ok(Ok(found)) => addresses.extend(found.map(|address| address.ip())), + Ok(Err(error)) => { + debug!(host = %host, error = %error, "An internet host didn't resolve") + } + Err(_) => debug!(host = %host, "An internet host didn't resolve in time"), + } + } + addresses +} diff --git a/src/transport/libp2p/tests.rs b/src/transport/libp2p/tests.rs new file mode 100644 index 0000000..0f52e23 --- /dev/null +++ b/src/transport/libp2p/tests.rs @@ -0,0 +1,1026 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use super::*; +use crate::document::replica::Epoch; +use crate::transport::{HeadsSummary, Status}; + +/// Memory ports are process-wide. +static NEXT_PORT: AtomicU64 = AtomicU64::new(40_000); + +/// A node on the in-memory transport, listening on a port of its own (or nowhere), +/// and its address with its peer ID (or just its peer ID). +fn memory_node_with( + listen: bool, + relays: Vec<Multiaddr>, + relay: bool, +) -> (Libp2pTransport, Multiaddr) { + let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); + let key = Keypair::generate_ed25519(); + let address: Multiaddr = format!("/memory/{port}").parse().unwrap(); + let peer = key.public().to_peer_id(); + let transport = Libp2pTransport::start(Libp2pConfig { + key, + listen: if listen { + vec![address.clone()] + } else { + Vec::new() + }, + announce: Vec::new(), + relays, + relay, + continuum: true, + mdns: false, + upnp: false, + ignore: Vec::new(), + prefer: Vec::new(), + internet_hosts: Vec::new(), + public: false, + memory: true, + keepalive: Duration::from_secs(60), + }) + .unwrap(); + let address = if listen { + address.with_p2p(peer).unwrap() + } else { + Multiaddr::empty().with(Protocol::P2p(peer)) + }; + (transport, address) +} + +fn memory_node() -> (Libp2pTransport, Multiaddr) { + memory_node_with(true, Vec::new(), false) +} + +fn status() -> ParsedMessage { + ParsedMessage::Status(Status { + heads: HeadsSummary::of(Vec::new()), + epoch: Epoch::GENESIS, + epoch_seq: 3, + ready: true, + syncs_with_you: false, + leaving: false, + }) +} + +/// Starts `transport`'s consumer, forwarding what it receives. +fn received(transport: Arc<Libp2pTransport>) -> mpsc::UnboundedReceiver<(String, ParsedMessage)> { + let (sender, receiver) = mpsc::unbounded_channel(); + tokio::spawn(async move { + transport + .start_consumer(Box::new(move |from, message, _| { + let sender = sender.clone(); + Box::pin(async move { + let _ = sender.send((from, message)); + Ok(()) + }) + })) + .await + }); + receiver +} + +#[tokio::test] +async fn messages_both_ways_over_one_connection() { + let (a, a_address) = memory_node(); + let (b, _) = memory_node(); + let (a, b) = (Arc::new(a), Arc::new(b)); + let mut at_a = received(Arc::clone(&a)); + let mut at_b = received(Arc::clone(&b)); + + assert_eq!( + b.resolve_seed(&a_address.to_string()).await.unwrap(), + a.local_id() + ); + b.send_message(a.local_id(), &status()).await.unwrap(); + let (from, message) = at_a.recv().await.unwrap(); + assert_eq!(from, b.local_id()); + assert!(matches!(message, ParsedMessage::Status(status) if status.epoch_seq == 3)); + + // a knows no address of b, but b's connection is there. + a.send_message(b.local_id(), &status()).await.unwrap(); + let (from, _) = at_b.recv().await.unwrap(); + assert_eq!(from, a.local_id()); + + assert_eq!(a.connection(b.local_id()), Some(Connection::Direct)); + assert_eq!(b.connection(a.local_id()), Some(Connection::Direct)); + let stats = a.stats().libp2p.unwrap(); + assert!(stats.connections.direct >= 1); + assert_eq!(stats.connections.relayed, 0); +} + +#[tokio::test] +async fn a_seed_without_a_peer_id_is_connected_to_to_find_out() { + let (a, a_address) = memory_node(); + let (b, _) = memory_node(); + let mut bare = a_address.clone(); + bare.pop(); + assert_eq!( + b.resolve_seed(&bare.to_string()).await.unwrap(), + a.local_id() + ); + // And b now knows where a is. + assert_eq!(b.locations(a.local_id()), [a_address.to_string()]); +} + +#[tokio::test] +async fn listen_addresses_carry_the_peer_id() { + let (a, a_address) = memory_node(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while a.local_addresses().is_empty() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(a.local_addresses(), [a_address.to_string()]); + assert_eq!( + a.locate(&a_address.to_string()).as_deref(), + Some(a.local_id()) + ); +} + +/// Waits up to 5 s for `condition`. +async fn eventually(mut condition: impl FnMut() -> bool) -> bool { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while !condition() { + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + true +} + +/// A node learns where a peer that dialled it listens (identify), and can then +/// pass it on. +#[tokio::test] +async fn a_dialled_node_learns_where_the_dialler_listens() { + let (a, a_address) = memory_node(); + let (b, b_address) = memory_node(); + let (a, b) = (Arc::new(a), Arc::new(b)); + let _at_a = received(Arc::clone(&a)); + b.resolve_seed(&a_address.to_string()).await.unwrap(); + b.send_message(a.local_id(), &status()).await.unwrap(); + let b_id = b.local_id().to_owned(); + assert!( + eventually(|| a.locations(&b_id) == [b_address.to_string()]).await, + "a knows {:?} of b", + a.locations(&b_id) + ); +} + +#[tokio::test] +async fn announced_addresses_are_published_instead() { + let key = Keypair::generate_ed25519(); + let peer = key.public().to_peer_id(); + let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); + let announced: Multiaddr = "/ip4/203.0.113.7/tcp/9661".parse().unwrap(); + let transport = Libp2pTransport::start(Libp2pConfig { + key, + listen: vec![format!("/memory/{port}").parse().unwrap()], + announce: vec![announced.clone()], + relays: Vec::new(), + relay: false, + continuum: true, + mdns: false, + upnp: false, + ignore: Vec::new(), + prefer: Vec::new(), + internet_hosts: Vec::new(), + public: false, + memory: true, + keepalive: Duration::from_secs(60), + }) + .unwrap(); + assert_eq!( + transport.local_addresses(), + [announced.with_p2p(peer).unwrap().to_string()] + ); +} + +#[test] +fn address_ranges() { + let range = |text: &str| text.parse::<IpRange>().unwrap(); + let address = |text: &str| text.parse::<Multiaddr>().unwrap(); + let ranges = [range("10.244.0.0/16"), range("42::/16"), range("192.0.2.7")]; + for left_out in [ + "/ip4/10.244.3.2/udp/9661/quic-v1", + "/ip6/42:1:a46e:be4a:cb58:e460:bf0:142c/tcp/9661", + "/ip4/192.0.2.7/tcp/1", + ] { + assert!(ignored(&ranges, &address(left_out)), "{left_out}"); + } + for kept in [ + "/ip4/10.23.1.144/udp/9661/quic-v1", + "/ip4/10.245.0.1/tcp/1", + "/ip6/2a0a:4cc0:c1:5eb0::1/udp/9661/quic-v1", + "/ip4/192.0.2.8/tcp/1", + "/dns4/example.org/tcp/1", + ] { + assert!(!ignored(&ranges, &address(kept)), "{kept}"); + } + // Through a relay: judged by the connection to the relay, not here. + let relayed = format!("/ip4/10.244.0.1/tcp/1/p2p/{}/p2p-circuit", peer()); + assert!(!ignored(&ranges, &address(&relayed))); + assert!(range("0.0.0.0/0").contains("203.0.113.1".parse().unwrap())); + assert!(!range("0.0.0.0/0").contains("::1".parse().unwrap())); + for wrong in ["10.244.0.0/33", "42::/129", "nowhere", "10.0.0.0/x"] { + assert!(wrong.parse::<IpRange>().is_err(), "{wrong}"); + } + // Mycelium's range: a prefix that isn't a multiple of 8. + let mycelium = [range("400::/7")]; + assert!(ignored( + &mycelium, + &address("/ip6/45f:fa21:4bdd:1::1/tcp/9661") + )); + assert!(ignored(&mycelium, &address("/ip6/5ff::1/tcp/9661"))); + assert!(!ignored(&mycelium, &address("/ip6/600::1/tcp/9661"))); +} + +/// A preferred range all ignored is refused; one with an ignored range in it is fine +/// (with a warning). +#[test] +fn preferred_and_ignored_ranges() { + let check = |ignore: &[&str], prefer: &[&str]| { + let given = |ranges: &[&str]| { + ranges + .iter() + .map(|range| range.to_string()) + .collect::<Vec<_>>() + }; + let (ignore, prefer) = (given(ignore), given(prefer)); + super::config::check_preferred( + &super::config::parse_ranges("libp2p-ignore", &ignore).unwrap(), + &super::config::parse_ranges("libp2p-prefer", &prefer).unwrap(), + &ignore, + &prefer, + ) + }; + assert!(check(&["10.244.0.0/16"], &["10.23.0.0/16"]).is_ok()); + assert!(check(&["10.0.0.0/8"], &["10.23.0.0/16"]).is_err()); + assert!(check(&["10.23.0.0/16"], &["10.23.0.0/16"]).is_err()); + assert!(check(&["10.23.5.0/24"], &["10.23.0.0/16"]).is_ok()); + assert!(check(&["42::/16"], &["10.0.0.0/8"]).is_ok()); +} + +/// Ranges left out aren't published; announced addresses are, whatever they are. +#[test] +fn ignored_addresses_are_not_published() { + let parse = |text: &str| text.parse::<Multiaddr>().unwrap(); + let lan = parse("/ip4/10.23.1.5/udp/9663/quic-v1"); + let vpn = parse("/ip4/10.244.3.2/udp/9663/quic-v1"); + let mut book = Book { + listening: vec![lan.clone(), vpn.clone()], + ignored: vec!["10.244.0.0/16".parse().unwrap()], + ..Book::default() + }; + assert_eq!(book.direct(), [lan]); + book.announced = vec![vpn.clone()]; + assert_eq!(book.direct(), [vpn]); +} + +/// A port forwarded by hand: once AutoNAT found the node dialable at the router's +/// address, that is published with the listen addresses; announced ones still win. +#[test] +fn the_address_autonat_confirmed_is_published() { + let parse = |text: &str| text.parse::<Multiaddr>().unwrap(); + let lan = parse("/ip4/10.23.1.5/udp/9663/quic-v1"); + let public = parse("/ip4/91.65.180.254/udp/9663/quic-v1"); + let mut book = Book { + listening: vec![lan.clone()], + ..Book::default() + }; + assert_eq!(book.direct(), [lan.clone()]); + book.public_address = Some(public.clone().with_p2p(peer()).unwrap()); + assert_eq!(book.direct(), [public.clone(), lan.clone()]); + let announced = parse("/dns4/example.org/udp/9663/quic-v1"); + book.announced = vec![announced.clone()]; + assert_eq!(book.direct(), [announced]); +} + +#[test] +fn local_addresses_are_left_out_unless_there_are_only_local_ones() { + let parse = |text: &str| text.parse::<Multiaddr>().unwrap(); + let loopback = parse("/ip4/127.0.0.1/tcp/1"); + let link = parse("/ip6/fe80::1/tcp/1"); + let lan = parse("/ip4/192.168.1.2/tcp/1"); + let wireguard = parse("/ip6/fd00::2/tcp/1"); + assert_eq!( + published(&[ + loopback.clone(), + link.clone(), + lan.clone(), + wireguard.clone() + ]), + [lan, wireguard] + ); + assert_eq!( + published(&[loopback.clone(), link.clone()]), + [loopback, link] + ); +} + +/// b listens nowhere but through the relay r; a reaches it there. +#[tokio::test] +async fn a_node_nobody_can_dial_is_reached_through_a_relay() { + let (_r, r_address) = memory_node_with(true, Vec::new(), true); + let (b, _) = memory_node_with(false, vec![r_address.clone()], false); + let b = Arc::new(b); + let mut at_b = received(Arc::clone(&b)); + assert!( + eventually(|| !b.local_addresses().is_empty()).await, + "b got no slot on the relay" + ); + let relayed = b.local_addresses()[0].clone(); + assert!( + relayed.starts_with(&format!("{r_address}/p2p-circuit/")), + "{relayed}" + ); + + assert!(eventually(|| b.stats().libp2p.unwrap().relay_slots.held == 1).await); + let slots = b.stats().libp2p.unwrap().relay_slots; + assert_eq!((slots.granted, slots.not_granted), (1, 0)); + + let (a, _) = memory_node(); + assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); + a.send_message(b.local_id(), &status()).await.unwrap(); + let (from, _) = at_b.recv().await.unwrap(); + assert_eq!(from, a.local_id()); + assert!(a.connection(b.local_id()).is_some()); +} + +/// A seed that's a peer ID only is reached through the relays the node knows. +#[tokio::test] +async fn a_peer_id_seed_is_reached_through_a_relay() { + let (_r, r_address) = memory_node_with(true, Vec::new(), true); + let (b, _) = memory_node_with(false, vec![r_address.clone()], false); + let b = Arc::new(b); + let mut at_b = received(Arc::clone(&b)); + assert!( + eventually(|| !b.local_addresses().is_empty()).await, + "b got no slot on the relay" + ); + let (a, _) = memory_node_with(true, vec![r_address.clone()], false); + let seed = format!("/p2p/{}", b.local_id()); + assert_eq!(a.resolve_seed(&seed).await.unwrap(), b.local_id()); + assert_eq!(a.resolve_seed(b.local_id()).await.unwrap(), b.local_id()); + // Nothing to dial is remembered. + assert!(!lock(&a.book).addresses.contains_key(&b.peer_id)); + a.send_message(b.local_id(), &status()).await.unwrap(); + let (from, _) = at_b.recv().await.unwrap(); + assert_eq!(from, a.local_id()); + + // Without a relay, there's nowhere to look. + let (c, _) = memory_node(); + assert!(c.resolve_seed(&seed).await.is_err()); +} + +/// b listens nowhere; a, which can be dialled, reaches it through the relay r: b +/// then dials a directly (identify told it where), and both keep that. +#[tokio::test] +async fn a_relayed_connection_is_upgraded_where_one_side_can_be_dialled() { + let (_r, r_address) = memory_node_with(true, Vec::new(), true); + let (b, _) = memory_node_with(false, vec![r_address.clone()], false); + let b = Arc::new(b); + let mut at_b = received(Arc::clone(&b)); + assert!( + eventually(|| !b.local_addresses().is_empty()).await, + "b got no slot on the relay" + ); + let (a, _) = memory_node(); + let relayed = b.local_addresses()[0].clone(); + assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); + a.send_message(b.local_id(), &status()).await.unwrap(); + at_b.recv().await.unwrap(); + assert!( + eventually(|| b.connection(a.local_id()) == Some(Connection::Direct)).await, + "b didn't dial a directly" + ); + assert_eq!(a.connection(b.local_id()), Some(Connection::Direct)); + // The relayed one goes (`DUPLICATE_GRACE`); the direct one stays. + let deadline = tokio::time::Instant::now() + DUPLICATE_GRACE * 2; + while a.stats().libp2p.unwrap().connections.relayed > 0 { + assert!( + tokio::time::Instant::now() < deadline, + "the relayed connection stayed" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + b.send_message(a.local_id(), &status()).await.unwrap(); + assert_eq!(a.connection(b.local_id()), Some(Connection::Direct)); +} + +/// A relay that is no node (`--relay-only`) relays, and takes no messages itself. +#[tokio::test] +async fn a_relay_that_is_no_node() { + let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed); + let key = Keypair::generate_ed25519(); + let r_peer = key.public().to_peer_id(); + let r_listen: Multiaddr = format!("/memory/{port}").parse().unwrap(); + let r = Libp2pTransport::start(Libp2pConfig { + key, + listen: vec![r_listen.clone()], + announce: Vec::new(), + relays: Vec::new(), + relay: true, + continuum: false, + mdns: false, + upnp: false, + ignore: Vec::new(), + prefer: Vec::new(), + internet_hosts: Vec::new(), + public: false, + memory: true, + keepalive: Duration::from_secs(60), + }) + .unwrap(); + let r_address = r_listen.with_p2p(r_peer).unwrap(); + let (b, _) = memory_node_with(false, vec![r_address.clone()], false); + let b = Arc::new(b); + let mut at_b = received(Arc::clone(&b)); + assert!( + eventually(|| !b.local_addresses().is_empty()).await, + "b got no slot on the relay" + ); + let (a, _) = memory_node(); + let relayed = b.local_addresses()[0].clone(); + assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id()); + a.send_message(b.local_id(), &status()).await.unwrap(); + let (from, _) = at_b.recv().await.unwrap(); + assert_eq!(from, a.local_id()); + + // The relay itself doesn't speak continuum: it takes no messages, and a send to + // it is refused once identify has told (the relay's own refusal comes too late: + // the protocol goes with the first data, `swarm_config`), saying what to give + // as a seed instead. + assert_eq!( + a.resolve_seed(&r_address.to_string()).await.unwrap(), + r.local_id() + ); + assert!( + eventually(|| lock(&a.book).not_nodes.get(&r.peer_id) == Some(&true)).await, + "identify didn't tell" + ); + let refused = a.send_message(r.local_id(), &status()).await.unwrap_err(); + assert!( + refused.to_string().contains("is a relay, not a node"), + "{refused:#}" + ); + assert!( + r.start_consumer(Box::new(|_, _, _| Box::pin(async { Ok(()) }))) + .await + .is_err() + ); +} + +/// A relay configured without its peer ID is found out, and used. +#[tokio::test] +async fn a_relay_configured_without_its_peer_id() { + let (_r, r_address) = memory_node_with(true, Vec::new(), true); + let mut bare = r_address.clone(); + bare.pop(); + let (b, _) = memory_node_with(false, vec![bare], false); + assert!( + eventually(|| !b.local_addresses().is_empty()).await, + "b got no slot on the relay" + ); + assert!(b.local_addresses()[0].starts_with(&format!("{r_address}/p2p-circuit/"))); +} + +fn peer() -> PeerId { + Keypair::generate_ed25519().public().to_peer_id() +} + +#[test] +fn relays_are_picked_when_needed() { + use ::libp2p::autonat::NatStatus; + let local = peer(); + let offers: HashMap<PeerId, Multiaddr> = (0..4) + .map(|port| (peer(), format!("/memory/{port}").parse().unwrap())) + .collect(); + let check = + |direct, nat: &NatStatus, using: &[PeerId]| relay_check(local, direct, nat, &offers, using); + // Dialable, or not known yet with an address to dial: none. + assert!(check(true, &NatStatus::Public(Multiaddr::empty()), &[]).is_empty()); + assert!(check(false, &NatStatus::Public(Multiaddr::empty()), &[]).is_empty()); + assert!(check(true, &NatStatus::Unknown, &[]).is_empty()); + // No address to dial, or AutoNAT says it can't be dialled: some. + assert_eq!(check(false, &NatStatus::Unknown, &[]).len(), AUTO_RELAYS); + let picked = check(true, &NatStatus::Private, &[]); + assert_eq!(picked.len(), AUTO_RELAYS); + // One in use: the next in the same order. + let more = check(true, &NatStatus::Private, &[picked[0].0]); + assert_eq!(more.len(), AUTO_RELAYS - 1); + assert_eq!(more[0].0, picked[1].0); + let all: Vec<PeerId> = picked.iter().map(|(relay, _)| *relay).collect(); + assert!(check(true, &NatStatus::Private, &all).is_empty()); + // Never itself. + let only_us = HashMap::from([(local, Multiaddr::empty())]); + assert!(relay_check(local, false, &NatStatus::Private, &only_us, &[]).is_empty()); +} + +/// A relay says so in its addresses; b, which nobody can dial, learns of it (as from +/// the member list) and becomes reachable through it. +#[tokio::test] +async fn a_member_that_relays_is_picked() { + let (r, _) = memory_node_with(true, Vec::new(), true); + assert!(eventually(|| !r.local_addresses().is_empty()).await); + let offer = r + .local_addresses() + .into_iter() + .find(|address| address.ends_with("/p2p-circuit")) + .expect("the relay offers itself"); + let (b, _) = memory_node_with(false, Vec::new(), false); + assert_eq!(b.locate(&offer).as_deref(), Some(r.local_id())); + assert!( + eventually(|| b + .local_addresses() + .iter() + .any(|address| address.contains("/p2p-circuit/"))) + .await, + "b isn't reachable through r: {:?}", + b.local_addresses() + ); +} + +/// A peer on the LAN that speaks continuum becomes a seed once, at its address. +#[test] +fn public_ranges() { + let public = |ip: &str| book::is_public(ip.parse().unwrap()); + assert!(public("203.0.114.5")); + assert!(public("2a01:4f8::1")); + for not in [ + "10.1.2.3", + "100.64.0.1", + "192.0.2.1", + "198.18.0.1", + "224.0.0.1", + "255.255.255.255", + "::1", + "fd00::1", + "fe80::1", + "2001:db8::1", + "64:ff9b::1", + // Mycelium's overlay. + "45f:fa21:4bdd::1", + ] { + assert!(!public(not), "{not}"); + } +} + +/// An address where a dial reached another peer isn't kept for the one dialled, even +/// when it tells it again (identify) or the member list does. +#[test] +fn addresses_of_another_peer_are_forgotten() { + let mut book = Book::default(); + let them = peer(); + let lan: Multiaddr = "/ip4/192.168.1.5/tcp/9661".parse().unwrap(); + let elsewhere: Multiaddr = "/ip4/203.0.114.9/tcp/9661".parse().unwrap(); + book.remember(them, lan.clone()); + book.remember(them, elsewhere.clone()); + book.not_theirs(them, lan.clone().with_p2p(them).unwrap()); + assert_eq!(book.addresses[&them], [elsewhere.clone()]); + book.remember(them, lan.clone()); + book.identified(them, vec![lan, elsewhere.clone()]); + assert_eq!(book.addresses[&them], [elsewhere]); +} + +#[test] +fn lan_peers_become_seeds_once() { + let mut book = Book::default(); + let lan = peer(); + let address: Multiaddr = "/ip4/192.168.1.5/tcp/9661".parse().unwrap(); + book.discover(lan); // No address known: nothing. + assert!(book.discovered.is_empty()); + book.remember(lan, address.clone()); + book.discover(lan); + book.discover(lan); + assert_eq!(book.discovered, [address.with_p2p(lan).unwrap()]); +} + +#[test] +fn named_relays() { + let (local, r1, r2) = (peer(), peer(), peer()); + let at = |port: u16| -> Multiaddr { format!("/ip6/fd00::1/tcp/{port}").parse().unwrap() }; + let mut book = Book::default(); + // With a peer ID: known at once; without: to be found out, once. + assert_eq!( + book.name_relay(at(1).with_p2p(r1).unwrap(), Named::Configured, local), + None + ); + assert_eq!( + book.name_relay(at(2), Named::Configured, local), + Some(at(2)) + ); + assert_eq!(book.name_relay(at(2), Named::Configured, local), None); + assert_eq!( + book.name_relay(at(3).with_p2p(local).unwrap(), Named::Configured, local), + None + ); + assert!(book.relay_found(at(2), r2, local)); + assert_eq!(book.named(Named::Configured).len(), 2); + // A relay named with its peer ID isn't found out again; one without is. + assert_eq!(book.find_again(r1), None); + assert_eq!(book.find_again(r2), Some(at(2))); + assert!(!book.named(Named::Configured).contains_key(&r2)); + + // Listed: configured wins; unlisted ones go, unless also configured. + let r3 = peer(); + let listed = vec![ + at(1).with_p2p(r1).unwrap(), + at(4).with_p2p(r3).unwrap(), + at(5), + ]; + assert_eq!(book.learn_listed(listed.clone(), local), Some(vec![at(5)])); + assert_eq!(book.learn_listed(listed, local), None); + assert_eq!(book.named_relays[&r1].named, Named::Configured); + assert_eq!(book.named(Named::Listed).keys().collect::<Vec<_>>(), [&r3]); + assert_eq!(book.learn_listed(Vec::new(), local), Some(Vec::new())); + assert!(book.named(Named::Listed).is_empty()); + assert!(!book.finding.contains_key(&at(5))); + assert!(!book.relay_found(at(5), peer(), local)); // No longer listed. + assert_eq!(book.named(Named::Configured).len(), 1); +} + +#[test] +fn known_addresses() { + let mut book = Book::default(); + let (relay, us) = (peer(), peer()); + let direct: Multiaddr = "/ip6/fd00::1/tcp/9661".parse().unwrap(); + let relayed: Multiaddr = format!("/ip6/fd00::2/tcp/9661/p2p/{relay}/p2p-circuit/p2p/{us}") + .parse() + .unwrap(); + assert!(!book.knows(us, &[direct.clone()])); + book.remember(us, relayed.clone()); + book.remember(us, direct.clone()); + // In any order, with or without the peer ID. + assert!(book.knows(us, &[relayed.clone(), direct.clone()])); + assert!(book.knows(us, &[direct.clone().with_p2p(us).unwrap()])); + assert!(!book.knows(us, &["/ip6/fd00::3/tcp/9661".parse().unwrap()])); +} + +/// A connected peer's identify says where it is: older direct addresses go, and others +/// aren't taken while it stays connected; those through relays and by name stay. +#[test] +fn identify_replaces_direct_addresses() { + let mut book = Book::default(); + let (relay, them) = (peer(), peer()); + let old: Multiaddr = "/ip4/10.244.0.7/tcp/9661".parse().unwrap(); + let lan: Multiaddr = "/ip4/192.168.1.7/tcp/9661".parse().unwrap(); + let named: Multiaddr = "/dns4/node.example/tcp/9661".parse().unwrap(); + let relayed: Multiaddr = format!("/ip4/192.0.2.1/tcp/9661/p2p/{relay}/p2p-circuit") + .parse() + .unwrap(); + book.remember(them, old.clone()); + book.remember(them, named.clone()); + book.remember(them, relayed.clone()); + + book.identified(them, vec![lan.clone().with_p2p(them).unwrap()]); + assert!(book.knows(them, &[lan.clone(), named.clone(), relayed.clone()])); + assert!(!book.knows(them, &[old.clone()])); + // The member list, say, still has the old one. + book.remember(them, old.clone()); + assert!(!book.knows(them, &[old.clone()])); + + // Disconnected (`driver`): taken again, it may have moved there. + book.identified.remove(&them); + book.remember(them, old.clone()); + assert!(book.knows(them, &[old])); +} + +/// Of a peer with more addresses than are kept, those that matter stay: in the ranges +/// `libp2p-prefer` names first, then public ones, then through relays, then other +/// private ones. +#[test] +fn identified_addresses_preferred_first() { + let mut book = Book { + preferred: vec!["10.23.0.0/16".parse().unwrap()], + ..Book::default() + }; + let (relay, them) = (peer(), peer()); + let lan: Multiaddr = "/ip4/10.23.7.7/tcp/9661".parse().unwrap(); + // Outside the documentation ranges, which aren't public (`is_public`). + let public: Multiaddr = "/ip4/203.0.114.7/tcp/9661".parse().unwrap(); + let relayed: Multiaddr = format!("/ip4/192.0.2.1/tcp/9661/p2p/{relay}/p2p-circuit") + .parse() + .unwrap(); + let bridges: Vec<Multiaddr> = (0..ADDRESSES_PER_PEER) + .map(|i| format!("/ip4/172.17.{i}.1/tcp/9661").parse().unwrap()) + .collect(); + let mut listen = bridges.clone(); + listen.extend([relayed.clone(), public.clone(), lan.clone()]); + + book.identified(them, listen); + let known = &book.addresses[&them]; + assert_eq!(known.len(), ADDRESSES_PER_PEER); + assert_eq!(known[..3], [lan.clone(), public, relayed]); + + // Our own: those listened on in the preferred ranges published first. + book.listening = vec![bridges[0].clone(), lan.clone()]; + assert_eq!(book.direct(), [lan, bridges[0].clone()]); +} + +/// A peer's addresses in a range this node leaves out aren't kept: never dialled, they +/// would take a place, and be passed on. +#[test] +fn ignored_addresses_not_kept() { + let mut book = Book { + ignored: vec!["10.244.0.0/16".parse().unwrap()], + ..Book::default() + }; + let them = peer(); + let vpn: Multiaddr = "/ip4/10.244.3.2/tcp/9661".parse().unwrap(); + let lan: Multiaddr = "/ip4/10.23.1.144/tcp/9661".parse().unwrap(); + let relay = peer(); + // Through a relay there: dialling the relay is refused. + let via_vpn: Multiaddr = format!("/ip4/10.244.0.1/tcp/9661/p2p/{relay}/p2p-circuit") + .parse() + .unwrap(); + let via_lan: Multiaddr = format!("/ip4/10.23.0.1/tcp/9661/p2p/{relay}/p2p-circuit") + .parse() + .unwrap(); + book.remember(them, vpn.clone()); + book.remember(them, via_vpn.clone()); + book.identified(them, vec![vpn.clone(), lan.clone(), via_lan.clone()]); + assert!(book.knows(them, &[lan, via_lan])); + assert!(!book.knows(them, &[vpn])); + assert!(!book.knows(them, &[via_vpn])); +} + +/// A node connected directly over the WLAN dials the ethernet address it later hears +/// of, once; and when an interface of ours comes, every address again, the one in use +/// too. +#[test] +fn better_paths_explored() { + let mut book = Book::default(); + let them = peer(); + let wlan: Multiaddr = "/ip4/192.168.1.7/tcp/9661".parse().unwrap(); + let ethernet: Multiaddr = "/ip4/192.168.2.7/tcp/9661".parse().unwrap(); + book.continuum.insert(them); + book.remember(them, wlan.clone()); + // Connected (inbound: from another port) over the WLAN. + book.connections.insert( + ConnectionId::new_unchecked(1), + (them, false, "/ip4/192.168.1.7/tcp/40000".parse().unwrap()), + ); + assert!(book.to_explore(them, false).is_empty()); + book.identified(them, vec![wlan.clone(), ethernet.clone()]); + assert_eq!(book.to_explore(them, false), [ethernet.clone()]); + assert!(book.to_explore(them, false).is_empty()); + // An interface of ours came (`driver`): all of them again. + book.explored.clear(); + let mut again = book.to_explore(them, true); + again.sort(); + let mut both = vec![wlan, ethernet]; + both.sort(); + assert_eq!(again, both); +} + +/// Addresses without a route from here are left out, as if ignored: not kept, nor +/// dialled; and own public ones of a family without a route to the internet (by +/// `--libp2p-internet-hosts`) aren't published. The routing table is asked for real only by `has_route`. +#[test] +fn unroutable_addresses_left_out() { + assert!(has_route("127.0.0.1".parse().unwrap())); + // No IPv6 here. + let mut book = Book { + routes: Routes::with_probe(|ip| ip.is_ipv4()), + listening: vec![ + "/ip4/192.168.1.2/tcp/9661".parse().unwrap(), + "/ip6/2a0a:4cc0::2/tcp/9661".parse().unwrap(), + ], + ..Book::default() + }; + let them = peer(); + let v4: Multiaddr = "/ip4/192.168.1.7/tcp/9661".parse().unwrap(); + let v6: Multiaddr = "/ip6/2a0a:4cc0::7/tcp/9661".parse().unwrap(); + book.identified(them, vec![v4.clone(), v6.clone()]); + assert!(book.knows(them, &[v4])); + assert!(!book.knows(them, &[v6])); + // Nobody asked whether there's a route to the internet: everything is published. + assert_eq!(book.direct().len(), 2); + book.routes.internet(&[ + "1.1.1.1".parse().unwrap(), + "2606:4700:4700::1111".parse().unwrap(), + ]); + assert_eq!( + book.direct(), + ["/ip4/192.168.1.2/tcp/9661".parse::<Multiaddr>().unwrap()] + ); +} + +/// `--libp2p-internet-hosts` takes addresses as they are (names are resolved). +#[tokio::test] +async fn internet_hosts_resolved() { + let resolved = resolve_internet_hosts(&["1.1.1.1".to_owned(), "::1".to_owned()]).await; + assert_eq!( + resolved, + [ + "1.1.1.1".parse::<std::net::IpAddr>().unwrap(), + "::1".parse().unwrap() + ] + ); +} + +/// Peers nobody mentions any more are forgotten, but connected ones. +#[test] +fn stale_peers_are_forgotten() { + let (old, current, connected) = (peer(), peer(), peer()); + let address: Multiaddr = "/ip6/fd00::1/tcp/9661".parse().unwrap(); + let mut book = Book::default(); + for peer in [old, current, connected] { + book.remember(peer, address.clone()); + } + book.relays.insert(old, address.clone()); + book.continuum.insert(old); + // From after the mentions above (taken at `Instant::now()`), however fast or + // slow the clock moved meanwhile. + let later = tokio::time::Instant::now() + STALE_AFTER; + // `current` was mentioned again since. + book.seen.insert(current, later); + assert_eq!(book.prune(later, |peer| *peer == connected), 1); + assert!(!book.addresses.contains_key(&old)); + assert!(!book.relays.contains_key(&old) && !book.continuum.contains(&old)); + assert!(book.addresses.contains_key(¤t)); + assert!(book.addresses.contains_key(&connected)); + assert_eq!(book.prune(later, |_| false), 1); // `connected`, once it isn't. +} + +#[test] +fn relay_failures_are_short() { + assert_eq!( + relay_failure_text( + "Custom { kind: Other, error: Other(Left(Left(Left(Reservation(Protocol(NoAddressesInReservation)))))) }", + "x" + ), + "the relay gave no address to be reached through it" + ); + assert_eq!( + relay_failure_text( + "Codec(Error(Custom { kind: Other, error: Failed }))", + "Failed" + ), + "the relay refused the circuit relay protocol (is it a relay?)" + ); + assert_eq!( + relay_failure_text("Io(…)", "connection reset"), + "connection reset" + ); + assert_eq!(relay_failure(&Ok(())), "closed"); +} + +/// Only refusals pause a relay; a slot that ended otherwise is asked for again. +#[test] +fn only_refusals_pause_relays() { + let refused = [ + ("Reservation(Protocol(NoAddressesInReservation))", "x"), + ("Reservation(Status(ResourceLimitExceeded))", "x"), + ("Reservation(Status(PermissionDenied))", "x"), + ("Denied { cause: \"connection gated\" }", "x"), + ("Upgrade(UnsupportedProtocol)", "x"), + ("Dial(WrongPeerId { obtained: … })", "x"), + ( + "Codec(Error(Custom { kind: Other, error: Failed }))", + "Failed", + ), + ]; + for (debug, innermost) in refused { + assert!(relay_refused(debug, innermost), "{debug}"); + } + let not = [ + ("Io(Custom { kind: Other, error: Timeout })", "Timeout"), + ( + "Io(Custom { kind: Other, error: Right(Closed) })", + "connection is closed", + ), + ("Io(…)", "connection reset"), + ]; + for (debug, innermost) in not { + assert!(!relay_refused(debug, innermost), "{debug}"); + } +} + +#[test] +fn listen_addresses_offered_for_hole_punching() { + let parse = |text: &str| text.parse::<Multiaddr>().unwrap(); + assert!(offered(&parse("/ip4/192.168.1.2/udp/9661/quic-v1"))); + assert!(offered(&parse("/ip6/fd00::2/tcp/9661"))); + assert!(!offered(&parse("/ip4/127.0.0.1/udp/9661/quic-v1"))); + assert!(!offered(&parse("/ip6/fe80::1/tcp/9661"))); + let relay = PeerId::random(); + assert!(!offered(&parse(&format!( + "/ip4/192.168.1.2/tcp/9661/p2p/{relay}/p2p-circuit" + )))); +} + +/// libp2p's nested dial errors come out as each address's cause, once. +#[test] +fn dial_errors_are_short() { + use ::libp2p::core::transport::TransportError; + let peer = peer(); + let address: Multiaddr = format!("/ip6/fd00::1/tcp/9661/p2p/{peer}").parse().unwrap(); + let nested = std::io::Error::other( + "Multiple dial errors occurred:\n - Network is unreachable (os error 101): Multiple dial errors occurred:\n - Network is unreachable (os error 101): Network is unreachable (os error 101)", + ); + let error = + ::libp2p::swarm::DialError::Transport(vec![(address, TransportError::Other(nested))]); + assert_eq!( + describe_dial_error(&error), + "/ip6/fd00::1/tcp/9661: Network is unreachable (os error 101)" + ); + let refused = ::libp2p::swarm::DialError::Transport(vec![( + "/ip4/10.0.0.1/tcp/1".parse().unwrap(), + TransportError::Other(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)), + )]); + assert_eq!( + describe_dial_error(&refused), + "/ip4/10.0.0.1/tcp/1: connection refused" + ); +} + +/// A relay doesn't dial the nodes it relays for through itself. +#[test] +fn no_circuits_through_ourselves() { + let (us, other_relay, target) = (peer(), peer(), peer()); + let through = |relay: PeerId| -> Multiaddr { + format!("/ip6/fd00::1/tcp/9661/p2p/{relay}/p2p-circuit") + .parse() + .unwrap() + }; + let direct: Multiaddr = "/ip6/fd00::2/tcp/9661".parse().unwrap(); + assert_eq!(relay_of(&through(us)), Some(us)); + assert_eq!(relay_of(&direct), None); + assert_eq!(relay_of(&direct.clone().with_p2p(target).unwrap()), None); + assert_eq!( + dialable(&[through(us), through(other_relay), direct.clone()], us), + [through(other_relay), direct] + ); +} + +#[tokio::test] +async fn unknown_peers_are_refused() { + let (b, _) = memory_node(); + let stranger = Keypair::generate_ed25519() + .public() + .to_peer_id() + .to_string(); + assert!(b.send_message(&stranger, &status()).await.is_err()); + assert!(b.resolve_seed(&stranger).await.is_err()); + assert_eq!(b.connection(&stranger), None); +} + +#[test] +fn a_direct_connection_wins_over_a_relayed_one() { + let mut book = Book::default(); + let (relayed_only, both) = (peer(), peer()); + let mut id = 0; + let mut connect = |book: &mut Book, peer: PeerId, relayed: bool| { + id += 1; + let address: Multiaddr = format!("/ip4/10.0.0.{id}/tcp/1").parse().unwrap(); + book.connections + .insert(ConnectionId::new_unchecked(id), (peer, relayed, address)); + }; + connect(&mut book, relayed_only, true); + connect(&mut book, both, true); + connect(&mut book, both, false); + assert_eq!(book.connection(&relayed_only), Some(Connection::Relayed)); + assert_eq!(book.connection(&both), Some(Connection::Direct)); + assert_eq!(book.connection(&peer()), None); + assert_eq!( + book.location(&both).map(|address| address.to_string()), + Some("/ip4/10.0.0.3/tcp/1".to_owned()) + ); + let connections = book.stats().connections; + assert_eq!((connections.direct, connections.relayed), (1, 2)); + // Of several, the oldest direct one stays; a relayed one only without a direct. + connect(&mut book, both, false); + assert_eq!( + book.duplicates(&both), + [ + ConnectionId::new_unchecked(4), + ConnectionId::new_unchecked(2) + ] + ); + assert!(book.duplicates(&relayed_only).is_empty()); + // Also where one of them carries a slot, ours or the peer's: messages would go over + // either. + book.slots.insert(both); + assert_eq!(book.duplicates(&both).len(), 2); + book.slots.clear(); + book.slots_here.insert(both); + assert_eq!(book.duplicates(&both).len(), 2); + book.slots_here.clear(); + // Once measured, the fastest direct one stays, however young. + book.rtts + .insert(ConnectionId::new_unchecked(3), Duration::from_millis(40)); + book.rtts + .insert(ConnectionId::new_unchecked(4), Duration::from_millis(2)); + assert_eq!( + book.duplicates(&both), + [ + ConnectionId::new_unchecked(3), + ConnectionId::new_unchecked(2) + ] + ); + assert_eq!( + book.location(&both).map(|address| address.to_string()), + Some("/ip4/10.0.0.4/tcp/1".to_owned()) + ); +} diff --git a/src/transport/mod.rs b/src/transport/mod.rs index fb13ad9..96f7651 100644 --- a/src/transport/mod.rs +++ b/src/transport/mod.rs @@ -1,7 +1,5 @@ use anyhow::Result; use async_trait::async_trait; -use automerge::ChangeHash; -use automerge::sync::Message; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::boxed::Box; @@ -11,6 +9,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; pub(crate) use crate::document::replica::Epoch; +use crate::document::ChangeHash; + // Used by the transports that don't authenticate their peers themselves (`tor`). #[cfg_attr(not(feature = "tor"), allow(dead_code))] pub(crate) mod challenge; @@ -18,6 +18,7 @@ pub(crate) mod firewall; pub(crate) mod identity; #[cfg(feature = "libp2p")] pub(crate) mod libp2p; +#[cfg(feature = "mycelium")] pub(crate) mod mycelium; pub(crate) mod paths; #[cfg(test)] @@ -28,6 +29,67 @@ pub(crate) mod tor; pub(crate) mod traffic; pub(crate) mod wire; +#[cfg(not(any(feature = "mycelium", feature = "libp2p", feature = "tor")))] +compile_error!( + "continuum needs a transport: build with at least one of the mycelium, libp2p and tor features, e.g. `cargo build --features mycelium`" +); + +/// The options of the transports this build has, each checked by its transport when the +/// configuration is loaded (`config::Settings::check`, after the transports enabled): +/// Mycelium's always (its conflicts are errors wherever they come from, as before); +/// libp2p's and Tor's if the node uses them. Options of transports not built aren't +/// checked: they're ignored, and said so (`config::Loaded::unused_transport_options`). +pub(crate) fn check(settings: &crate::config::Settings) -> Result<()> { + let uses = |name: &str| settings.transports.iter().any(|used| used == name); + let _ = uses; + #[cfg(feature = "mycelium")] + mycelium::check(settings)?; + #[cfg(feature = "libp2p")] + if uses("libp2p") { + libp2p::check(settings)?; + } + #[cfg(feature = "tor")] + if uses("tor") { + tor::TorTransport::check(settings)?; + } + Ok(()) +} + +/// Starts the transport `name` (one of `config::TRANSPORTS`) as `settings` configure +/// it: its scheme and the transport, or None if `stop` came first (some take a while to +/// start: a supervised Mycelium daemon, arti). `key`: the node key, which some prove +/// themselves. +pub(crate) async fn start( + name: &str, + settings: &crate::config::Settings, + key: &std::sync::Arc<crate::identity::NodeKey>, + stop: tokio::sync::watch::Receiver<bool>, +) -> Result<Option<(&'static str, std::sync::Arc<dyn Transport>)>> { + use std::sync::Arc; + let _ = (key, &stop); + let started: Option<(&'static str, Arc<dyn Transport>)> = match name { + #[cfg(feature = "mycelium")] + "mycelium" => mycelium::start(settings, stop) + .await? + .map(|transport| (mycelium::SCHEME, Arc::new(transport) as Arc<dyn Transport>)), + #[cfg(feature = "libp2p")] + "libp2p" => { + let _ = stop; + let config = libp2p::Libp2pConfig::from_settings(settings, false)?; + Some(( + libp2p::SCHEME, + Arc::new(libp2p::Libp2pTransport::start(config)?) as Arc<dyn Transport>, + )) + } + #[cfg(feature = "tor")] + "tor" => tor::TorTransport::from_settings(settings, key, stop) + .await? + .map(|transport| (tor::SCHEME, Arc::new(transport) as Arc<dyn Transport>)), + other => anyhow::bail!("transports: no {other} in this build"), + }; + Ok(started) +} + /// Random identifier of one run of a node (from start to exit). pub(crate) type RunId = [u8; 16]; @@ -137,6 +199,9 @@ pub(crate) struct Status { /// The sender would sync with the recipient: counts it as an active member on its /// own epoch. pub(crate) syncs_with_you: bool, + /// The sender is shutting down: its farewell (`keepalive::farewell`). The + /// recipient holds it dormant at once, rather than after `dormant_after` of silence. + pub(crate) leaving: bool, } /// Cheap to clone: large parts are shared (`bytes::Bytes`). @@ -154,8 +219,17 @@ pub(crate) enum ParsedMessage { Me(IdentityProof), /// Asks for a PONG with this nonce, over the same path (`transport::paths`). Ping(u64), - /// Answers a PING. - Pong(u64), + /// Answers a PING: its nonce, and when it was answered by the answerer's clock + /// (`at`), for comparing clocks (`transport::paths`). + Pong { + nonce: u64, + at: crate::clock::UnixMillis, + }, + /// Asks a member how it is linked to its peers, for a map of the swarm + /// (`swarm::links`). + Links(LinksRequest), + /// Answers a LINKS. + LinksReply(LinksReply), } /// A node's proof that it holds the key of its node ID (`identity`), sending from its @@ -281,6 +355,8 @@ pub(crate) enum MessageKind { Me, Ping, Pong, + Links, + LinksReply, } impl ParsedMessage { @@ -295,7 +371,9 @@ impl ParsedMessage { ParsedMessage::Who(_) => MessageKind::Who, ParsedMessage::Me(_) => MessageKind::Me, ParsedMessage::Ping(_) => MessageKind::Ping, - ParsedMessage::Pong(_) => MessageKind::Pong, + ParsedMessage::Pong { .. } => MessageKind::Pong, + ParsedMessage::Links(_) => MessageKind::Links, + ParsedMessage::LinksReply(_) => MessageKind::LinksReply, } } @@ -310,14 +388,79 @@ impl ParsedMessage { Some((status.epoch, status.epoch_seq)) } ParsedMessage::Blob(blob) => Some((blob.epoch, blob.epoch_seq)), + ParsedMessage::Links(request) => Some((request.epoch, request.epoch_seq)), + ParsedMessage::LinksReply(reply) => Some((reply.epoch, reply.epoch_seq)), ParsedMessage::Who(_) | ParsedMessage::Me(_) | ParsedMessage::Ping(_) - | ParsedMessage::Pong(_) => None, + | ParsedMessage::Pong { .. } => None, } } } +/// LINKS: what a member is asked for a map of the swarm (`swarm::links`). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct LinksRequest { + #[serde(with = "crate::transport::wire::codec::epoch")] + pub(crate) epoch: Epoch, + /// See `SyncEnvelope::epoch_seq`. + pub(crate) epoch_seq: u32, + /// Repeated in the reply: which request it answers. + pub(crate) nonce: u64, + /// For this long (seconds), a LINKS_REPLY again whenever the links change (a + /// subscription; the next LINKS renews it). 0: this reply only, and a subscription + /// ends. + pub(crate) subscribe_secs: u32, +} + +/// LINKS_REPLY: how the sender is linked to its neighbours, as it measures them. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct LinksReply { + #[serde(with = "crate::transport::wire::codec::epoch")] + pub(crate) epoch: Epoch, + /// See `SyncEnvelope::epoch_seq`. + pub(crate) epoch_seq: u32, + /// The request's. + pub(crate) nonce: u64, + /// At most `swarm::links::MAX_LINKS`. + pub(crate) links: Vec<Link>, +} + +/// One of a member's links, in a LINKS_REPLY. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct Link { + /// The peer's node ID. + pub(crate) peer: String, + /// The transport its messages go over now (the scheme of the path in use), if any. + pub(crate) transport: Option<String>, + /// Only through a relay (libp2p). + pub(crate) relayed: bool, + /// The path in use is down: none to the peer answers (`paths`), whatever its + /// connection was. + pub(crate) down: bool, + /// The round trip on that path, in microseconds, if measured. + pub(crate) rtt_us: Option<u32>, + /// Silent for long (`peers.md`, dormancy). + pub(crate) dormant: bool, + /// The peer's other transports (`swarm::links::MAX_OTHERS` at most): where else + /// it proved itself, those paths up. + pub(crate) others: Vec<OtherPath>, + /// How far the peer's clock is ahead of the member's (`paths::ClockOffset`), if + /// measured: with every member's, a map of the swarm shows whose clock is off. + pub(crate) clock: Option<paths::ClockOffset>, +} + +/// A transport of a link besides the one in use, in a LINKS_REPLY. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct OtherPath { + /// Its scheme (`mycelium`, `libp2p`, `tor`). + pub(crate) transport: String, + /// The round trip of its fastest path, in microseconds, if any is measured. + pub(crate) rtt_us: Option<u32>, + /// Only through a relay (libp2p): no direct connection at any of its paths. + pub(crate) relayed: bool, +} + #[derive(Clone, Serialize, Deserialize)] pub(crate) struct HelloRequest { /// The sender's swarm, if it knows it (a new node doesn't yet). A receiver in @@ -361,17 +504,34 @@ pub(crate) struct Destination { pub(crate) proven: bool, /// The transport's connection to it, if the transport has connections. pub(crate) connection: Option<Connection>, + /// While it is connected only through a relay: how hole punching went. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) punching: Option<Punching>, /// Every address it proved itself at, best first, as the paths layer measures them /// (`transport::paths`). pub(crate) paths: Vec<stats::PathInfo>, } +/// Hole punching to a peer connected only through a relay (libp2p), once a punch failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Punching { + /// Punches that failed in a row. + pub(crate) failed: usize, + /// When it is tried again, in seconds from now (0: due). + pub(crate) next_in_secs: u64, +} + /// How a transport with connections (libp2p) is connected to a peer. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] +// Only libp2p and Tor have connections; without both (the cargo features) nothing makes +// one. +#[cfg_attr(not(any(feature = "libp2p", feature = "tor")), allow(dead_code))] pub(crate) enum Connection { + /// libp2p's to the peer itself; Tor's always (an onion circuit). Direct, - /// Only through a relay. + /// Only through a relay (libp2p). Relayed, } @@ -489,7 +649,7 @@ pub(crate) enum SyncBody { /// The changes in the history of `to` but not of `from`: what a peer at `from` lacks of /// a document at `to`. Every node holding `to` produces the same bytes for it -/// (`swarm::blob::delta_bytes`). +/// (`document::changes::delta_bytes`). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub(crate) struct DeltaSpec { /// Sorted. @@ -523,23 +683,13 @@ impl DeltaSpec { } } -/// An Automerge sync message in Automerge's own encoding. Decoded only where it is -/// applied (`decode`), after the sender proved to be a member and off the consumer: a -/// large message from anyone else costs nothing but its bytes. +/// A sync message as the document's engine encodes it (`document::sync`): bytes to the +/// transports. Decoded only where it is applied (`document::sync::decode`), after the sender +/// proved to be a member and off the consumer: a large message from anyone else costs +/// nothing but its bytes. #[derive(Clone)] pub(crate) struct SyncPayload(pub(crate) bytes::Bytes); -impl SyncPayload { - pub(crate) fn encode(message: Message) -> Self { - Self(message.encode().into()) - } - - pub(crate) fn decode(&self) -> Result<Message> { - Message::decode(&self.0) - .map_err(|error| anyhow::anyhow!("Automerge decode failed: {error:?}")) - } -} - /// Sizes instead of contents: logged messages stay short. impl std::fmt::Debug for SyncPayload { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -585,7 +735,7 @@ impl<T: std::fmt::Display> std::fmt::Display for Addressed<T> { /// consumer hands them on as soon as they are read (`hand_on_probe`), not after the /// messages read before them are handled (which can take a while: sync). pub(crate) fn is_probe(message: &ParsedMessage) -> bool { - matches!(message, ParsedMessage::Ping(_) | ParsedMessage::Pong(_)) + matches!(message, ParsedMessage::Ping(_) | ParsedMessage::Pong { .. }) } /// Hands a PING or PONG (`is_probe`) to `handler` in a task of its own; `scheme` is the @@ -618,6 +768,13 @@ pub(crate) trait Transport: Send + Sync { /// a message sent is delivered or lost for good. fn delivery_limit(&self) -> std::time::Duration; + /// The first wait for an answer to a sync message (`Timing::sync_reply_timeout`): by + /// default `SYNC_REPLY_TIMEOUT`, longer on a transport whose round trips take + /// seconds. + fn sync_reply_timeout(&self) -> std::time::Duration { + crate::state::SYNC_REPLY_TIMEOUT + } + /// Where this node can be reached, as addresses other nodes can use: published in /// the member list (`swarm::members`). By default its identity, for a transport /// whose identities are addresses (Mycelium: a public key). @@ -668,6 +825,19 @@ pub(crate) trait Transport: Send + Sync { Vec::new() } + /// Has the transport tell `found` whenever it finds a seed by itself + /// (`discovered_seeds`). By default it never does. + fn on_seeds_found(&self, found: std::sync::Arc<tokio::sync::Notify>) { + let _ = found; + } + + /// Has the transport tell `reachable` whenever others may reach this node where they + /// couldn't before (libp2p: a relay granted it a slot, or AutoNAT found it public). + /// By default it never does. + fn on_reachable(&self, reachable: std::sync::Arc<tokio::sync::Notify>) { + let _ = reachable; + } + /// Whether this node has a way to reach `peer` at all: an address of it on one of /// its transports. Members it can't reach aren't its neighbours (`swarm::topology`). /// By default yes, as for `local_addresses`. @@ -712,6 +882,13 @@ pub(crate) trait Transport: Send + Sync { None } + /// How hole punching to `target` went, if a punch failed and it is still connected + /// only through a relay (libp2p). By default never. + fn punching(&self, target: &str) -> Option<Punching> { + let _ = target; + None + } + /// The smoothed round-trip time of the path messages to `peer` go over, if /// measured (`transport::paths`). By default none. fn rtt(&self, peer: &str) -> Option<std::time::Duration> { @@ -719,6 +896,13 @@ pub(crate) trait Transport: Send + Sync { None } + /// How far `peer`'s clock is ahead of ours, as its PONGs say (`paths::ClockOffset`): + /// the narrowest bound of its paths. By default unknown. + fn clock_offset(&self, peer: &str) -> Option<paths::ClockOffset> { + let _ = peer; + None + } + /// Whether the transport answers PINGs with PONGs itself (the paths layer, /// `transport::paths`); if not (a bare transport, in tests), the node does. By /// default not. diff --git a/src/transport/mycelium/mod.rs b/src/transport/mycelium/mod.rs index 5e8dc05..9078134 100644 --- a/src/transport/mycelium/mod.rs +++ b/src/transport/mycelium/mod.rs @@ -91,9 +91,145 @@ pub(crate) struct MyceliumTransport { local_id: String, } +/// This transport's options, checked when the configuration is loaded +/// (`config::Settings::check`): a daemon elsewhere (`mycelium-rest-url`) by a URL, and +/// none of the options only a supervised one takes with it; a private network with +/// both its key and its name. +pub(crate) fn check(settings: &crate::config::Settings) -> Result<()> { + if let Some(url) = &settings.mycelium_rest_url { + Url::parse(url).with_context(|| format!("mycelium-rest-url {url:?}"))?; + if let Some((name, _)) = supervised_only(settings).iter().find(|(_, set)| *set) { + anyhow::bail!( + "{name} is for a supervised Mycelium daemon and can't be combined with mycelium-rest-url" + ); + } + } + if settings.mycelium_key_create && settings.mycelium_key_file.is_none() { + anyhow::bail!("mycelium-key-create creates the key at mycelium-key-file; set that too"); + } + // A private network has both; one alone would be ignored or invented. + if settings.mycelium_network_key_file.is_some() != settings.mycelium_network_name.is_some() { + anyhow::bail!("mycelium-network-key-file and mycelium-network-name go together"); + } + Ok(()) +} + +/// The options only a supervised Mycelium daemon takes, and whether each is set: the +/// other side of `config::CONFLICTS`' `mycelium-rest-url` (a test keeps them the same). +pub(crate) fn supervised_only(settings: &crate::config::Settings) -> [(&'static str, bool); 5] { + [ + ("mycelium-key-file", settings.mycelium_key_file.is_some()), + ("mycelium-key-create", settings.mycelium_key_create), + ( + "mycelium-network-key-file", + settings.mycelium_network_key_file.is_some(), + ), + ( + "mycelium-network-name", + settings.mycelium_network_name.is_some(), + ), + ( + "mycelium-tcp-listen-port", + settings.mycelium_tcp_listen_port.is_some(), + ), + ] +} + +/// The transport as `settings` configure it (`transport::start`): its daemon supervised +/// (started, which can take a while) or one running elsewhere. None if `stop` came +/// first. +pub(crate) async fn start( + settings: &crate::config::Settings, + mut stop: tokio::sync::watch::Receiver<bool>, +) -> Result<Option<MyceliumTransport>> { + // The daemon's key, 32 raw bytes (as `daemon` hands it an ephemeral one): read, + // or made, here, so a missing file fails before the daemon starts. + if let Some(path) = &settings.mycelium_key_file { + let (bytes, created) = crate::files::key_file( + path, + "Mycelium key", + settings.mycelium_key_create, + "mycelium-key-create", + "a new Mycelium address", + 32, + )?; + if bytes.len() != 32 { + anyhow::bail!( + "The Mycelium key in {} has {} bytes; it has 32", + path.display(), + bytes.len() + ); + } + if created { + tracing::info!(path = %path.display(), "Created a new Mycelium key"); + } + } + let spawn_config = MyceliumSpawnConfig { + rest_url_opt: settings + .mycelium_rest_url + .as_deref() + .and_then(|url| Url::parse(url).ok()), + key_file_opt: settings.mycelium_key_file.clone(), + network_key_file: settings.mycelium_network_key_file.clone(), + network_name: settings.mycelium_network_name.clone(), + tcp_port: settings.mycelium_tcp_listen_port, + peers: settings.mycelium_peers.clone(), + }; + let setup = tokio::select! { + setup = MyceliumSetup::bootstrap(spawn_config) => setup?, + _ = stop.wait_for(|stop| *stop) => return Ok(None), + }; + // Without a password it may carry (the option is secret). + let mut shown_url = setup.rest_url.clone(); + let _ = shown_url.set_password(None); + tracing::debug!(url = shown_url.to_string(), "Using Mycelium REST API"); + tokio::select! { + transport = MyceliumTransport::connect(setup) => Ok(Some(transport?)), + _ = stop.wait_for(|stop| *stop) => Ok(None), + } +} + +/// How long the daemon's API may take to come up: a daemon just started listens a +/// moment later (`admin_info`). +const API_READY_WITHIN: Duration = Duration::from_secs(30); + +/// The daemon's identity, from its API at `url` (`/api/v1/admin`), once the API answers: +/// a daemon we just started doesn't listen yet, so refused connections are tried again +/// (50 ms, doubling up to a second) for `API_READY_WITHIN`; anything else is an error. +async fn admin_info(client: &reqwest::Client, url: Url) -> Result<MyceliumAdminInfo> { + let deadline = tokio::time::Instant::now() + API_READY_WITHIN; + let mut wait = Duration::from_millis(50); + loop { + match client.get(url.clone()).send().await { + Ok(response) => { + return response + .error_for_status()? + .json() + .await + .context("Querying local Mycelium identity"); + } + Err(error) if error.is_connect() && tokio::time::Instant::now() + wait < deadline => { + tracing::debug!(url = %url, error = %error, "Mycelium's API doesn't answer yet; trying again"); + tokio::time::sleep(wait).await; + wait = (wait * 2).min(Duration::from_secs(1)); + } + Err(error) => { + return Err(error).with_context(|| { + format!( + "Querying local Mycelium identity (no answer within {API_READY_WITHIN:?})" + ) + }); + } + } + } +} + impl MyceliumTransport { pub(crate) async fn connect(setup: MyceliumSetup) -> Result<Self> { let rest_url = setup.rest_url.clone(); + // reqwest's TLS has no crypto provider of its own (`Cargo.toml`): ring, for the + // whole process, unless one is installed already. + let _ = rustls::crypto::ring::default_provider().install_default(); let client = reqwest::Client::builder() .timeout(REQUEST_TIMEOUT) @@ -102,14 +238,7 @@ impl MyceliumTransport { .timeout(Duration::from_secs(MYCELIUM_POLL_TIMEOUT_SECS + 30)) .build()?; - let admin: MyceliumAdminInfo = client - .get(rest_url.join("/api/v1/admin")?) - .send() - .await? - .error_for_status()? - .json() - .await - .context("Querying local Mycelium identity")?; + let admin = admin_info(&client, rest_url.join("/api/v1/admin")?).await?; let transport = Self { setup: tokio::sync::Mutex::new(setup), @@ -246,31 +375,17 @@ impl Transport for MyceliumTransport { .into()); } - // Mycelium's message ID, for GET /api/v1/messages/status/{id} when chasing - // a message that never arrived. - if tracing::enabled!(tracing::Level::DEBUG) { - let Ok(body) = response + // Mycelium's message ID, for GET /api/v1/messages/status/{id} when chasing a + // message that never arrived. The message itself is logged above, in the paths + // layer (`transport::paths`). + if tracing::enabled!(tracing::Level::TRACE) { + let id = response .bytes() .await - .map_err(anyhow::Error::from) - .and_then(|bytes| Ok(serde_json::from_slice::<serde_json::Value>(&bytes)?)) - else { - tracing::debug!(to = %Addressed(SCHEME, &message.peer), "Sent message (couldn't decode response from Mycelium REST API)"); - return Ok(()); - }; - let Ok(outbound) = serde_json::from_slice::<OutboundMessage>(&message.body) else { - tracing::debug!(to = %Addressed(SCHEME, &message.peer), body_len = %message.body.len(), "Sent message (couldn't parse outbound message)"); - return Ok(()); - }; - let Ok(parsed) = decode_payload(&outbound.topic, &outbound.payload) else { - tracing::debug!(to = %Addressed(SCHEME, &message.peer), message = ?outbound, "Sent message (couldn't decode outbound message)"); - return Ok(()); - }; - let Some(id) = body.get("id").and_then(|id| id.as_str()) else { - tracing::debug!(to = %Addressed(SCHEME, &message.peer), message = ?parsed, "Sent message (couldn't read id from Mycelium REST API response)"); - return Ok(()); - }; - tracing::debug!(to = %Addressed(SCHEME, &message.peer), message = ?parsed, id = %id, "Sent message"); + .ok() + .and_then(|bytes| serde_json::from_slice::<serde_json::Value>(&bytes).ok()) + .and_then(|body| body.get("id")?.as_str().map(str::to_owned)); + tracing::trace!(to = %Addressed(SCHEME, &message.peer), id = ?id, "Mycelium took the message"); } Ok(()) } @@ -300,12 +415,11 @@ impl Transport for MyceliumTransport { while let Some(inbound_message) = rx.recv().await { match parse_message(&inbound_message) { Ok((sender, parsed)) => { - tracing::debug!( + // The message itself is logged above, in the paths layer. + tracing::trace!( from = %Addressed(SCHEME, &inbound_message.src_pk), - message = ?parsed, id = %inbound_message.id, - //payload_bytes = raw_bytes.len(), - "Received message" + "Mycelium delivered a message" ); let payload_len = base64_decoded_len(&inbound_message.payload); if is_probe(&parsed) { @@ -464,6 +578,13 @@ pub(crate) fn encode_message(peer: &str, message: &ParsedMessage) -> Result<Enco }) } +/// Mycelium's envelope, for simulations that run on it (`transport::sim::Codec`). +#[cfg(test)] +pub(crate) const CODEC: crate::transport::sim::Codec = crate::transport::sim::Codec { + encode: encode_message, + decode: decode_message, +}; + /// The message in a body made by [`encode_message`]. #[cfg(test)] pub(crate) fn decode_message(body: &[u8]) -> Result<ParsedMessage> { @@ -562,10 +683,10 @@ pub(crate) enum TransportError { #[cfg(test)] mod tests { use super::*; + use crate::document::ChangeHash; use crate::transport::{ BlobBody, BlobId, BlobMessage, Epoch, HeadsSummary, MAX_LISTED_HEADS, Status, }; - use automerge::ChangeHash; /// Mycelium splits messages into chunks of this size. const MYCELIUM_CHUNK: usize = 1300; @@ -595,6 +716,7 @@ mod tests { epoch_seq: u32::MAX, ready: true, syncs_with_you: true, + leaving: false, }; let payload = wire::encode(&ParsedMessage::StatusReply(status.clone())).unwrap(); // Mycelium's message data: a length byte, the topic, then our payload. @@ -657,20 +779,51 @@ mod tests { )); } + /// Every kind decodes from Mycelium's API body (its envelope: the topic, the payload + /// in base64) to what was encoded. #[test] - fn messages_round_trip_through_the_api_body() { - let status = ParsedMessage::Status(Status { - heads: HeadsSummary::of(vec![ChangeHash([1; 32])]), - epoch: Epoch::GENESIS, - epoch_seq: 0, - ready: true, - syncs_with_you: true, - }); - let encoded = encode_message("peer", &status).unwrap(); - assert!(matches!( - decode_message(&encoded.body).unwrap(), - ParsedMessage::Status(_) - )); + fn every_kind_round_trips_through_the_api_body() { + for message in wire::samples() { + let encoded = encode_message("peer", &message).unwrap(); + assert_eq!(encoded.kind, message.kind()); + let decoded = decode_message(&encoded.body).unwrap(); + assert_eq!( + wire::encode(&decoded).unwrap(), + wire::encode(&message).unwrap(), + "{message:?}" + ); + assert_eq!(encoded.payload_len, wire::encode(&message).unwrap().len()); + } + } + + /// A swarm whose every message goes through Mycelium's envelope: joining, writes + /// reaching everyone, and concurrent writes merging, as the simulations do with the + /// wire encoding alone. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_swarm_converges_over_the_envelope() { + use crate::simtest::{Sim, read, write}; + use crate::swarm::bootstrap::Start; + use serde_json::json; + const SOON: Duration = Duration::from_secs(10); + + let sim = Sim::with_codec(11, CODEC); + let a = sim.start("a", &[], Start::Genesis); + let b = sim.start("b", &["a"], Start::Join); + let c = sim.start("c", &["a"], Start::Join); + let (a, b, c) = (&a, &b, &c); + sim.wait_ready(&[a, b, c], SOON).await; + write(b, "users/bob", json!({"name": "Bob"})).await; + sim.wait_until("c has b's write", SOON, move || async move { + read(c, "users/bob/name").await == Some(json!("Bob")) + }) + .await; + write(a, "users/alice", json!(1)).await; + write(c, "users/carol", json!(3)).await; + sim.wait_converged(&[a, b, c], SOON).await; + for node in [a, b, c] { + assert_eq!(read(node, "users/alice").await, Some(json!(1))); + assert_eq!(read(node, "users/carol").await, Some(json!(3))); + } } #[test] diff --git a/src/transport/paths.rs b/src/transport/paths.rs index 3baa092..164574d 100644 --- a/src/transport/paths.rs +++ b/src/transport/paths.rs @@ -8,7 +8,7 @@ //! transport of its address, hands every transport's messages to one handler under the //! sender's address, and puts together what the transports say about themselves //! (addresses, seeds they found, figures). The transports are in order of preference, -//! as configured (`--transports`). +//! as configured (`--transport-order`). //! //! A message to a node with several addresses goes to the first of them that takes it //! (`send_first`), in the order `rank` gives. A send that fails is tried at once at the @@ -16,7 +16,8 @@ //! next carries on within the same send. //! //! Paths measures the addresses of such nodes itself (and every path the node's -//! keepalive PINGs go over, below): every `PROBE_EVERY`, each address it had a choice of +//! keepalive PINGs go over, below): every `PROBE_EVERY`, or every keepalive interval if +//! that is longer, each address it had a choice of //! within `ACTIVE_FOR` gets a PING with a random nonce, which the //! other side's paths answers at once with a PONG, over the same path. Answers give each //! path's round-trip time (smoothed); `UNANSWERED_DOWN` PINGs in a row unanswered, or a @@ -30,6 +31,11 @@ //! alive. The node's keepalive is a PING too (`swarm::peer::keepalive`), which paths //! sends like any message, and notes as the probe of the path it went over. //! +//! Every message on every transport passes here, so this is where messages are logged +//! (debug): `Sent message` once a transport took one, `Received message` as one +//! arrives, each with its address and what it says. Failed sends are logged by those who +//! sent them, with what they do about it. +//! //! The debug firewall (`transport::firewall`) works here, below everything else, through //! a `Gate`: messages from an address it blocks are dropped as they arrive, and sends to //! one are dropped silently, as by a lossy network. @@ -42,6 +48,7 @@ use anyhow::{Context, Result, anyhow}; use tokio::time::Instant; use tracing::{debug, info}; +use crate::clock::{Clock, UnixMillis}; use crate::locks::MutexExt; use crate::transport::stats::{PathCounts, PathInfo, PathsStats, TransportStats}; use crate::transport::traffic::Traffic; @@ -84,15 +91,16 @@ pub(crate) fn scheme_of(address: &str) -> &str { /// A path whose send failed is down for this long (unless it answers a PING, or a /// send there works again). const DOWN_FOR: Duration = Duration::from_secs(60); -/// How often a path in use is pinged, how long an answer may take, and how many PINGs -/// in a row unanswered make it down. +/// How often a path in use is pinged at most (every keepalive interval, if that is +/// longer: `Table::probe_every`), how long an answer may take, and how many PINGs in a +/// row unanswered make it down. const PROBE_EVERY: Duration = Duration::from_secs(30); const PROBE_TIMEOUT: Duration = Duration::from_secs(15); const UNANSWERED_DOWN: u32 = 3; /// How often the probe task looks for paths due. const PROBE_TICK: Duration = Duration::from_secs(5); -/// Paths this node had a choice of within this long are probed; ones it didn't use for -/// twice as long are forgotten. +/// Paths this node had a choice of, or sent over, within this long are probed; ones it +/// didn't use for twice as long are forgotten. const ACTIVE_FOR: Duration = Duration::from_secs(300); /// The path in use is changed for one this much faster (a fraction of its round-trip /// time), and at least `SWITCH_MIN`. @@ -100,14 +108,48 @@ const SWITCH_GAIN: f64 = 0.25; const SWITCH_MIN: Duration = Duration::from_millis(5); /// Paths kept at most: they are of peers this node chose among, but bounded anyway. const MAX_PATHS: usize = 4096; +/// A path's clock sample with the narrowest bound is kept this long at most; then the +/// next one takes its place. (A clock set anew shows at once: `Table::clock`.) +const CLOCK_KEPT: Duration = Duration::from_secs(300); + +/// How far a peer's clock is ahead of ours (behind if negative), from a PING and its +/// PONG: `offset_ms`, give or take `within_ms`. The PONG says its sender's time when it +/// answered, which was after our PING went and before the PONG came: within half the +/// round trip of its middle, however differently long the two ways took. +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ClockOffset { + pub(crate) offset_ms: i64, + pub(crate) within_ms: u64, +} + +impl ClockOffset { + /// From a PONG stamped `at` by its sender, which came at `heard` by our clock, `rtt` + /// after its PING went. + pub(crate) fn of(at: UnixMillis, heard: UnixMillis, rtt: Duration) -> Self { + let half_us = rtt.as_micros() / 2; + let middle_us = i128::from(heard.as_u64()) * 1000 - half_us as i128; + let offset_us = i128::from(at.as_u64()) * 1000 - middle_us; + Self { + offset_ms: i64::try_from(offset_us / 1000).unwrap_or(i64::MAX), + // And a millisecond for the stamps' resolution. + within_ms: u64::try_from(half_us.div_ceil(1000)).unwrap_or(u64::MAX) + 1, + } + } +} /// What paths knows about one path (an address as paths keep it). #[derive(Debug, Default)] struct PathState { /// When it was last among several to choose from (`rank`). used: Option<Instant>, + /// When a message last went over it: a peer's only path is measured too, if its + /// keepalives don't (a busy pair has none: `PeerState::take_keepalive_due`). + carried: Option<Instant>, /// Smoothed round-trip time of its PINGs. rtt: Option<Duration>, + /// Where the transport went when `rtt` was measured (`Paths::location`). + measured_at: Option<String>, /// The PING out, by nonce, and when it went. probe: Option<(u64, Instant)>, last_probe: Option<Instant>, @@ -115,6 +157,9 @@ struct PathState { unanswered: u32, /// When a send there last failed. failed_at: Option<Instant>, + /// The peer's clock as its PONGs on this path say (the narrowest bound lately), and + /// when that sample came. + clock: Option<(ClockOffset, Instant)>, } impl PathState { @@ -127,29 +172,58 @@ impl PathState { } /// Every path's state, and which one is in use for each set to choose from. -#[derive(Debug, Default)] +#[derive(Debug)] struct Table { paths: HashMap<String, PathState>, /// By the set's addresses (sorted, joined): the one in use. chosen: HashMap<String, String>, counts: PathCounts, + /// How often a path in use is pinged: `PROBE_EVERY`, or the node's keepalive + /// interval if that is longer, so that the probes don't add to the keepalives a + /// long interval is chosen to spare. A busy pair, whose keepalives don't go, still + /// gets its paths measured this often. + probe_every: Duration, +} + +impl Default for Table { + fn default() -> Self { + Self::new(PROBE_EVERY) + } } impl Table { + fn new(keepalive: Duration) -> Self { + Self { + paths: HashMap::new(), + chosen: HashMap::new(), + counts: PathCounts::default(), + probe_every: PROBE_EVERY.max(keepalive), + } + } + /// `usable`, with each one's place in the configured order, best first (as - /// `Paths::rank` says), noting the choice. - fn rank(&mut self, mut usable: Vec<(String, usize)>, now: Instant) -> Vec<String> { + /// `Paths::rank` says). With `sending`, for a message about to go: they count as + /// used (`ACTIVE_FOR`), and the one in use may change for a faster one. Without, a + /// look at the choice as it stands, changing nothing (`/peers`, figures, LINKS). + fn rank( + &mut self, + mut usable: Vec<(String, usize)>, + now: Instant, + sending: bool, + ) -> Vec<String> { usable.sort_by_key(|(_, preference)| *preference); usable.dedup_by(|a, b| a.0 == b.0); if usable.len() < 2 { return usable.into_iter().map(|(address, _)| address).collect(); } - if self.paths.len() >= MAX_PATHS { - self.forget_unused(now); - } - for (address, _) in &usable { - if self.paths.len() < MAX_PATHS || self.paths.contains_key(address) { - self.paths.entry(address.clone()).or_default().used = Some(now); + if sending { + if self.paths.len() >= MAX_PATHS { + self.forget_unused(now); + } + for (address, _) in &usable { + if self.paths.len() < MAX_PATHS || self.paths.contains_key(address) { + self.paths.entry(address.clone()).or_default().used = Some(now); + } } } let rtt = |table: &Self, address: &str| table.paths.get(address).and_then(|path| path.rtt); @@ -180,6 +254,7 @@ impl Table { .min_by_key(|(_, rtt)| *rtt); if let (Some((fastest, fastest_rtt)), Some(current_rtt)) = (fastest, rtt(self, ¤t)) + && sending && *fastest != current && current_rtt.saturating_sub(fastest_rtt) >= SWITCH_MIN.max(current_rtt.mul_f64(SWITCH_GAIN)) @@ -194,7 +269,7 @@ impl Table { self.counts.switches += 1; current = fastest.clone(); } - if self.chosen.len() < MAX_PATHS || self.chosen.contains_key(&key) { + if sending && (self.chosen.len() < MAX_PATHS || self.chosen.contains_key(&key)) { self.chosen.insert(key, current.clone()); } // The rest: measured ones by round-trip time, then unmeasured ones in the @@ -228,15 +303,22 @@ impl Table { } /// A send to `address` worked. - fn sent(&mut self, address: &str) { - if let Some(path) = self.paths.get_mut(address) { - path.failed_at = None; + fn sent(&mut self, address: &str, now: Instant) { + if self.paths.len() >= MAX_PATHS && !self.paths.contains_key(address) { + self.forget_unused(now); + if self.paths.len() >= MAX_PATHS { + return; + } } + let path = self.paths.entry(address.to_owned()).or_default(); + path.failed_at = None; + path.carried = Some(now); } - /// The PINGs due now, with their nonces (from `nonce`): to every path used within - /// `ACTIVE_FOR` that wasn't pinged within `PROBE_EVERY` and has no PING out. A PING - /// out for `PROBE_TIMEOUT` counts as unanswered. + /// The PINGs due now, with their nonces (from `nonce`): to every path chosen among + /// others or sent over within `ACTIVE_FOR` that wasn't pinged (by us, or by a + /// keepalive) within `probe_every` and has no PING out. A PING out for + /// `PROBE_TIMEOUT` counts as unanswered. fn due(&mut self, now: Instant, mut nonce: impl FnMut() -> u64) -> Vec<(String, u64)> { self.forget_unused(now); let mut due = Vec::new(); @@ -251,12 +333,13 @@ impl Table { info!(address = %address, "A path stopped answering PINGs; using others"); } } - let active = path - .used - .is_some_and(|used| now.saturating_duration_since(used) < ACTIVE_FOR); + let lately = |at: Option<Instant>| { + at.is_some_and(|at| now.saturating_duration_since(at) < ACTIVE_FOR) + }; + let active = lately(path.used) || lately(path.carried); let rested = path .last_probe - .is_none_or(|last| now.saturating_duration_since(last) >= PROBE_EVERY); + .is_none_or(|last| now.saturating_duration_since(last) >= self.probe_every); if active && rested && path.probe.is_none() { let nonce = nonce(); path.probe = Some((nonce, now)); @@ -269,8 +352,8 @@ impl Table { } /// The node sent a PING with `nonce` to `address` (its keepalive): the probe of that - /// path, which its PONG measures, even if it is a peer's only path (then it isn't - /// pinged otherwise: `due` pings only paths there was a choice of). + /// path, which its PONG measures, even if it is a peer's only path; `due` then + /// doesn't ping it again within `probe_every`. fn pinged(&mut self, address: &str, nonce: u64, now: Instant) { if self.paths.len() >= MAX_PATHS && !self.paths.contains_key(address) { self.forget_unused(now); @@ -295,7 +378,23 @@ impl Table { } } - /// A PONG with `nonce` from `address`: the round trip, if it answers our PING out. + /// The transport of `address` now goes to `location` (a libp2p peer over another + /// connection: through a relay before, directly now): round trips measured before + /// were of another way, and would weigh on the next ones for minutes (`pong` + /// smooths), so they are dropped. + fn located(&mut self, address: &str, location: &str) { + let Some(path) = self.paths.get_mut(address) else { + return; + }; + if path.measured_at.as_deref() != Some(location) { + if path.measured_at.is_some() { + path.rtt = None; + } + path.measured_at = Some(location.to_owned()); + } + } + + /// A PONG with `nonce` from `address`: its round trip, if it answers our PING out. fn pong(&mut self, address: &str, nonce: u64, now: Instant) -> Option<Duration> { let path = self.paths.get_mut(address)?; let (expected, sent) = path.probe?; @@ -314,7 +413,26 @@ impl Table { None => sample, }); self.counts.probes_answered += 1; - path.rtt + Some(sample) + } + + /// A clock sample from a PONG over `address`: kept if its bound is as narrow as the + /// one kept, or that one is older than `CLOCK_KEPT`, or the two can't both be right + /// (their bounds don't overlap: a clock was set anew). The narrowest bound is the + /// round trip least delayed by queues and relays. + fn clock(&mut self, address: &str, offset: ClockOffset, now: Instant) { + let Some(path) = self.paths.get_mut(address) else { + return; + }; + let replaces = path.clock.is_none_or(|(kept, at)| { + offset.within_ms <= kept.within_ms + || now.saturating_duration_since(at) >= CLOCK_KEPT + || offset.offset_ms.abs_diff(kept.offset_ms) + > offset.within_ms.saturating_add(kept.within_ms) + }); + if replaces { + path.clock = Some((offset, now)); + } } /// Forgets the paths neither used nor pinged for twice `ACTIVE_FOR`, and choices @@ -324,7 +442,7 @@ impl Table { at.is_some_and(|at| now.saturating_duration_since(at) < ACTIVE_FOR * 2) }; self.paths - .retain(|_, path| recent(path.used) || recent(path.last_probe)); + .retain(|_, path| recent(path.used) || recent(path.carried) || recent(path.last_probe)); let paths = &self.paths; self.chosen .retain(|key, _| key.split(' ').all(|address| paths.contains_key(address))); @@ -338,6 +456,7 @@ impl Table { .and_then(|path| path.rtt) .map(|rtt| rtt.as_secs_f64() * 1000.0), down: path.is_some_and(|path| path.is_down(now)), + connection: None, } } } @@ -349,17 +468,22 @@ pub(crate) struct Paths { traffic: Option<Arc<Traffic>>, /// The debug firewall, if any. gate: Option<Weak<dyn Gate>>, + /// The node's wall clock, which its PONGs say the time by. + clock: Arc<Clock>, /// A std mutex: never held across an await. table: Mutex<Table>, } impl Paths { /// Over `transports`, by scheme, in order of preference; its own PINGs and PONGs - /// counted in `traffic`, the firewall through `gate`. + /// counted in `traffic`, the firewall through `gate`, PONGs stamped by `clock`; + /// paths pinged every `keepalive` at least (`Table::probe_every`). pub(crate) fn new( transports: Vec<(&'static str, Arc<dyn Transport>)>, traffic: Option<Arc<Traffic>>, gate: Option<Weak<dyn Gate>>, + clock: Arc<Clock>, + keepalive: Duration, ) -> Self { Self { links: transports @@ -368,7 +492,8 @@ impl Paths { .collect(), traffic, gate, - table: Mutex::default(), + clock, + table: Mutex::new(Table::new(keepalive)), } } @@ -421,14 +546,18 @@ impl Paths { /// The usable ones of `addresses` (as paths keep them), best first: paths that are /// up before those down, the one in use first, then by round-trip time, then in - /// the configured order (module docs). Several addresses are paths to measure. - pub(crate) fn rank(&self, addresses: &[String]) -> Vec<String> { + /// the configured order (module docs). For a send (`sending`): several addresses + /// are paths to measure, and the one in use may change; else only a look + /// (`Table::rank`). + pub(crate) fn rank(&self, addresses: &[String], sending: bool) -> Vec<String> { let usable: Vec<(String, usize)> = addresses .iter() .filter(|address| self.usable(address)) .map(|address| (address.clone(), self.preference(address))) .collect(); - self.table.lock_or_recover().rank(usable, Instant::now()) + self.table + .lock_or_recover() + .rank(usable, Instant::now(), sending) } /// The smoothed round-trip time of the path to `address`, if measured. @@ -436,6 +565,12 @@ impl Paths { self.table.lock_or_recover().paths.get(address)?.rtt } + /// The peer's clock at `address`, as its PONGs there say (`ClockOffset`). + pub(crate) fn clock_offset(&self, address: &str) -> Option<ClockOffset> { + let table = self.table.lock_or_recover(); + table.paths.get(address)?.clock.map(|(offset, _)| offset) + } + /// What paths knows about each of `addresses`. pub(crate) fn info(&self, addresses: &[String]) -> Vec<PathInfo> { let now = Instant::now(); @@ -449,9 +584,11 @@ impl Paths { self.located(infos) } - /// `infos` with where each path goes (`located`); asked outside the table's lock. + /// `infos` with where each path goes (`located`) and the transport's connection + /// there; asked outside the table's lock. fn located(&self, mut infos: Vec<PathInfo>) -> Vec<PathInfo> { for info in &mut infos { + info.connection = self.connection(&info.address); info.address = self.location(&info.address); } infos @@ -507,17 +644,40 @@ impl Paths { .unwrap_or_default() } - /// Each transport's (`Transport::local_addresses`), with its scheme. - pub(crate) fn local_addresses(&self) -> Vec<String> { + /// The longest of the transports' (`Transport::sync_reply_timeout`): an answer may + /// come over any of them. + pub(crate) fn sync_reply_timeout(&self) -> Duration { self.links .iter() - .flat_map(|link| { + .map(|link| link.inner.sync_reply_timeout()) + .max() + .unwrap_or(crate::state::SYNC_REPLY_TIMEOUT) + } + + /// Each transport's (`Transport::local_addresses`), with its scheme: taken in turns, + /// each transport's first in the configured order, then each one's second, and so on, + /// so a list cut short (`members::MAX_ADDRESSES`) still has every transport's best. + pub(crate) fn local_addresses(&self) -> Vec<String> { + let mut each: Vec<std::vec::IntoIter<String>> = self + .links + .iter() + .map(|link| { link.inner .local_addresses() .into_iter() .map(|target| link.address(&target)) + .collect::<Vec<_>>() + .into_iter() }) - .collect() + .collect(); + let mut addresses = Vec::new(); + loop { + let before = addresses.len(); + addresses.extend(each.iter_mut().filter_map(Iterator::next)); + if addresses.len() == before { + return addresses; + } + } } /// The schemes of the transports, in order of preference. @@ -541,6 +701,20 @@ impl Paths { .collect() } + /// Each transport's (`Transport::on_seeds_found`). + pub(crate) fn on_seeds_found(&self, found: std::sync::Arc<tokio::sync::Notify>) { + for link in &self.links { + link.inner.on_seeds_found(found.clone()); + } + } + + /// Each transport's (`Transport::on_reachable`). + pub(crate) fn on_reachable(&self, reachable: std::sync::Arc<tokio::sync::Notify>) { + for link in &self.links { + link.inner.on_reachable(reachable.clone()); + } + } + /// The address of the node a seed leads to, as paths keep it: an address of ours /// (`<scheme>:...`) whose transport reads it at once; with a scheme of ours, what /// that transport resolves the rest to (an address the transport can't read as an @@ -623,7 +797,7 @@ impl Paths { match self.send(&encoded).await { Ok(()) => { let mut table = self.table.lock_or_recover(); - table.sent(address); + table.sent(address, Instant::now()); if !failures.is_empty() { table.counts.failovers += 1; drop(table); @@ -665,7 +839,16 @@ impl Paths { let mut inner = message.clone(); inner.peer = target.clone(); inner.address = target; - link.inner.send_encoded(&inner).await + link.inner.send_encoded(&inner).await?; + match &message.message { + Some(parsed) => { + debug!(to = %message.address, bytes = message.payload_len, message = ?parsed, "Sent message"); + } + None => { + debug!(to = %message.address, bytes = message.payload_len, kind = ?message.kind, "Sent message"); + } + } + Ok(()) } /// How the transport of `address` is connected there (`Transport::connection`). @@ -674,6 +857,12 @@ impl Paths { link.inner.connection(&target) } + /// How hole punching to `address` went (`Transport::punching`). + pub(crate) fn punching(&self, address: &str) -> Option<crate::transport::Punching> { + let (link, target) = self.split(address)?; + link.inner.punching(&target) + } + /// Paths' own figures, and every transport's. pub(crate) fn stats(&self) -> TransportStats { let mut stats = TransportStats::default(); @@ -696,11 +885,14 @@ impl Paths { stats } - /// Sends a PING (or with `pong`, a PONG) with `nonce` to `address`, counted - /// (`traffic`). + /// Sends a PING (or with `pong`, a PONG) with `nonce` to `address`, counted and + /// logged (`traffic`). async fn send_probe(&self, address: &str, pong: bool, nonce: u64) -> Result<()> { let message = if pong { - ParsedMessage::Pong(nonce) + ParsedMessage::Pong { + nonce, + at: self.clock.now_millis(), + } } else { ParsedMessage::Ping(nonce) }; @@ -711,6 +903,12 @@ impl Paths { Ok(()) => traffic.sent(address, encoded.kind, encoded.payload_len), Err(_) => traffic.refused(address, encoded.kind, encoded.payload_len), } + // Logged too, under the address: the PINGs and PONGs that come back go up to + // the node and are logged there, and an exchange is seen whole. + let way = if sent.is_ok() { "sent" } else { "refused" }; + traffic + .log + .note(way, address, &message, encoded.payload_len); } sent } @@ -762,6 +960,7 @@ impl Paths { tracing::trace!(from = %address, "Firewall dropped inbound message"); return Ok(()); } + debug!(from = %address, bytes = payload_len, message = ?message, "Received message"); match message { ParsedMessage::Ping(nonce) => { let paths = Arc::clone(&self); @@ -772,10 +971,15 @@ impl Paths { } }); } - ParsedMessage::Pong(nonce) => { - self.table - .lock_or_recover() - .pong(&address, nonce, Instant::now()); + ParsedMessage::Pong { nonce, at } => { + let heard = self.clock.now_millis(); + let location = self.location(&address); + let now = Instant::now(); + let mut table = self.table.lock_or_recover(); + table.located(&address, &location); + if let Some(rtt) = table.pong(&address, nonce, now) { + table.clock(&address, ClockOffset::of(at, heard, rtt), now); + } } _ => {} } @@ -891,6 +1095,8 @@ mod tests { ], None, None, + Arc::default(), + Duration::ZERO, )) }; (on_both("a"), on_both("b")) @@ -952,27 +1158,39 @@ mod tests { let mut table = Table::default(); let mut now = Instant::now(); let set = paths(&["libp2p:x", "mycelium:x"]); - assert_eq!(table.rank(set.clone(), now), ["libp2p:x", "mycelium:x"]); + assert_eq!( + table.rank(set.clone(), now, true), + ["libp2p:x", "mycelium:x"] + ); // libp2p a bit slower: not enough to switch. for _ in 0..8 { now = answer(&mut table, now, &[("libp2p:x", 110), ("mycelium:x", 100)]); } - assert_eq!(table.rank(set.clone(), now), ["libp2p:x", "mycelium:x"]); + assert_eq!( + table.rank(set.clone(), now, true), + ["libp2p:x", "mycelium:x"] + ); assert_eq!(table.counts.switches, 0); // Much slower: switched, and it stays switched while that holds. for _ in 0..8 { now = answer(&mut table, now, &[("libp2p:x", 300), ("mycelium:x", 100)]); } - assert_eq!(table.rank(set.clone(), now), ["mycelium:x", "libp2p:x"]); - assert_eq!(table.rank(set.clone(), now), ["mycelium:x", "libp2p:x"]); + assert_eq!( + table.rank(set.clone(), now, true), + ["mycelium:x", "libp2p:x"] + ); + assert_eq!( + table.rank(set.clone(), now, true), + ["mycelium:x", "libp2p:x"] + ); assert_eq!(table.counts.switches, 1); // Small differences under SWITCH_MIN never switch. let lan = paths(&["libp2p:y", "mycelium:y"]); - table.rank(lan.clone(), now); + table.rank(lan.clone(), now, true); for _ in 0..8 { now = answer(&mut table, now, &[("libp2p:y", 4), ("mycelium:y", 1)]); } - assert_eq!(table.rank(lan, now), ["libp2p:y", "mycelium:y"]); + assert_eq!(table.rank(lan, now, true), ["libp2p:y", "mycelium:y"]); } #[test] @@ -980,14 +1198,17 @@ mod tests { let mut table = Table::default(); let mut now = Instant::now(); let set = paths(&["libp2p:x", "mycelium:x"]); - table.rank(set.clone(), now); + table.rank(set.clone(), now, true); // libp2p stops answering: down after UNANSWERED_DOWN PINGs, then last. for _ in 0..=UNANSWERED_DOWN { now = answer(&mut table, now, &[("mycelium:x", 100)]); - table.rank(set.clone(), now); + table.rank(set.clone(), now, true); } assert!(table.info("libp2p:x", now).down); - assert_eq!(table.rank(set.clone(), now), ["mycelium:x", "libp2p:x"]); + assert_eq!( + table.rank(set.clone(), now, true), + ["mycelium:x", "libp2p:x"] + ); // It answers again: up, and measured. now = answer(&mut table, now, &[("libp2p:x", 50), ("mycelium:x", 100)]); assert!(!table.info("libp2p:x", now).down); @@ -996,10 +1217,36 @@ mod tests { table.failed("mycelium:x", now); assert!(table.info("mycelium:x", now).down); assert!(!table.info("mycelium:x", now + DOWN_FOR).down); - table.sent("mycelium:x"); + table.sent("mycelium:x", now); assert!(!table.info("mycelium:x", now).down); } + /// A look at the choice (`/peers`, figures) changes nothing: no path counts as + /// used, and no faster one is switched to. + #[test] + fn a_look_changes_nothing() { + let mut table = Table::default(); + let mut now = Instant::now(); + let set = paths(&["libp2p:x", "mycelium:x"]); + assert_eq!( + table.rank(set.clone(), now, false), + ["libp2p:x", "mycelium:x"] + ); + assert!(table.paths.is_empty() && table.chosen.is_empty()); + table.rank(set.clone(), now, true); + for _ in 0..4 { + now = answer(&mut table, now, &[("libp2p:x", 500), ("mycelium:x", 10)]); + } + // Much faster, but only looked at: still the one in use. + assert_eq!( + table.rank(set.clone(), now, false), + ["libp2p:x", "mycelium:x"] + ); + assert_eq!(table.counts.switches, 0); + assert_eq!(table.rank(set, now, true), ["mycelium:x", "libp2p:x"]); + assert_eq!(table.counts.switches, 1); + } + #[test] fn only_paths_in_use_are_pinged() { let mut table = Table::default(); @@ -1010,9 +1257,9 @@ mod tests { nonce }; // One address: nothing to choose, nothing to measure. - table.rank(paths(&["libp2p:z"]), now); + table.rank(paths(&["libp2p:z"]), now, true); assert!(table.due(now, &mut next).is_empty()); - table.rank(paths(&["libp2p:x", "mycelium:x"]), now); + table.rank(paths(&["libp2p:x", "mycelium:x"]), now, true); assert_eq!(table.due(now, &mut next).len(), 2); // Not again within PROBE_EVERY; a stale answer is ignored. assert!(table.due(now + PROBE_TIMEOUT, &mut next).is_empty()); @@ -1023,12 +1270,65 @@ mod tests { assert!(table.paths.is_empty() && table.chosen.is_empty()); } + /// A PONG's time against the middle of its round trip: the peer's clock, give or + /// take half the round trip; the narrowest bound kept, until it is old. + #[test] + fn clocks_compare_by_the_middle_of_the_round_trip() { + let rtt = Duration::from_millis(200); + let heard = UnixMillis(10_000); + let ahead = ClockOffset::of(UnixMillis(14_900), heard, rtt); + assert_eq!((ahead.offset_ms, ahead.within_ms), (5_000, 101)); + let behind = ClockOffset::of(UnixMillis(6_900), heard, rtt); + assert_eq!(behind.offset_ms, -3_000); + + let mut table = Table::default(); + let now = Instant::now(); + table.sent("libp2p:z", now); + table.clock("libp2p:z", ahead, now); + let wider = ClockOffset::of(UnixMillis(15_000), heard, rtt * 4); + table.clock("libp2p:z", wider, now); + assert_eq!( + table.paths["libp2p:z"].clock.map(|(kept, _)| kept), + Some(ahead) + ); + table.clock("libp2p:z", wider, now + CLOCK_KEPT); + assert_eq!( + table.paths["libp2p:z"].clock.map(|(kept, _)| kept), + Some(wider) + ); + // A clock set anew: a sample the kept one contradicts takes its place at once, + // its bound wider or not. + let set_anew = ClockOffset::of(UnixMillis(25_000), heard, rtt * 8); + table.clock("libp2p:z", set_anew, now + CLOCK_KEPT); + assert_eq!( + table.paths["libp2p:z"].clock.map(|(kept, _)| kept), + Some(set_anew) + ); + } + + /// A peer's only path is measured too once messages go over it, unless its + /// keepalives just did. + #[test] + fn a_path_sent_over_is_pinged() { + let mut table = Table::default(); + let now = Instant::now(); + table.sent("libp2p:z", now); + assert_eq!(table.due(now, || 1), [("libp2p:z".to_owned(), 1)]); + table.pong("libp2p:z", 1, now + Duration::from_millis(20)); + assert!(table.info("libp2p:z", now).rtt_ms.is_some()); + // A keepalive's PING within PROBE_EVERY: none of ours. + let later = now + PROBE_EVERY; + table.pinged("libp2p:z", 2, later); + table.sent("libp2p:z", later); + assert!(table.due(later, || 3).is_empty()); + } + /// A PING's round trip counts from when it goes, not from when it was found due. #[test] fn round_trips_count_from_the_send() { let mut table = Table::default(); let now = Instant::now(); - table.rank(paths(&["libp2p:x", "mycelium:x"]), now); + table.rank(paths(&["libp2p:x", "mycelium:x"]), now, true); let due = table.due(now, || 7); assert_eq!(due.len(), 2); // Sent a second late (after another path's slow send). @@ -1040,6 +1340,27 @@ mod tests { assert!((rtt - 100.0).abs() < 1e-6, "{rtt}"); } + /// Round trips of another way to the peer (through a relay, then directly) don't + /// weigh on those of the new one. + #[test] + fn a_path_that_goes_elsewhere_is_measured_anew() { + let mut table = Table::default(); + let now = Instant::now(); + table.rank(paths(&["libp2p:x", "mycelium:x"]), now, true); + let ping = |table: &mut Table, nonce: u64, at: Instant, ms: u64| { + table.pinged("libp2p:x", nonce, at); + table.pong("libp2p:x", nonce, at + Duration::from_millis(ms)); + }; + table.located("libp2p:x", "libp2p:/relayed"); + ping(&mut table, 1, now, 120); + table.located("libp2p:x", "libp2p:/relayed"); + ping(&mut table, 2, now + Duration::from_secs(30), 120); + table.located("libp2p:x", "libp2p:/lan"); + ping(&mut table, 3, now + Duration::from_secs(60), 2); + let rtt = table.info("libp2p:x", now).rtt_ms.unwrap(); + assert!((rtt - 2.0).abs() < 1e-6, "{rtt}"); + } + /// The node's keepalive PING on a path is its probe: paths doesn't ping it too, and /// the PONG measures it. #[test] @@ -1051,7 +1372,7 @@ mod tests { nonce += 1; nonce }; - table.rank(paths(&["libp2p:x", "mycelium:x"]), now); + table.rank(paths(&["libp2p:x", "mycelium:x"]), now, true); table.pinged("libp2p:x", 7, now); let due = table.due(now, &mut next); assert_eq!(due.len(), 1); @@ -1105,11 +1426,15 @@ mod tests { vec![("sim", net.transport("b") as Arc<dyn Transport>)], None, Some(Arc::downgrade(&gate)), + Arc::default(), + Duration::ZERO, )); let a = Arc::new(Paths::new( vec![("sim", net.transport("a") as Arc<dyn Transport>)], None, None, + Arc::default(), + Duration::ZERO, )); let received = |paths: &Arc<Paths>| { let (sender, received) = tokio::sync::mpsc::unbounded_channel(); @@ -1167,7 +1492,7 @@ mod tests { Ok(seed.to_owned()) } fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - crate::transport::mycelium::encode_message(peer, message) + (crate::transport::sim::WIRE.encode)(peer, message) } async fn send_encoded(&self, _message: &EncodedMessage) -> Result<()> { Err(anyhow!("broken")) @@ -1187,6 +1512,7 @@ mod tests { epoch_seq: 0, ready: false, syncs_with_you: false, + leaving: false, }) } @@ -1202,11 +1528,15 @@ mod tests { ], None, None, + Arc::default(), + Duration::ZERO, ); let b = Arc::new(Paths::new( vec![("sim", net.transport("b") as Arc<dyn Transport>)], None, None, + Arc::default(), + Duration::ZERO, )); let (sender, mut received) = tokio::sync::mpsc::unbounded_channel(); let handler: Arc<PathHandler> = Arc::new(move |from, _, _| { @@ -1219,13 +1549,13 @@ mod tests { tokio::spawn(async move { b.start_consumer(handler).await }); let addresses = vec!["sim:b".to_owned(), "broken:b".to_owned()]; - let ranked = a.rank(&addresses); + let ranked = a.rank(&addresses, true); assert_eq!(ranked, ["broken:b", "sim:b"]); // The configured order. let mut encoded = a.encode(&ranked[0], &some_status()).unwrap(); encoded.peer = "node-b".to_owned(); assert_eq!(a.send_first(&encoded, &ranked).await.unwrap(), "sim:b"); assert_eq!(received.recv().await.unwrap(), "sim:a"); - assert_eq!(a.rank(&addresses), ["sim:b", "broken:b"]); + assert_eq!(a.rank(&addresses, true), ["sim:b", "broken:b"]); // Without the message kept, nothing else can be tried: the transport's error. encoded.message = None; diff --git a/src/transport/sim.rs b/src/transport/sim.rs index 500050b..2f167dc 100644 --- a/src/transport/sim.rs +++ b/src/transport/sim.rs @@ -1,8 +1,9 @@ //! A simulated network for tests: many nodes in one process, each with a //! [`SimTransport`] on a shared [`SimNet`]. //! -//! Messages go through the Mycelium wire encoding (`mycelium::encode_message`), so the -//! codec is exercised as in production. The net delays, loses and duplicates them per +//! Messages go through the wire encoding (`WIRE`, as libp2p and Tor send it), so the +//! codec is exercised as in production; a net can use a transport's envelope instead +//! (`SimNet::with_codec`; Mycelium's tests run a swarm on its envelope). The net delays, loses and duplicates them per //! link, from a seeded generator; nodes can be taken down (everything to and from them //! is lost) and brought up again; and a test can inject a message as if a node had sent //! it, e.g. one that arrives late. @@ -24,11 +25,14 @@ use bytes::Bytes; use tokio::sync::{mpsc, oneshot}; use tracing::{debug, warn}; -use super::mycelium::{decode_message, encode_message}; use super::{EncodedMessage, MessageHandler, MessageKind, ParsedMessage, Transport}; use crate::locks::MutexExt; use crate::testrand::Rng; +/// How long the simulated net may take to deliver a message: well above its delays. +/// A catch-up source that stops sending sync messages is given up after this. +pub(crate) const DELIVERY_LIMIT: Duration = Duration::from_secs(3); + /// How one direction of a link behaves. #[derive(Clone, Debug)] pub(crate) struct LinkParams { @@ -79,6 +83,9 @@ struct NetInner { links: HashMap<(String, String), LinkParams>, down: HashSet<String>, inboxes: HashMap<String, mpsc::UnboundedSender<Delivery>>, + /// Per node, its current run (`restart`): what a transport of an earlier run sends + /// is lost, as from a process that is gone. + runs: HashMap<String, u64>, counts: NetCounts, } @@ -86,17 +93,26 @@ struct NetInner { #[derive(Clone)] pub(crate) struct SimNet { inner: Arc<Mutex<NetInner>>, + codec: Codec, } impl SimNet { + /// Messages in the wire encoding alone (`WIRE`). pub(crate) fn new(seed: u64) -> Self { + Self::with_codec(seed, WIRE) + } + + /// Messages put into bytes by `codec`. + pub(crate) fn with_codec(seed: u64, codec: Codec) -> Self { Self { + codec, inner: Arc::new(Mutex::new(NetInner { rng: Rng::new(seed), default_link: LinkParams::default(), links: HashMap::new(), down: HashSet::new(), inboxes: HashMap::new(), + runs: HashMap::new(), counts: NetCounts::default(), })), } @@ -109,15 +125,31 @@ impl SimNet { /// A transport for a new node `id` on this net. pub(crate) fn transport(&self, id: &str) -> Arc<SimTransport> { let (tx, rx) = mpsc::unbounded_channel(); - let previous = self.lock().inboxes.insert(id.to_owned(), tx); - assert!(previous.is_none(), "node {id} is already on the net"); + let run = { + let mut inner = self.lock(); + let previous = inner.inboxes.insert(id.to_owned(), tx); + assert!(previous.is_none(), "node {id} is already on the net"); + *inner.runs.entry(id.to_owned()).or_default() + }; Arc::new(SimTransport { id: id.to_owned(), + run, net: self.clone(), inbox: tokio::sync::Mutex::new(Some(rx)), }) } + /// Node `id`'s process is gone, as if it crashed: its transport neither sends nor + /// receives any more (messages on their way to it are lost), and [`transport`] + /// may give `id` a new one, for its next run. + /// + /// [`transport`]: Self::transport + pub(crate) fn restart(&self, id: &str) { + let mut inner = self.lock(); + inner.inboxes.remove(id); + *inner.runs.entry(id.to_owned()).or_default() += 1; + } + /// How links without their own parameters behave. pub(crate) fn set_default_link(&self, params: LinkParams) { self.lock().default_link = params; @@ -145,12 +177,13 @@ impl SimNet { } /// Hands a message to the net: decides now whether and when it arrives. - fn send(&self, from: &str, message: &EncodedMessage) { + fn send(&self, from: &str, run: u64, message: &EncodedMessage) { let delays = { let mut guard = self.lock(); let inner = &mut *guard; inner.counts.sent += 1; - if inner.down.contains(from) || inner.down.contains(&message.peer) { + let gone = inner.runs.get(from).is_some_and(|current| *current != run); + if gone || inner.down.contains(from) || inner.down.contains(&message.peer) { inner.counts.lost += 1; return; } @@ -215,7 +248,7 @@ impl SimNet { /// net. For messages a real node wouldn't send at that moment, e.g. one sent long /// ago and delivered late. pub(crate) async fn inject(&self, from: &str, to: &str, message: &ParsedMessage) -> Result<()> { - let encoded = encode_message(to, message)?; + let encoded = (self.codec.encode)(to, message)?; let (handled_tx, handled_rx) = oneshot::channel(); let delivery = Delivery { from: from.to_owned(), @@ -245,10 +278,35 @@ impl SimNet { /// are node IDs. pub(crate) struct SimTransport { id: String, + /// The run of its node it belongs to (`SimNet::restart`). + run: u64, net: SimNet, inbox: tokio::sync::Mutex<Option<mpsc::UnboundedReceiver<Delivery>>>, } +/// How a simulated net puts messages into bytes and back (`SimNet::with_codec`): the +/// wire encoding alone (`WIRE`, as libp2p and Tor send it), or a transport's envelope +/// around it (Mycelium's: `mycelium::CODEC`), so a whole swarm runs on that envelope. +#[derive(Clone, Copy)] +pub(crate) struct Codec { + pub(crate) encode: fn(&str, &ParsedMessage) -> Result<EncodedMessage>, + pub(crate) decode: fn(&[u8]) -> Result<ParsedMessage>, +} + +/// The wire encoding alone (`transport::wire`). +pub(crate) const WIRE: Codec = Codec { + encode: wire_encode, + decode: wire_decode, +}; + +fn wire_encode(peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { + super::wire::encoded(peer, message) +} + +fn wire_decode(body: &[u8]) -> Result<ParsedMessage> { + Ok(super::wire::decode(body)?) +} + #[async_trait] impl Transport for SimTransport { fn local_id(&self) -> &str { @@ -260,8 +318,8 @@ impl Transport for SimTransport { !id.is_empty() } - fn delivery_limit(&self) -> std::time::Duration { - crate::simtest::DELIVERY_LIMIT + fn delivery_limit(&self) -> Duration { + DELIVERY_LIMIT } /// A seed is a node ID; one no node has doesn't resolve (yet). @@ -274,13 +332,13 @@ impl Transport for SimTransport { } fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - encode_message(peer, message) + (self.net.codec.encode)(peer, message) } /// Always accepted: the net loses messages silently, like Mycelium once it has /// taken them. async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { - self.net.send(&self.id, message); + self.net.send(&self.id, self.run, message); Ok(()) } @@ -290,7 +348,7 @@ impl Transport for SimTransport { bail!("consumer of {} already started", self.id); }; while let Some(delivery) = inbox.recv().await { - match decode_message(&delivery.body) { + match (self.net.codec.decode)(&delivery.body) { Ok(message) => { debug!(node = %self.id, from = %delivery.from, message = ?message, "Simulated delivery"); if let Err(error) = handler(delivery.from, message, delivery.payload_len).await @@ -328,6 +386,7 @@ mod tests { epoch_seq: 0, ready: true, syncs_with_you: false, + leaving: false, }) } diff --git a/src/transport/stats.rs b/src/transport/stats.rs index 398ffdf..68fc3c6 100644 --- a/src/transport/stats.rs +++ b/src/transport/stats.rs @@ -6,6 +6,8 @@ use std::collections::BTreeMap; use serde::Serialize; +use super::Connection; + #[derive(Debug, Clone, Default, Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct TransportStats { @@ -32,12 +34,12 @@ impl TransportStats { } } -/// How the paths to peers with several addresses do (`transport::paths`). +/// How the paths to peers do (`transport::paths`). #[derive(Debug, Clone, Default, Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct PathsStats { - /// Every path measured: those of peers this node chose among lately, and those its - /// keepalives went over. + /// Every path measured: those of peers this node chose among or sent to lately, and + /// those its keepalives went over. pub(crate) paths: Vec<PathInfo>, #[serde(flatten)] pub(crate) counts: PathCounts, @@ -58,6 +60,10 @@ pub(crate) struct PathInfo { /// Its PINGs went unanswered three times in a row, or a send there failed within a /// minute: used only when no other path works. pub(crate) down: bool, + /// The transport's connection there, if it has connections (libp2p: direct or + /// relayed; `Transport::connection`). + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) connection: Option<Connection>, } /// Counts since start. @@ -82,6 +88,9 @@ pub(crate) struct IdentityStats { /// Nodes proven at an address, by transport (scheme); a node on several counts on /// each. pub(crate) proven: BTreeMap<String, usize>, + /// Of those, the nodes the transport reaches only through a relay (no direct + /// connection at its address there), by transport; absent where none is. + pub(crate) relayed: BTreeMap<String, usize>, /// Nodes known only from hints (the member list, HELLO_REPLYs), proven nowhere. pub(crate) hinted: usize, #[serde(flatten)] diff --git a/src/transport/tor.rs b/src/transport/tor.rs index 39e0f31..7ecd624 100644 --- a/src/transport/tor.rs +++ b/src/transport/tor.rs @@ -1,4 +1,4 @@ -//! Tor as a transport (`--transports tor`, in builds with the `tor` feature), for +//! Tor as a transport (`--tor-enable`, in builds with the `tor` feature), for //! reachability: every node runs an onion service (arti), reachable from anywhere //! without a forwarded port or a relay. Not for anonymity: a node's other addresses are //! in the member list next to its onion address. @@ -65,15 +65,19 @@ pub(crate) const SCHEME: &str = "tor"; /// The onion service's nickname in arti's state (one per node). const NICKNAME: &str = "continuum"; -/// The largest message read. Sync messages above 16 chunks travel as blobs, and a chunk -/// is at most 4 MiB. -const MAX_MESSAGE: usize = 8 << 20; +/// The largest message read and sent: the largest sync message a node of any +/// configuration sends whole (`MAX_SYNC_MESSAGE`: 16 chunks of 4 MiB; larger ones travel +/// as blobs), with room for its envelope. Over Tor at `MIN_RATE` that is a quarter of an +/// hour, which is the transport's delivery limit then. +const MAX_MESSAGE: usize = crate::swarm::blob::MAX_SYNC_MESSAGE + (1 << 20); /// How long reaching an onion service may take (building circuits, the rendezvous). const CONNECT_TIMEOUT: Duration = Duration::from_secs(60); /// How long the challenge handshake may take, a few round trips. const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30); /// How long a message may take, before what its size adds (`stream_timeout`). const STREAM_TIMEOUT: Duration = Duration::from_secs(30); +/// The first wait for an answer to a sync message (`Transport::sync_reply_timeout`). +const SYNC_REPLY_TIMEOUT: Duration = Duration::from_secs(30); /// The slowest a message is expected to go over Tor, in bytes per second. const MIN_RATE: u64 = 64 << 10; /// Messages read and not handled yet, at most. @@ -194,6 +198,39 @@ pub(crate) struct TorTransport { } impl TorTransport { + /// This transport's options, checked when the configuration is loaded + /// (`config::Settings::check`), if the node uses it: where arti keeps its state. + pub(crate) fn check(settings: &crate::config::Settings) -> Result<()> { + if settings.tor_state_dir.is_none() { + bail!( + "tor needs tor-state-dir: where arti keeps its state, the onion service's key among it" + ); + } + Ok(()) + } + + /// The transport as `settings` configure it (`transport::start`), proving `key`. + /// None if `stop` came first (bootstrapping arti can take a while). + pub(crate) async fn from_settings( + settings: &crate::config::Settings, + key: &Arc<NodeKey>, + mut stop: watch::Receiver<bool>, + ) -> Result<Option<Self>> { + let state_dir = settings + .tor_state_dir + .clone() + .context("tor needs tor-state-dir")?; + let config = TorConfig { + key: Arc::clone(key), + state_dir, + port: settings.tor_port, + }; + tokio::select! { + transport = Self::start(config) => transport.map(Some), + _ = stop.wait_for(|stop| *stop) => Ok(None), + } + } + /// Launches our onion service, and bootstraps arti in the background (that takes a /// while: it fetches what it needs to know of the Tor network). Until it is done, /// dials wait for it (arti bootstraps on demand), and the onion service is published @@ -534,6 +571,12 @@ impl Transport for TorTransport { stream_timeout(MAX_MESSAGE) } + /// Round trips over Tor take a second or more, and the first message to a peer + /// waits for a connection (`CONNECT_TIMEOUT`, `HANDSHAKE_TIMEOUT`). + fn sync_reply_timeout(&self) -> Duration { + SYNC_REPLY_TIMEOUT + } + fn local_addresses(&self) -> Vec<String> { vec![self.shared.local_id.clone()] } @@ -544,6 +587,15 @@ impl Transport for TorTransport { parse_location(target) } + /// Direct while a connection to its onion service is open: an onion circuit, with no + /// relay of ours in between. + fn connection(&self, target: &str) -> Option<crate::transport::Connection> { + let location = parse_location(target)?; + lock(&self.shared.connections) + .contains_key(&location) + .then_some(crate::transport::Connection::Direct) + } + /// An onion service is its own transport identity: nothing to find out. async fn resolve_seed(&self, seed: &str) -> Result<String> { let location = parse_location(seed) @@ -555,15 +607,7 @@ impl Transport for TorTransport { } fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - let body = wire::encode(message)?; - Ok(EncodedMessage { - peer: peer.to_owned(), - address: peer.to_owned(), - payload_len: body.len(), - body: body.into(), - kind: message.kind(), - message: None, - }) + wire::encoded(peer, message) } async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { diff --git a/src/transport/traffic.rs b/src/transport/traffic.rs index b237db2..b7907fb 100644 --- a/src/transport/traffic.rs +++ b/src/transport/traffic.rs @@ -8,7 +8,8 @@ //! A message dropped by the debug firewall (in the paths layer, below) still counts as //! sent (the firewall plays a lossy network) but not as received. -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, MutexGuard}; use anyhow::Result; @@ -29,6 +30,11 @@ impl Counter { self.messages += 1; self.bytes += bytes as u64; } + + fn absorb(&mut self, other: Counter) { + self.messages += other.messages; + self.bytes += other.bytes; + } } #[derive(Debug, Default, Clone, Copy, Serialize)] @@ -45,9 +51,42 @@ pub(crate) struct ByKind { pub(crate) identity: Counter, /// PING and PONG (`transport::paths`), counted by the paths layer under the address. pub(crate) paths: Counter, + /// LINKS and LINKS_REPLY (`swarm::links`). + pub(crate) links: Counter, } impl ByKind { + fn absorb(&mut self, other: &ByKind) { + for kind in [ + MessageKind::Hello, + MessageKind::HelloReply, + MessageKind::Sync, + MessageKind::Status, + MessageKind::StatusReply, + MessageKind::Blob, + MessageKind::Who, + MessageKind::Ping, + MessageKind::Links, + ] { + let theirs = *other.get(kind); + self.get_mut(kind).absorb(theirs); + } + } + + fn get(&self, kind: MessageKind) -> &Counter { + match kind { + MessageKind::Hello => &self.hello, + MessageKind::HelloReply => &self.hello_reply, + MessageKind::Sync => &self.sync, + MessageKind::Status => &self.status, + MessageKind::StatusReply => &self.status_reply, + MessageKind::Blob => &self.blob, + MessageKind::Who | MessageKind::Me => &self.identity, + MessageKind::Ping | MessageKind::Pong => &self.paths, + MessageKind::Links | MessageKind::LinksReply => &self.links, + } + } + fn get_mut(&mut self, kind: MessageKind) -> &mut Counter { match kind { MessageKind::Hello => &mut self.hello, @@ -58,6 +97,7 @@ impl ByKind { MessageKind::Blob => &mut self.blob, MessageKind::Who | MessageKind::Me => &mut self.identity, MessageKind::Ping | MessageKind::Pong => &mut self.paths, + MessageKind::Links | MessageKind::LinksReply => &mut self.links, } } } @@ -82,6 +122,169 @@ struct Table { #[derive(Debug, Default)] pub(crate) struct Traffic { table: Mutex<Table>, + /// The messages one by one, while the dashboard looks. + pub(crate) log: MessageLog, +} + +/// Messages kept in the log (`MessageLog`) at most, and the text of one. +const LOGGED: usize = 1000; +const LOGGED_TEXT: usize = 8 * 1024; + +/// The node's messages one by one, for the dashboard's console (`api::dashboard`): +/// kept only while it looks (`watch`), the latest `LOGGED`. Those this layer sees: the +/// node's own, not the paths layer's PINGs and PONGs nor the identity layer's WHO and +/// ME (counted under their addresses, `ByKind`), but the PINGs and PONGs the node is +/// handed. +#[derive(Debug, Default)] +pub(crate) struct MessageLog { + watching: AtomicBool, + inner: Mutex<LogInner>, +} + +#[derive(Debug, Default)] +struct LogInner { + next: u64, + entries: VecDeque<Logged>, +} + +/// One message, as the dashboard shows it. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Logged { + /// Counts up, from 1. + pub(crate) seq: u64, + /// Unix milliseconds, by the system's clock. + pub(crate) at: u64, + /// `sent`, `received` or `refused` (the transport wouldn't send it). + pub(crate) way: &'static str, + pub(crate) peer: String, + /// The message's type, as the docs name it: `HELLO_REPLY`, `SYNC`, … + #[serde(rename = "type")] + pub(crate) kind: &'static str, + /// Its kind among the counters (`ByKind`'s names): `helloReply`, `sync`, … + pub(crate) counted: &'static str, + pub(crate) bytes: usize, + /// The message, written out (`Debug`; sizes for its bytes), `LOGGED_TEXT` at most. + pub(crate) text: String, + /// What ties it to other messages (`related`): those with the same are one + /// exchange. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) related: Option<String>, + /// The address it went to or came from, where the paths layer logged it under that + /// (its own PINGs and PONGs) and the node there is known: `peer` is the node then + /// (`api::dashboard`). + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) address: Option<String>, +} + +/// What ties `message`, with `peer`, to other messages: a PING and its PONG (the +/// nonce alone: a PING may go to an address and its PONG come back up from the node +/// there, `paths`), a LINKS and its replies (the nonce), a HELLO and its replies (the run of the +/// node that sent the HELLO, which the reply echoes), a blob's requests and answers, from +/// any peer (its ID), and a delta offered in a SYNC with the blob it is fetched as. None +/// for the rest (a STATUS and its reply carry nothing of the kind). +fn related(peer: &str, message: &ParsedMessage) -> Option<String> { + use crate::transport::{Hex, SyncBody}; + Some(match message { + ParsedMessage::Ping(nonce) | ParsedMessage::Pong { nonce, .. } => { + format!("ping {nonce}") + } + ParsedMessage::Links(request) => format!("links {peer} {}", request.nonce), + ParsedMessage::LinksReply(reply) => format!("links {peer} {}", reply.nonce), + ParsedMessage::Hello(hello) => format!("hello {peer} {}", Hex(&hello.run_id)), + ParsedMessage::HelloReply(reply) => format!("hello {peer} {}", Hex(&reply.run_id)), + ParsedMessage::Blob(blob) => format!("blob {}", blob.id), + ParsedMessage::Sync(envelope) => match &envelope.body { + SyncBody::Delta(spec) => { + format!("blob {}", crate::transport::BlobId::Delta(spec.id())) + } + SyncBody::Message(_) => return None, + }, + _ => return None, + }) +} + +// Read by the dashboard only. +#[cfg_attr(not(feature = "dashboard"), allow(dead_code))] +impl MessageLog { + /// Keeps messages from now on (`true`), or no more, forgetting those kept. + pub(crate) fn watch(&self, on: bool) { + self.watching.store(on, Ordering::Relaxed); + if !on { + self.inner.lock_or_recover().entries.clear(); + } + } + + pub(crate) fn note( + &self, + way: &'static str, + peer: &str, + message: &ParsedMessage, + bytes: usize, + ) { + if !self.watching.load(Ordering::Relaxed) { + return; + } + let at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |since| since.as_millis() as u64); + let mut text = format!("{message:#?}"); + if text.len() > LOGGED_TEXT { + let mut cut = LOGGED_TEXT; + while !text.is_char_boundary(cut) { + cut -= 1; + } + text.truncate(cut); + text.push_str("\n…"); + } + let (kind, counted) = names(message.kind()); + let mut inner = self.inner.lock_or_recover(); + inner.next += 1; + let seq = inner.next; + inner.entries.push_back(Logged { + seq, + at, + way, + peer: peer.to_owned(), + kind, + counted, + bytes, + text, + related: related(peer, message), + address: None, + }); + if inner.entries.len() > LOGGED { + inner.entries.pop_front(); + } + } + + /// The messages after `seq`, the latest `most` of them. + pub(crate) fn after(&self, seq: u64, most: usize) -> Vec<Logged> { + let inner = self.inner.lock_or_recover(); + let newer: Vec<&Logged> = inner.entries.iter().filter(|e| e.seq > seq).collect(); + newer[newer.len().saturating_sub(most)..] + .iter() + .map(|entry| (*entry).clone()) + .collect() + } +} + +/// A message kind's type name, and its counter's (`ByKind`). +fn names(kind: MessageKind) -> (&'static str, &'static str) { + match kind { + MessageKind::Hello => ("HELLO", "hello"), + MessageKind::HelloReply => ("HELLO_REPLY", "helloReply"), + MessageKind::Sync => ("SYNC", "sync"), + MessageKind::Status => ("STATUS", "status"), + MessageKind::StatusReply => ("STATUS_REPLY", "statusReply"), + MessageKind::Blob => ("BLOB", "blob"), + MessageKind::Who => ("WHO", "identity"), + MessageKind::Me => ("ME", "identity"), + MessageKind::Ping => ("PING", "paths"), + MessageKind::Pong => ("PONG", "paths"), + MessageKind::Links => ("LINKS", "links"), + MessageKind::LinksReply => ("LINKS_REPLY", "links"), + } } impl Traffic { @@ -115,6 +318,22 @@ impl Traffic { self.lock().peers.get(peer).copied() } + /// A node's traffic: counted under it, and under `addresses`, where it is (the + /// layers below the node count what they send and receive on their own there: WHO + /// and ME, the paths layer's PINGs and PONGs). + pub(crate) fn node(&self, node: &str, addresses: &[String]) -> PeerTraffic { + let table = self.lock(); + let mut traffic = table.peers.get(node).copied().unwrap_or_default(); + for address in addresses.iter().filter(|address| address.as_str() != node) { + if let Some(there) = table.peers.get(address) { + traffic.sent.absorb(&there.sent); + traffic.received.absorb(&there.received); + traffic.refused.absorb(&there.refused); + } + } + traffic + } + pub(crate) fn total(&self) -> PeerTraffic { self.lock().total } @@ -152,6 +371,10 @@ impl Transport for CountingTransport { self.inner.delivery_limit() } + fn sync_reply_timeout(&self) -> std::time::Duration { + self.inner.sync_reply_timeout() + } + fn local_addresses(&self) -> Vec<String> { self.inner.local_addresses() } @@ -176,6 +399,14 @@ impl Transport for CountingTransport { self.inner.discovered_seeds() } + fn on_seeds_found(&self, found: std::sync::Arc<tokio::sync::Notify>) { + self.inner.on_seeds_found(found) + } + + fn on_reachable(&self, reachable: std::sync::Arc<tokio::sync::Notify>) { + self.inner.on_reachable(reachable) + } + fn transports(&self) -> Vec<String> { self.inner.transports() } @@ -204,6 +435,10 @@ impl Transport for CountingTransport { self.inner.rtt(peer) } + fn clock_offset(&self, peer: &str) -> Option<crate::transport::paths::ClockOffset> { + self.inner.clock_offset(peer) + } + async fn resolve_seed(&self, seed: &str) -> Result<String> { self.inner.resolve_seed(seed).await } @@ -221,6 +456,12 @@ impl Transport for CountingTransport { self.traffic .refused(&message.peer, message.kind, message.payload_len); } + if let Some(parsed) = &message.message { + let way = if result.is_ok() { "sent" } else { "refused" }; + self.traffic + .log + .note(way, &message.peer, parsed, message.payload_len); + } result } @@ -229,6 +470,7 @@ impl Transport for CountingTransport { self.inner .start_consumer(Box::new(move |sender, message, payload_len| { traffic.received(&sender, message.kind(), payload_len); + traffic.log.note("received", &sender, &message, payload_len); handler(sender, message, payload_len) })) .await diff --git a/src/transport/wire.rs b/src/transport/wire.rs index fcbee85..7717200 100644 --- a/src/transport/wire.rs +++ b/src/transport/wire.rs @@ -10,11 +10,11 @@ use thiserror::Error; -use super::{MessageKind, ParsedMessage}; +use super::{EncodedMessage, MessageKind, ParsedMessage}; /// The wire format's version, the first byte of every payload. Bump it with every change /// to a message type or a codec (`golden_bytes` fails then). -pub(crate) const VERSION: u8 = 6; +pub(crate) const VERSION: u8 = 14; /// What the second byte of a payload says. fn tag(kind: MessageKind) -> u8 { @@ -29,6 +29,8 @@ fn tag(kind: MessageKind) -> u8 { MessageKind::Me => 7, MessageKind::Ping => 8, MessageKind::Pong => 9, + MessageKind::Links => 10, + MessageKind::LinksReply => 11, } } @@ -50,6 +52,21 @@ pub(crate) enum WireError { Decode(#[from] rmp_serde::decode::Error), } +/// `message` to `peer`, encoded as it goes on the wire alone (no envelope: libp2p, +/// Tor, the simulator); `peer` is its address too. +#[cfg_attr(not(any(feature = "libp2p", feature = "tor", test)), allow(dead_code))] +pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result<EncodedMessage> { + let body = encode(message)?; + Ok(EncodedMessage { + peer: peer.to_owned(), + address: peer.to_owned(), + payload_len: body.len(), + body: body.into(), + kind: message.kind(), + message: None, + }) +} + /// The payload of `message`. pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> { let mut payload = vec![VERSION, tag(message.kind())]; @@ -64,9 +81,10 @@ pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> { ParsedMessage::Who(proof) | ParsedMessage::Me(proof) => { rmp_serde::encode::write(&mut payload, proof)? } - ParsedMessage::Ping(nonce) | ParsedMessage::Pong(nonce) => { - rmp_serde::encode::write(&mut payload, nonce)? - } + ParsedMessage::Ping(nonce) => rmp_serde::encode::write(&mut payload, nonce)?, + ParsedMessage::Pong { nonce, at } => rmp_serde::encode::write(&mut payload, &(nonce, at))?, + ParsedMessage::Links(request) => rmp_serde::encode::write(&mut payload, request)?, + ParsedMessage::LinksReply(reply) => rmp_serde::encode::write(&mut payload, reply)?, } Ok(payload) } @@ -98,7 +116,12 @@ pub(crate) fn decode(payload: &[u8]) -> Result<ParsedMessage, WireError> { } else if is(MessageKind::Ping) { ParsedMessage::Ping(rmp_serde::from_slice(body)?) } else if is(MessageKind::Pong) { - ParsedMessage::Pong(rmp_serde::from_slice(body)?) + let (nonce, at) = rmp_serde::from_slice(body)?; + ParsedMessage::Pong { nonce, at } + } else if is(MessageKind::Links) { + ParsedMessage::Links(rmp_serde::from_slice(body)?) + } else if is(MessageKind::LinksReply) { + ParsedMessage::LinksReply(rmp_serde::from_slice(body)?) } else { return Err(WireError::UnknownKind(kind)); }) @@ -180,7 +203,7 @@ pub(crate) mod codec { /// An Automerge change hash. pub(crate) mod change_hash { use super::*; - use automerge::ChangeHash; + use crate::document::ChangeHash; pub(crate) fn serialize<S: Serializer>( value: &ChangeHash, @@ -219,8 +242,8 @@ pub(crate) mod codec { /// empty), or as a digest (none listed). pub(crate) mod heads { use super::*; + use crate::document::ChangeHash; use crate::transport::{HeadsSummary, MAX_LISTED_HEADS}; - use automerge::ChangeHash; pub(crate) fn serialize<S: Serializer>( value: &HeadsSummary, @@ -267,7 +290,7 @@ pub(crate) mod codec { /// Change hashes, each a 32-byte binary. pub(crate) mod change_hashes { use super::*; - use automerge::ChangeHash; + use crate::document::ChangeHash; pub(crate) fn serialize<S: Serializer>( value: &[ChangeHash], @@ -325,15 +348,134 @@ pub(crate) mod codec { } } +/// One message of every kind, for the codecs' tests (this one's, and each transport's +/// envelope around it). +#[cfg(test)] +pub(crate) fn samples() -> Vec<ParsedMessage> { + use crate::document::ChangeHash; + use crate::document::replica::Epoch; + use crate::transport::{ + BlobBody, BlobId, BlobMessage, DeltaSpec, HeadsSummary, HelloReply, HelloRequest, + IdentityProof, ListedPeer, Status, SyncBody, SyncEnvelope, + }; + let epoch = Epoch { + generation: 2, + id: [3; 32], + }; + let status = Status { + heads: HeadsSummary::of(vec![ChangeHash([1; 32])]), + epoch, + epoch_seq: 1, + ready: true, + syncs_with_you: true, + leaving: false, + }; + let proof = IdentityProof { + key: [8; 32], + signature: [9; 64], + }; + vec![ + ParsedMessage::Hello(HelloRequest { + swarm_id: Some("swarm".into()), + epoch, + epoch_seq: 1, + auth: vec![1, 2, 3], + ready: true, + run_id: [4; 16], + handshake: 7, + }), + ParsedMessage::HelloReply(HelloReply { + swarm_id: "swarm".into(), + peers: vec![ListedPeer { + id: "p".into(), + addresses: vec!["a".into()], + }], + epoch, + epoch_seq: 1, + auth: Vec::new(), + ready: false, + run_id: [4; 16], + away_ms: Some(5), + sender_run_id: [6; 16], + }), + ParsedMessage::Sync(SyncEnvelope { + swarm_id: "swarm".into(), + epoch, + epoch_seq: 1, + body: SyncBody::Delta(DeltaSpec { + from: Vec::new(), + to: vec![ChangeHash([2; 32])], + }), + ready: true, + }), + ParsedMessage::Status(status.clone()), + ParsedMessage::StatusReply(status), + ParsedMessage::Blob(BlobMessage { + epoch, + epoch_seq: 1, + id: BlobId::Content([5; 32]), + body: BlobBody::Missing, + }), + ParsedMessage::Who(proof.clone()), + ParsedMessage::Me(proof), + ParsedMessage::Ping(7), + ParsedMessage::Pong { + nonce: u64::MAX, + at: crate::clock::UnixMillis(5), + }, + ParsedMessage::Links(crate::transport::LinksRequest { + epoch, + epoch_seq: 1, + nonce: 3, + subscribe_secs: 60, + }), + ParsedMessage::LinksReply(crate::transport::LinksReply { + epoch, + epoch_seq: 1, + nonce: 3, + links: vec![crate::transport::Link { + peer: "p".into(), + transport: Some("libp2p".into()), + relayed: true, + down: false, + rtt_us: Some(1500), + dormant: false, + others: vec![crate::transport::OtherPath { + transport: "mycelium".into(), + rtt_us: None, + relayed: false, + }], + clock: Some(crate::transport::paths::ClockOffset { + offset_ms: -2_500, + within_ms: 40, + }), + }], + }), + ] +} + #[cfg(test)] mod tests { use super::*; + use crate::document::ChangeHash; use crate::document::replica::Epoch; use crate::transport::{ BlobBody, BlobId, BlobMessage, HeadsSummary, HelloReply, HelloRequest, IdentityProof, ListedPeer, MAX_LISTED_HEADS, Status, }; - use automerge::ChangeHash; + + /// Every kind decodes to what was encoded: re-encoded, the same bytes. + #[test] + fn every_kind_round_trips() { + let samples = samples(); + assert_eq!(samples.len(), 12, "a kind is missing from `samples`"); + for message in samples { + let bytes = encode(&message).unwrap(); + let decoded = decode(&bytes).unwrap(); + assert_eq!(decoded.kind(), message.kind()); + assert_eq!(encode(&decoded).unwrap(), bytes, "{message:?}"); + } + } #[test] fn handshakes_round_trip() { @@ -398,6 +540,7 @@ mod tests { epoch_seq: 0, ready: true, syncs_with_you: false, + leaving: false, }; let decoded = decode(&encode(&ParsedMessage::StatusReply(status.clone())).unwrap()).unwrap(); @@ -420,6 +563,7 @@ mod tests { epoch_seq: 0, ready: true, syncs_with_you: true, + leaving: false, }; assert!(decode(&encode(&ParsedMessage::Status(status)).unwrap()).is_err()); } @@ -454,13 +598,15 @@ mod tests { } fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() + crate::transport::Hex(bytes).to_string() } /// The exact bytes of a few messages. If this fails after a change to a message type /// or a codec, the change is a new wire format: bump `VERSION` and take the new bytes. #[test] fn golden_bytes() { + // The version byte follows `VERSION`: bumped for a change to other messages. + let v = format!("{VERSION:02x}"); let epoch = Epoch { generation: 2, id: [3; 32], @@ -479,7 +625,7 @@ mod tests { }); assert_eq!( hex(&encode(&hello).unwrap()), - format!("060097a5737761726d{epoch_hex}01c403010203c3{}07", run("04")) + format!("{v}0097a5737761726d{epoch_hex}01c403010203c3{}07", run("04")) ); let reply = ParsedMessage::HelloReply(HelloReply { @@ -499,7 +645,7 @@ mod tests { assert_eq!( hex(&encode(&reply).unwrap()), format!( - "060199a5737761726d9192a17091a161{epoch_hex}01c400c2{}05{}", + "{v}0199a5737761726d9192a17091a161{epoch_hex}01c400c2{}05{}", run("04"), run("06") ) @@ -511,10 +657,11 @@ mod tests { epoch_seq: 1, ready: true, syncs_with_you: false, + leaving: false, }); assert_eq!( hex(&encode(&status).unwrap()), - format!("0603959290c400{epoch_hex}01c3c2") + format!("{v}03969290c400{epoch_hex}01c3c2c2") ); let who = ParsedMessage::Who(IdentityProof { @@ -523,7 +670,7 @@ mod tests { }); assert_eq!( hex(&encode(&who).unwrap()), - format!("060692c420{}c440{}", "08".repeat(32), "09".repeat(64)) + format!("{v}0692c420{}c440{}", "08".repeat(32), "09".repeat(64)) ); let ParsedMessage::Me(proof) = decode( &encode(&ParsedMessage::Me(IdentityProof { @@ -537,14 +684,16 @@ mod tests { }; assert_eq!((proof.key, proof.signature), ([8; 32], [9; 64])); - assert_eq!(hex(&encode(&ParsedMessage::Ping(7)).unwrap()), "060807"); - assert_eq!( - hex(&encode(&ParsedMessage::Pong(u64::MAX)).unwrap()), - "0609cfffffffffffffffff" - ); - let ParsedMessage::Pong(nonce) = decode(&[VERSION, 9, 0x2a]).unwrap() else { + assert_eq!(hex(&encode(&ParsedMessage::Ping(7)).unwrap()), format!("{v}0807")); + let pong = ParsedMessage::Pong { + nonce: u64::MAX, + at: crate::clock::UnixMillis(5), + }; + assert_eq!(hex(&encode(&pong).unwrap()), format!("{v}0992cfffffffffffffffff05")); + let ParsedMessage::Pong { nonce, at } = decode(&[VERSION, 9, 0x92, 0x2a, 0x07]).unwrap() + else { panic!("not a PONG"); }; - assert_eq!(nonce, 42); + assert_eq!((nonce, at.as_u64()), (42, 7)); } } |