summaryrefslogtreecommitdiffstats
path: root/src/transport
diff options
context:
space:
mode:
Diffstat (limited to 'src/transport')
-rw-r--r--src/transport/challenge.rs277
-rw-r--r--src/transport/firewall.rs92
-rw-r--r--src/transport/identity.rs748
-rw-r--r--src/transport/libp2p.rs1970
-rw-r--r--src/transport/mod.rs647
-rw-r--r--src/transport/mycelium.rs1276
-rw-r--r--src/transport/mycelium.rs.orig975
-rw-r--r--src/transport/mycelium/daemon.rs333
-rw-r--r--src/transport/mycelium/mod.rs683
-rw-r--r--src/transport/paths.rs1247
-rw-r--r--src/transport/sim.rs36
-rw-r--r--src/transport/stats.rs189
-rw-r--r--src/transport/tor.rs673
-rw-r--r--src/transport/traffic.rs143
-rw-r--r--src/transport/wire.rs550
15 files changed, 7344 insertions, 2495 deletions
diff --git a/src/transport/challenge.rs b/src/transport/challenge.rs
new file mode 100644
index 0000000..f06d2fe
--- /dev/null
+++ b/src/transport/challenge.rs
@@ -0,0 +1,277 @@
+//! A challenge handshake that proves, at the start of a connection, which key is at the
+//! other end: for transports that don't authenticate their peers themselves (an onion
+//! service learns nothing about who connects to it, `transport::tor`).
+//!
+//! Both sides send a HELLO: their public key and a fresh random nonce. Then each sends
+//! a PROOF: its signature over a label, its role (dialer or listener), the connection's
+//! context, and both HELLOs (its own first). Each checks the other's signature against
+//! the key in the other's HELLO, and from then on knows that key is at the other end.
+//!
+//! * A recorded PROOF is useless elsewhere: it signs the other side's nonce, fresh for
+//! each connection.
+//! * The roles keep a side's own PROOF from being reflected back to it.
+//! * The context binds the handshake to the connection as the dialer meant it: the
+//! listener's address, as the dialer dialed it and as the listener knows itself. A
+//! party in the middle that passes one side's handshake on to someone else fails
+//! there: the context it was signed for is a different one.
+//!
+//! What it doesn't do: protect what follows. That is the connection's job, so the
+//! handshake is only enough over one that is encrypted and tamper-proof end to end, and
+//! authenticates the listener to the dialer (a Tor stream to an onion service). Over a
+//! bare network (UDP, say), it would need key agreement and protected messages as well.
+//!
+//! The signing key is the node key (`crate::identity`); the label keeps these
+//! signatures apart from the node key's others (WHO and ME).
+
+use std::time::Duration;
+
+use anyhow::{Result, anyhow, bail};
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+
+use crate::identity::{self, NodeKey};
+
+const LABEL: &[u8] = b"continuum/challenge/1";
+/// The first byte of a HELLO: this handshake's version.
+const VERSION: u8 = 1;
+const HELLO_BYTES: usize = 1 + 32 + 32;
+const PROOF_BYTES: usize = 64;
+
+/// Which end of the connection a side is.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum Role {
+ Dialer,
+ Listener,
+}
+
+impl Role {
+ fn byte(self) -> u8 {
+ match self {
+ Self::Dialer => 0,
+ Self::Listener => 1,
+ }
+ }
+
+ fn other(self) -> Self {
+ match self {
+ Self::Dialer => Self::Listener,
+ Self::Listener => Self::Dialer,
+ }
+ }
+}
+
+/// One side's HELLO.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) struct Hello {
+ pub(crate) key: [u8; 32],
+ pub(crate) nonce: [u8; 32],
+}
+
+impl Hello {
+ /// Ours, with a fresh nonce.
+ pub(crate) fn new(key: &NodeKey) -> Result<Self> {
+ let mut nonce = [0u8; 32];
+ getrandom::fill(&mut nonce).map_err(|error| anyhow!("No randomness: {error}"))?;
+ Ok(Self {
+ key: key.public(),
+ nonce,
+ })
+ }
+
+ pub(crate) fn to_bytes(self) -> [u8; HELLO_BYTES] {
+ let mut bytes = [0u8; HELLO_BYTES];
+ bytes[0] = VERSION;
+ bytes[1..33].copy_from_slice(&self.key);
+ bytes[33..].copy_from_slice(&self.nonce);
+ bytes
+ }
+
+ pub(crate) fn from_bytes(bytes: &[u8; HELLO_BYTES]) -> Result<Self> {
+ if bytes[0] != VERSION {
+ bail!(
+ "challenge handshake version {} (we speak {VERSION})",
+ bytes[0]
+ );
+ }
+ let mut hello = Self {
+ key: [0; 32],
+ nonce: [0; 32],
+ };
+ hello.key.copy_from_slice(&bytes[1..33]);
+ hello.nonce.copy_from_slice(&bytes[33..]);
+ Ok(hello)
+ }
+}
+
+/// What the side in `role` signs: the label, its role, the context, then its own HELLO
+/// and the other side's.
+fn signed(role: Role, context: &str, own: &Hello, other: &Hello) -> Vec<u8> {
+ let mut bytes = LABEL.to_vec();
+ bytes.push(role.byte());
+ bytes.extend_from_slice(&(context.len() as u32).to_be_bytes());
+ bytes.extend_from_slice(context.as_bytes());
+ for hello in [own, other] {
+ bytes.extend_from_slice(&hello.key);
+ bytes.extend_from_slice(&hello.nonce);
+ }
+ bytes
+}
+
+/// Our PROOF, as the side in `role`.
+pub(crate) fn prove(
+ key: &NodeKey,
+ role: Role,
+ context: &str,
+ own: &Hello,
+ other: &Hello,
+) -> [u8; PROOF_BYTES] {
+ key.sign(&signed(role, context, own, other))
+}
+
+/// Whether `proof` is the other side's (the one not in `role`) for this handshake.
+pub(crate) fn verify(
+ role: Role,
+ context: &str,
+ own: &Hello,
+ other: &Hello,
+ proof: &[u8; PROOF_BYTES],
+) -> bool {
+ identity::verify(
+ &other.key,
+ &signed(role.other(), context, other, own),
+ proof,
+ )
+}
+
+/// Runs the handshake on `stream` as the side in `role`, within `timeout`: returns the
+/// public key at the other end. `context`: the listener's address, as dialed (module
+/// docs).
+pub(crate) async fn run<S>(
+ stream: &mut S,
+ key: &NodeKey,
+ role: Role,
+ context: &str,
+ timeout: Duration,
+) -> Result<[u8; 32]>
+where
+ S: AsyncRead + AsyncWrite + Unpin,
+{
+ let exchange = async {
+ let own = Hello::new(key)?;
+ stream.write_all(&own.to_bytes()).await?;
+ stream.flush().await?;
+ let mut hello = [0u8; HELLO_BYTES];
+ stream.read_exact(&mut hello).await?;
+ let other = Hello::from_bytes(&hello)?;
+ if other.key == own.key {
+ bail!("the other end has our own key");
+ }
+ stream
+ .write_all(&prove(key, role, context, &own, &other))
+ .await?;
+ stream.flush().await?;
+ let mut proof = [0u8; PROOF_BYTES];
+ stream.read_exact(&mut proof).await?;
+ if !verify(role, context, &own, &other, &proof) {
+ bail!("the other end's proof doesn't hold");
+ }
+ Ok(other.key)
+ };
+ tokio::time::timeout(timeout, exchange)
+ .await
+ .map_err(|_| anyhow!("no handshake within {timeout:?}"))?
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn key(seed: u8) -> NodeKey {
+ NodeKey::from_seed([seed; 32])
+ }
+
+ #[test]
+ fn a_proof_holds_for_its_handshake_only() {
+ let (a, b) = (key(1), key(2));
+ let (ha, hb) = (Hello::new(&a).unwrap(), Hello::new(&b).unwrap());
+ let context = "example.onion:9663";
+ let proof = prove(&a, Role::Dialer, context, &ha, &hb);
+ // b, the listener, checks a's proof.
+ assert!(verify(Role::Listener, context, &hb, &ha, &proof));
+ // Another context (passed on to another listener).
+ assert!(!verify(
+ Role::Listener,
+ "other.onion:9663",
+ &hb,
+ &ha,
+ &proof
+ ));
+ // Another connection: b's nonce is a new one.
+ let hb2 = Hello::new(&b).unwrap();
+ assert!(!verify(Role::Listener, context, &hb2, &ha, &proof));
+ // Reflected: a can't be fooled by its own proof as if b had sent it.
+ assert!(!verify(Role::Dialer, context, &ha, &hb, &proof));
+ // Someone else's key in the HELLO.
+ let hc = Hello {
+ key: key(3).public(),
+ nonce: ha.nonce,
+ };
+ assert!(!verify(Role::Listener, context, &hb, &hc, &proof));
+ }
+
+ #[test]
+ fn hellos_round_trip_and_versions_are_checked() {
+ let hello = Hello::new(&key(1)).unwrap();
+ assert_eq!(Hello::from_bytes(&hello.to_bytes()).unwrap(), hello);
+ let mut bytes = hello.to_bytes();
+ bytes[0] = VERSION + 1;
+ assert!(Hello::from_bytes(&bytes).is_err());
+ }
+
+ #[tokio::test]
+ async fn both_ends_learn_each_others_key() {
+ let (mut dialer, mut listener) = tokio::io::duplex(1024);
+ let (a, b) = (key(1), key(2));
+ let context = "example.onion:9663";
+ let (at_a, at_b) = tokio::join!(
+ run(
+ &mut dialer,
+ &a,
+ Role::Dialer,
+ context,
+ Duration::from_secs(5)
+ ),
+ run(
+ &mut listener,
+ &b,
+ Role::Listener,
+ context,
+ Duration::from_secs(5)
+ ),
+ );
+ assert_eq!(at_a.unwrap(), b.public());
+ assert_eq!(at_b.unwrap(), a.public());
+ }
+
+ #[tokio::test]
+ async fn a_different_context_fails_on_both_ends() {
+ let (mut dialer, mut listener) = tokio::io::duplex(1024);
+ let (a, b) = (key(1), key(2));
+ let (at_a, at_b) = tokio::join!(
+ run(
+ &mut dialer,
+ &a,
+ Role::Dialer,
+ "meant.onion:1",
+ Duration::from_secs(5)
+ ),
+ run(
+ &mut listener,
+ &b,
+ Role::Listener,
+ "actual.onion:1",
+ Duration::from_secs(5)
+ ),
+ );
+ assert!(at_a.is_err() && at_b.is_err());
+ }
+}
diff --git a/src/transport/firewall.rs b/src/transport/firewall.rs
index b8e05cd..7b0ec0d 100644
--- a/src/transport/firewall.rs
+++ b/src/transport/firewall.rs
@@ -1,105 +1,49 @@
+//! The debug firewall (`/debug/firewall`, with `--debug-api`): drops messages from and
+//! to peers, by node ID, to test partitions on a real swarm. It works at the bottom of
+//! the node, in the paths layer (`transport::paths::Gate`), so it drops everything a
+//! partition would: WHO and ME, PINGs and their PONGs, as well as the node's messages.
+
use std::collections::HashSet;
-use std::sync::{Arc, RwLock, RwLockReadGuard};
+use std::sync::{RwLock, RwLockReadGuard};
-use anyhow::Result;
-use async_trait::async_trait;
use serde::{Deserialize, Serialize};
-use utoipa::ToSchema;
-use crate::transport::EncodedMessage;
-use crate::transport::{MessageHandler, ParsedMessage, Transport};
+use crate::locks::RwLockExt;
-#[derive(Debug, Clone, Default, Serialize, Deserialize, ToSchema)]
-pub struct FirewallRules {
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub(crate) struct FirewallRules {
/// Messages from these peers are dropped before the application sees them.
#[serde(default)]
- pub inbound: HashSet<String>,
+ pub(crate) inbound: HashSet<String>,
/// Messages to these peers are silently dropped, like a lossy network.
#[serde(default)]
- pub outbound: HashSet<String>,
+ pub(crate) outbound: HashSet<String>,
}
/// Uses a std lock: checks are tiny and the guard is never held across an await.
#[derive(Default)]
-pub struct Firewall {
+pub(crate) struct Firewall {
rules: RwLock<FirewallRules>,
}
impl Firewall {
fn read(&self) -> RwLockReadGuard<'_, FirewallRules> {
- self.rules
- .read()
- .unwrap_or_else(|poisoned| poisoned.into_inner())
+ self.rules.read_or_recover()
}
- pub fn rules(&self) -> FirewallRules {
+ pub(crate) fn rules(&self) -> FirewallRules {
self.read().clone()
}
- pub fn set_rules(&self, rules: FirewallRules) {
- *self
- .rules
- .write()
- .unwrap_or_else(|poisoned| poisoned.into_inner()) = rules;
+ pub(crate) fn set_rules(&self, rules: FirewallRules) {
+ *self.rules.write_or_recover() = rules;
}
- pub fn blocks_inbound(&self, peer: &str) -> bool {
+ pub(crate) fn blocks_inbound(&self, peer: &str) -> bool {
self.read().inbound.contains(peer)
}
- pub fn blocks_outbound(&self, peer: &str) -> bool {
+ pub(crate) fn blocks_outbound(&self, peer: &str) -> bool {
self.read().outbound.contains(peer)
}
}
-
-pub struct FirewallTransport {
- inner: Arc<dyn Transport>,
- firewall: Arc<Firewall>,
-}
-
-impl FirewallTransport {
- pub fn new(inner: Arc<dyn Transport>, firewall: Arc<Firewall>) -> Self {
- Self { inner, firewall }
- }
-}
-
-#[async_trait]
-impl Transport for FirewallTransport {
- fn local_id(&self) -> &str {
- self.inner.local_id()
- }
-
- async fn resolve_seed(&self, seed: &str) -> Result<String> {
- self.inner.resolve_seed(seed).await
- }
-
- fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- self.inner.encode(peer, message)
- }
-
- async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> {
- if self.firewall.blocks_outbound(&message.peer) {
- tracing::trace!(peer = %message.peer, "Firewall dropped outbound message");
- return Ok(());
- }
- self.inner.send_encoded(message).await
- }
-
- async fn start_consumer(&self, handler: MessageHandler) -> Result<()> {
- let firewall = Arc::clone(&self.firewall);
-
- self.inner
- .start_consumer(Box::new(move |sender, message, payload_len| {
- if firewall.blocks_inbound(&sender) {
- tracing::trace!(peer = %sender, "Firewall dropped inbound message");
- return Box::pin(async { Ok(()) });
- }
- handler(sender, message, payload_len)
- }))
- .await
- }
-
- async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> {
- self.inner.wait_for_exit().await
- }
-}
diff --git a/src/transport/identity.rs b/src/transport/identity.rs
new file mode 100644
index 0000000..94faded
--- /dev/null
+++ b/src/transport/identity.rs
@@ -0,0 +1,748 @@
+//! The identity layer: between the node and its transports, whose identities are
+//! addresses (Mycelium's public keys), so that the rest of the node deals in node IDs
+//! (`crate::identity`) only, whatever transport reaches a peer. A node may have several
+//! transports; a peer is reached on one both have, so nodes on several transports
+//! carry the traffic between the others.
+//!
+//! A node proves who it is to an address with WHO (which asks back) or ME (the answer):
+//! its public node key, and a signature over its own address and the recipient's
+//! (`signed`). The transport authenticates the sender's address, so a proof is good for
+//! that pair of addresses only: replayed from another address, to another recipient or
+//! on another transport, it fails. Once a node proved itself at an address, messages
+//! from there are its.
+//!
+//! * A message from an address no node proved itself at is dropped, and the address is
+//! asked WHO (at most every `ASK_INTERVAL`): after a restart, the first message from
+//! a peer brings its proof back, and the peer's node retries what was lost.
+//! * A seed (`resolve_seed`) is an address, or something one of the transports resolves
+//! to one; it is asked WHO, and its ME says which node is there.
+//! * Where other nodes are comes from the member list and HELLO_REPLYs
+//! (`learn_addresses`): hints, not proofs. A message to a node goes to the addresses
+//! it proved itself at, best first (`Paths::rank`, `Paths::send_first`: the next
+//! when a send fails); with only hints, those are asked WHO, and the send is refused
+//! until one answers (the node retries refused sends). A node proven on some
+//! transports is asked WHO at its hints on the others too, now and then
+//! (`OTHER_PATHS_ASK_INTERVAL`), so that each transport both have becomes a path.
+//!
+//! Addresses and the transports behind them are the paths layer's
+//! (`transport::paths`): the books keep addresses as paths keep them.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex, MutexGuard};
+use std::time::Duration;
+
+use anyhow::{Result, anyhow, bail};
+use async_trait::async_trait;
+use tokio::sync::oneshot;
+use tokio::time::Instant;
+use tracing::{debug, info};
+
+use crate::identity::{self, NodeKey};
+use crate::locks::MutexExt;
+use crate::transport::firewall::Firewall;
+use crate::transport::paths::{Gate, PathHandler, Paths, scheme_of};
+use crate::transport::stats::{IdentityCounts, IdentityStats, TransportStats};
+use crate::transport::traffic::Traffic;
+use crate::transport::{
+ Destination, EncodedMessage, HandlerFn, IdentityProof, MessageHandler, MessageKind,
+ ParsedMessage, Transport,
+};
+
+pub(crate) const WHO_LABEL: &[u8] = b"continuum/who/1";
+pub(crate) const ME_LABEL: &[u8] = b"continuum/me/1";
+/// An address is asked WHO at most this often, unless a seed is being resolved.
+const ASK_INTERVAL: Duration = Duration::from_secs(5);
+/// A node's addresses on transports it hasn't proven itself on are asked WHO at most
+/// this often (`Book::unproven_hints`): over Tor, a WHO is a dial that takes seconds,
+/// and an address that doesn't answer shouldn't be pressed.
+const OTHER_PATHS_ASK_INTERVAL: Duration = Duration::from_secs(60);
+/// How long resolving a seed waits for its ME.
+const RESOLVE_TIMEOUT: Duration = Duration::from_secs(10);
+/// Addresses kept per node from hints: about one per transport.
+const MAX_HINTS: usize = 8;
+/// Addresses with a proven node, nodes with hints, and addresses asked, each at most:
+/// anyone can send us proofs (keys are free), so the books are bounded.
+const MAX_ENTRIES: usize = 4096;
+
+/// A send refused because the peer hasn't proven where it is yet (`send_encoded`):
+/// expected when a peer known only from hints is first contacted, and retried.
+#[derive(Debug, thiserror::Error)]
+#[error("{0} hasn't proven where it is yet; asked where it is said to be")]
+pub(crate) struct NotProven(String);
+
+/// Whether `error` is (or was caused by) a [`NotProven`].
+pub(crate) fn is_not_proven(error: &anyhow::Error) -> bool {
+ error.chain().any(|cause| cause.is::<NotProven>())
+}
+
+/// What a node under the identity layer needs (`node::NodeConfig::identity`).
+#[derive(Clone)]
+pub(crate) struct IdentityConfig {
+ pub(crate) key: Arc<NodeKey>,
+ /// The address scheme of the node's transport (`NodeConfig::transport`), e.g.
+ /// `mycelium`.
+ pub(crate) scheme: &'static str,
+ /// Further transports, by scheme, after the first in preference.
+ pub(crate) more: Vec<(&'static str, Arc<dyn Transport>)>,
+}
+
+/// The bytes a WHO (`label`: `WHO_LABEL`) or ME proof signs: the label, then the
+/// sender's and the recipient's address, each with its length.
+pub(crate) fn signed(label: &[u8], from: &str, to: &str) -> Vec<u8> {
+ let mut bytes = label.to_vec();
+ for address in [from, to] {
+ bytes.extend_from_slice(&(address.len() as u32).to_be_bytes());
+ bytes.extend_from_slice(address.as_bytes());
+ }
+ bytes
+}
+
+pub(crate) struct IdentityTransport {
+ shared: Arc<Shared>,
+}
+
+struct Shared {
+ paths: Arc<Paths>,
+ /// The debug firewall, by node ID; paths applies it (`Gate`).
+ firewall: Option<Arc<Firewall>>,
+ key: Arc<NodeKey>,
+ node_id: String,
+ /// WHO and ME are counted here, under the address (`traffic`).
+ traffic: Arc<Traffic>,
+ // A std mutex: every access is short and never spans an await.
+ book: Mutex<Book>,
+}
+
+/// Who is where, by address.
+#[derive(Default)]
+struct Book {
+ /// The node that proved itself at an address, and when.
+ at: HashMap<String, (String, Instant)>,
+ /// The addresses each node last proved itself at, one per scheme.
+ proven: HashMap<String, Vec<String>>,
+ /// Where others say a node is.
+ hints: HashMap<String, Vec<String>>,
+ /// When each address was last asked WHO.
+ asked: HashMap<String, Instant>,
+ /// Seeds being resolved: who waits for a ME from where.
+ waiting: HashMap<String, Vec<oneshot::Sender<String>>>,
+ /// Since start (`stats`).
+ counts: IdentityCounts,
+}
+
+impl Book {
+ /// `node` proved itself at `address`. Returns whether that is news.
+ fn bind(&mut self, address: &str, node: &str, now: Instant) -> bool {
+ let news = self.at.get(address).is_none_or(|(known, _)| known != node);
+ // One address per node and scheme, and one node per address: a node that
+ // moved, or an address taken over by another node, leaves no stale entry.
+ let scheme = scheme_of(address);
+ let addresses = self.proven.entry(node.to_owned()).or_default();
+ if let Some(slot) = addresses.iter_mut().find(|at| scheme_of(at) == scheme) {
+ if *slot != address {
+ let previous = std::mem::replace(slot, address.to_owned());
+ self.at.remove(&previous);
+ self.counts.address_changes += 1;
+ }
+ } else {
+ addresses.push(address.to_owned());
+ }
+ if let Some((previous, _)) = self.at.insert(address.to_owned(), (node.to_owned(), now))
+ && previous != node
+ {
+ self.forget_proof(&previous, address);
+ }
+ if self.at.len() > MAX_ENTRIES
+ && let Some(oldest) = self
+ .at
+ .iter()
+ .min_by_key(|(_, (_, at))| *at)
+ .map(|(address, _)| address.clone())
+ && let Some((node, _)) = self.at.remove(&oldest)
+ {
+ self.forget_proof(&node, &oldest);
+ }
+ for waiter in self.waiting.remove(address).into_iter().flatten() {
+ let _ = waiter.send(node.to_owned());
+ }
+ news
+ }
+
+ fn forget_proof(&mut self, node: &str, address: &str) {
+ if let Some(addresses) = self.proven.get_mut(node) {
+ addresses.retain(|at| at != address);
+ if addresses.is_empty() {
+ self.proven.remove(node);
+ }
+ }
+ }
+
+ /// What the books hold now, and the counts.
+ fn stats(&self) -> IdentityStats {
+ let mut proven = std::collections::BTreeMap::new();
+ for addresses in self.proven.values() {
+ for address in addresses {
+ *proven.entry(scheme_of(address).to_owned()).or_insert(0) += 1;
+ }
+ }
+ IdentityStats {
+ proven,
+ hinted: self
+ .hints
+ .keys()
+ .filter(|node| !self.proven.contains_key(*node))
+ .count(),
+ counts: self.counts,
+ }
+ }
+
+ /// Where `node` is said to be on transports it hasn't proven itself on yet.
+ fn unproven_hints(&self, node: &str) -> Vec<String> {
+ let proven = self.proven.get(node);
+ self.hints
+ .get(node)
+ .into_iter()
+ .flatten()
+ .filter(|hint| {
+ proven.is_none_or(|proven| {
+ !proven
+ .iter()
+ .any(|address| scheme_of(address) == scheme_of(hint))
+ })
+ })
+ .cloned()
+ .collect()
+ }
+
+ /// Whether to ask `address` WHO now (always, if `force`), noting that it was asked.
+ fn ask(&mut self, address: &str, now: Instant, force: bool) -> bool {
+ let interval = if force { Duration::ZERO } else { ASK_INTERVAL };
+ self.ask_every(address, now, interval)
+ }
+
+ /// Whether to ask `address` WHO now, if it wasn't within `interval`, noting that it
+ /// was asked.
+ fn ask_every(&mut self, address: &str, now: Instant, interval: Duration) -> bool {
+ if self
+ .asked
+ .get(address)
+ .is_some_and(|at| now.duration_since(*at) < interval)
+ {
+ return false;
+ }
+ if self.asked.len() >= MAX_ENTRIES {
+ // The longest interval: other paths' asks are kept until they are due again.
+ self.asked
+ .retain(|_, at| now.duration_since(*at) < OTHER_PATHS_ASK_INTERVAL);
+ }
+ self.asked.insert(address.to_owned(), now);
+ true
+ }
+}
+
+impl IdentityTransport {
+ /// Over `transport`, whose addresses have `config.scheme`, and `config.more`.
+ /// `firewall`: the debug firewall, if any, which paths applies by address.
+ pub(crate) fn new(
+ transport: Arc<dyn Transport>,
+ config: IdentityConfig,
+ traffic: Arc<Traffic>,
+ firewall: Option<Arc<Firewall>>,
+ ) -> Self {
+ let node_id = config.key.id();
+ let shared = Arc::new_cyclic(|shared: &std::sync::Weak<Shared>| {
+ let gate: std::sync::Weak<dyn Gate> = shared.clone();
+ Shared {
+ paths: Arc::new(Paths::new(
+ std::iter::once((config.scheme, transport))
+ .chain(config.more)
+ .collect(),
+ Some(Arc::clone(&traffic)),
+ firewall.is_some().then_some(gate),
+ )),
+ firewall,
+ key: config.key,
+ node_id,
+ traffic,
+ book: Mutex::default(),
+ }
+ });
+ Self { shared }
+ }
+}
+
+/// The firewall's rules are by node ID: an address is blocked if the node proven at it
+/// is. Addresses no node proved itself at aren't (nothing could name them).
+impl Gate for Shared {
+ fn blocks_inbound(&self, address: &str) -> bool {
+ self.blocks(address, Firewall::blocks_inbound)
+ }
+
+ fn blocks_outbound(&self, address: &str) -> bool {
+ self.blocks(address, Firewall::blocks_outbound)
+ }
+}
+
+impl Shared {
+ fn blocks(&self, address: &str, rule: fn(&Firewall, &str) -> bool) -> bool {
+ let Some(firewall) = &self.firewall else {
+ return false;
+ };
+ let node = self.lock().at.get(address).map(|(node, _)| node.clone());
+ node.is_some_and(|node| rule(firewall, &node))
+ }
+}
+
+impl Shared {
+ fn lock(&self) -> MutexGuard<'_, Book> {
+ self.book.lock_or_recover()
+ }
+
+ /// Our proof for `address` (as paths keep it), from our address on its transport.
+ fn proof(&self, label: &[u8], address: &str) -> Option<IdentityProof> {
+ let message = signed(label, &self.paths.own_address(address)?, address);
+ Some(IdentityProof {
+ key: self.key.public(),
+ signature: self.key.sign(&message),
+ })
+ }
+
+ /// Notes the node `proof` proves to be at `address` (as paths keep it); None if it
+ /// proves nothing.
+ fn prove(&self, address: &str, label: &[u8], proof: &IdentityProof) -> Option<String> {
+ let message = signed(label, address, &self.paths.own_address(address)?);
+ if !identity::verify(&proof.key, &message, &proof.signature) {
+ self.lock().counts.proofs_rejected += 1;
+ debug!(from = %address, "Ignoring an identity proof that doesn't hold");
+ return None;
+ }
+ let node = identity::node_id_of(&proof.key);
+ if node == self.node_id {
+ debug!(from = %address, "Ignoring a proof of our own node key");
+ return None;
+ }
+ if self.lock().bind(address, &node, Instant::now()) {
+ debug!(node = %node, address = %address, "Node proved its address");
+ }
+ Some(node)
+ }
+
+ /// Asks `address` WHO, unless it was asked recently (`force`: anyway).
+ fn ask(self: &Arc<Self>, address: &str, force: bool) {
+ if !self.lock().ask(address, Instant::now(), force) {
+ return;
+ }
+ self.send_who(address);
+ }
+
+ /// Asks `node`'s addresses on transports it hasn't proven itself on WHO, now and then
+ /// (`OTHER_PATHS_ASK_INTERVAL`): its ME there proves another path to it, which the
+ /// paths layer then measures, and fails over to.
+ fn ask_other_paths(self: &Arc<Self>, node: &str) {
+ let due: Vec<String> = {
+ let mut book = self.lock();
+ let now = Instant::now();
+ book.unproven_hints(node)
+ .into_iter()
+ .filter(|address| book.ask_every(address, now, OTHER_PATHS_ASK_INTERVAL))
+ .collect()
+ };
+ for address in due {
+ self.send_who(&address);
+ }
+ }
+
+ fn send_who(self: &Arc<Self>, address: &str) {
+ let shared = Arc::clone(self);
+ let address = address.to_owned();
+ tokio::spawn(async move { shared.send_proof(MessageKind::Who, &address).await });
+ }
+
+ /// Sends a WHO or ME (`kind`) to `address`.
+ async fn send_proof(&self, kind: MessageKind, address: &str) {
+ let Some(address) = self.paths.canonical(address) else {
+ return;
+ };
+ let address = address.as_str();
+ let message = if kind == MessageKind::Who {
+ self.proof(WHO_LABEL, address).map(ParsedMessage::Who)
+ } else {
+ self.proof(ME_LABEL, address).map(ParsedMessage::Me)
+ };
+ let Some(message) = message else {
+ return;
+ };
+ let encoded = match self.paths.encode(address, &message) {
+ Ok(encoded) => encoded,
+ Err(error) => {
+ debug!(to = %address, error = %format!("{error:#}"), "Couldn't encode an identity proof");
+ return;
+ }
+ };
+ match self.paths.send(&encoded).await {
+ Ok(()) => self
+ .traffic
+ .sent(address, encoded.kind, encoded.payload_len),
+ Err(error) => {
+ self.traffic
+ .refused(address, encoded.kind, encoded.payload_len);
+ debug!(to = %address, error = %format!("{error:#}"), "Couldn't send an identity proof");
+ }
+ }
+ }
+
+ /// A message from `address`.
+ async fn receive(
+ self: Arc<Self>,
+ address: String,
+ message: ParsedMessage,
+ payload_len: usize,
+ handler: &HandlerFn,
+ ) -> Result<()> {
+ match message {
+ ParsedMessage::Who(proof) => {
+ self.traffic
+ .received(&address, MessageKind::Who, payload_len);
+ if self.prove(&address, WHO_LABEL, &proof).is_some() {
+ let shared = Arc::clone(&self);
+ tokio::spawn(async move { shared.send_proof(MessageKind::Me, &address).await });
+ }
+ Ok(())
+ }
+ ParsedMessage::Me(proof) => {
+ self.traffic
+ .received(&address, MessageKind::Me, payload_len);
+ self.prove(&address, ME_LABEL, &proof);
+ Ok(())
+ }
+ message => {
+ let node = self.lock().at.get(&address).map(|(node, _)| node.clone());
+ match node {
+ Some(node) => handler(node, message, payload_len).await,
+ None => {
+ self.lock().counts.dropped_unproven += 1;
+ debug!(
+ from = %address,
+ kind = ?message.kind(),
+ "Dropping a message from an address no node proved itself at; asking who is there"
+ );
+ self.ask(&address, false);
+ Ok(())
+ }
+ }
+ }
+ }
+ }
+
+ /// Where to send to `peer`: an address it proved itself at, else one it is said to
+ /// be at; the most preferred transport's first.
+ fn destination(&self, peer: &str) -> Option<String> {
+ self.proven_at(peer)
+ .into_iter()
+ .next()
+ .or_else(|| self.hinted_at(peer).into_iter().next())
+ }
+
+ /// The addresses `peer` proved itself at, best first (`Paths::rank`).
+ fn proven_at(&self, peer: &str) -> Vec<String> {
+ let proven = self.lock().proven.get(peer).cloned().unwrap_or_default();
+ self.paths.rank(&proven)
+ }
+
+ /// Where `peer` is said to be, best first.
+ fn hinted_at(&self, peer: &str) -> Vec<String> {
+ let hints = self.lock().hints.get(peer).cloned().unwrap_or_default();
+ self.paths.rank(&hints)
+ }
+}
+
+#[async_trait]
+impl Transport for IdentityTransport {
+ fn local_id(&self) -> &str {
+ &self.shared.node_id
+ }
+
+ fn is_peer_id(&self, id: &str) -> bool {
+ identity::is_node_id(id)
+ }
+
+ fn delivery_limit(&self) -> Duration {
+ self.shared.paths.delivery_limit()
+ }
+
+ fn local_addresses(&self) -> Vec<String> {
+ self.shared.paths.local_addresses()
+ }
+
+ /// Where `peer` proved itself, else where it is said to be, as others can use it.
+ fn addresses_of(&self, peer: &str) -> Vec<String> {
+ let addresses = {
+ let book = self.shared.lock();
+ book.proven
+ .get(peer)
+ .or_else(|| book.hints.get(peer))
+ .cloned()
+ .unwrap_or_default()
+ };
+ let mut locations = Vec::new();
+ for address in addresses {
+ for location in self.shared.paths.locations(&address) {
+ if !locations.contains(&location) {
+ locations.push(location);
+ }
+ }
+ }
+ locations
+ }
+
+ fn learn_addresses(&self, peer: &str, addresses: &[String]) {
+ if peer == self.shared.node_id || !identity::is_node_id(peer) {
+ return;
+ }
+ let mut known: Vec<String> = Vec::new();
+ for address in addresses {
+ if let Some(address) = self.shared.paths.canonical(address)
+ && !self.shared.paths.is_own(&address)
+ && !known.contains(&address)
+ {
+ known.push(address);
+ }
+ }
+ known.truncate(MAX_HINTS);
+ if known.is_empty() {
+ return;
+ }
+ let mut book = self.shared.lock();
+ if book.hints.len() < MAX_ENTRIES || book.hints.contains_key(peer) {
+ book.hints.insert(peer.to_owned(), known);
+ }
+ }
+
+ fn can_reach(&self, peer: &str) -> bool {
+ self.shared.destination(peer).is_some()
+ }
+
+ fn transports(&self) -> Vec<String> {
+ self.shared.paths.transports()
+ }
+
+ /// The address `encode` picks, the transport's connection there, and every path
+ /// to `peer` as paths measures it, best first.
+ fn destination(&self, peer: &str) -> Option<Destination> {
+ let proven = self.shared.proven_at(peer);
+ let (address, is_proven) = match proven.first() {
+ Some(address) => (address.clone(), true),
+ None => (self.shared.hinted_at(peer).into_iter().next()?, false),
+ };
+ let connection = self.shared.paths.connection(&address);
+ Some(Destination {
+ address,
+ proven: is_proven,
+ connection,
+ paths: self.shared.paths.info(&proven),
+ })
+ }
+
+ /// The paths layer answers them (`Paths::receive`).
+ fn answers_pings(&self) -> bool {
+ true
+ }
+
+ /// The best path's, where messages to `peer` go.
+ fn rtt(&self, peer: &str) -> Option<Duration> {
+ let address = self.shared.proven_at(peer).into_iter().next()?;
+ self.shared.paths.rtt(&address)
+ }
+
+ /// The identity layer's, and each transport's.
+ fn stats(&self) -> TransportStats {
+ let mut stats = self.shared.paths.stats();
+ stats.merge(TransportStats {
+ identity: Some(self.shared.lock().stats()),
+ ..TransportStats::default()
+ });
+ stats
+ }
+
+ fn discovered_seeds(&self) -> Vec<String> {
+ self.shared.paths.discovered_seeds()
+ }
+
+ /// A node ID resolves to itself if this node knows where it is; anything else is
+ /// an address (`<scheme>:...`) or what one of the transports resolves to one (the
+ /// first that does), asked WHO.
+ async fn resolve_seed(&self, seed: &str) -> Result<String> {
+ if identity::is_node_id(seed) {
+ if self.shared.destination(seed).is_some() {
+ return Ok(seed.to_owned());
+ }
+ bail!("a node ID alone doesn't say where that node is; give its address");
+ }
+ let address = self.shared.paths.resolve(seed).await?;
+ if self.shared.paths.is_own(&address) {
+ bail!("{seed} is this node");
+ }
+ let receiver = {
+ let mut book = self.shared.lock();
+ if let Some((node, _)) = book.at.get(&address) {
+ return Ok(node.clone());
+ }
+ let (sender, receiver) = oneshot::channel();
+ let waiting = book.waiting.entry(address.clone()).or_default();
+ waiting.retain(|waiter| !waiter.is_closed());
+ waiting.push(sender);
+ receiver
+ };
+ self.shared.ask(&address, true);
+ match tokio::time::timeout(RESOLVE_TIMEOUT, receiver).await {
+ Ok(Ok(node)) => {
+ info!(seed = %seed, node = %node, "Seed is node");
+ Ok(node)
+ }
+ _ => {
+ self.shared.lock().counts.seeds_unanswered += 1;
+ Err(anyhow!(
+ "no node answered at {address} within {RESOLVE_TIMEOUT:?}"
+ ))
+ }
+ }
+ }
+
+ /// For the address `destination` picks (a send there is refused until `peer`
+ /// proved itself there, `send_encoded`).
+ fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
+ let Some(address) = self.shared.destination(peer) else {
+ bail!("no address known for {peer} on this node's transports");
+ };
+ let mut encoded = self.shared.paths.encode(&address, message)?;
+ encoded.peer = peer.to_owned();
+ Ok(encoded)
+ }
+
+ /// To the addresses `peer` proved itself at, best first, the next when a send
+ /// fails (`Paths::send_first`); refused if it proved itself nowhere yet.
+ async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> {
+ let proven = self.shared.proven_at(&message.peer);
+ if !proven.is_empty() {
+ self.shared.ask_other_paths(&message.peer);
+ }
+ if proven.is_empty() {
+ let hints = {
+ let mut book = self.shared.lock();
+ book.counts.refused_unproven += 1;
+ book.hints.get(&message.peer).cloned().unwrap_or_default()
+ };
+ for address in hints {
+ self.shared.ask(&address, false);
+ }
+ return Err(NotProven(message.peer.clone()).into());
+ }
+ self.shared
+ .paths
+ .send_first(message, &proven)
+ .await
+ .map(|_| ())
+ }
+
+ /// Every transport's consumer (`Paths::start_consumer`); returns when the first
+ /// one does.
+ async fn start_consumer(&self, handler: MessageHandler) -> Result<()> {
+ let handler: Arc<HandlerFn> = Arc::from(handler);
+ let shared = Arc::clone(&self.shared);
+ let receive: Arc<PathHandler> = Arc::new(move |address, message, payload_len| {
+ let shared = Arc::clone(&shared);
+ let handler = Arc::clone(&handler);
+ Box::pin(async move {
+ shared
+ .receive(address, message, payload_len, &*handler)
+ .await
+ })
+ });
+ self.shared.paths.start_consumer(receive).await
+ }
+
+ /// When the first transport exits.
+ async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> {
+ self.shared.paths.wait_for_exit().await
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn signed_bytes_say_both_addresses() {
+ // A length before each address: "a:bc" to "d" isn't "a:b" to "cd".
+ assert_ne!(
+ signed(WHO_LABEL, "a:bc", "d"),
+ signed(WHO_LABEL, "a:b", "cd")
+ );
+ assert_ne!(signed(WHO_LABEL, "a", "b"), signed(ME_LABEL, "a", "b"));
+ assert_ne!(signed(WHO_LABEL, "a", "b"), signed(WHO_LABEL, "b", "a"));
+ }
+
+ #[test]
+ fn one_address_per_node_and_scheme_and_one_node_per_address() {
+ let now = Instant::now();
+ let mut book = Book::default();
+ assert!(book.bind("s:x", "n1", now));
+ assert!(!book.bind("s:x", "n1", now));
+ // n1 moved to s:y: s:x is no longer n1's.
+ assert!(book.bind("s:y", "n1", now));
+ assert!(!book.at.contains_key("s:x"));
+ assert_eq!(book.proven["n1"], ["s:y"]);
+ // On another transport too.
+ assert!(book.bind("t:x", "n1", now));
+ assert_eq!(book.proven["n1"], ["s:y", "t:x"]);
+ // n2 took over s:y: n1 is proven on t only.
+ assert!(book.bind("s:y", "n2", now));
+ assert_eq!(book.proven["n1"], ["t:x"]);
+ assert_eq!(book.at["s:y"].0, "n2");
+ // Only n1's move from s:x to s:y changed an address.
+ assert_eq!(book.counts.address_changes, 1);
+ let stats = book.stats();
+ assert_eq!(stats.proven["s"], 1);
+ assert_eq!(stats.proven["t"], 1);
+ }
+
+ #[test]
+ fn other_paths_are_the_hints_on_transports_not_proven() {
+ let now = Instant::now();
+ let mut book = Book::default();
+ book.hints
+ .insert("n1".into(), vec!["s:x".into(), "t:x".into(), "u:x".into()]);
+ // Proven nowhere: every hint.
+ assert_eq!(book.unproven_hints("n1"), ["s:x", "t:x", "u:x"]);
+ // Proven on s (even at another address): the others.
+ book.bind("s:y", "n1", now);
+ assert_eq!(book.unproven_hints("n1"), ["t:x", "u:x"]);
+ assert!(book.ask_every("t:x", now, OTHER_PATHS_ASK_INTERVAL));
+ assert!(!book.ask_every("t:x", now + ASK_INTERVAL, OTHER_PATHS_ASK_INTERVAL));
+ assert!(book.ask_every(
+ "t:x",
+ now + OTHER_PATHS_ASK_INTERVAL,
+ OTHER_PATHS_ASK_INTERVAL
+ ));
+ assert!(book.unproven_hints("n2").is_empty());
+ }
+
+ #[test]
+ fn asking_is_rate_limited() {
+ let now = Instant::now();
+ let mut book = Book::default();
+ assert!(book.ask("s:x", now, false));
+ assert!(!book.ask("s:x", now, false));
+ assert!(book.ask("s:x", now, true));
+ assert!(book.ask("s:x", now + ASK_INTERVAL, false));
+ }
+
+ #[tokio::test]
+ async fn a_waiting_seed_learns_the_node() {
+ let mut book = Book::default();
+ let (sender, receiver) = oneshot::channel();
+ book.waiting.insert("s:x".into(), vec![sender]);
+ book.bind("s:x", "n1", Instant::now());
+ assert_eq!(receiver.await.unwrap(), "n1");
+ }
+}
diff --git a/src/transport/libp2p.rs b/src/transport/libp2p.rs
new file mode 100644
index 0000000..54bfc76
--- /dev/null
+++ b/src/transport/libp2p.rs
@@ -0,0 +1,1970 @@
+//! libp2p as a transport (`--transports libp2p`, in builds with the `libp2p` feature):
+//! direct connections over TCP (Noise, Yamux) and QUIC, to addresses this node knows.
+//!
+//! A node's transport identity is its libp2p peer ID; where it is, its listen addresses
+//! (multiaddrs ending in the peer ID, `locations`). Seeds are such multiaddrs, or ones
+//! without the peer ID, which a connection finds out. Messages go one per stream of the
+//! `/continuum/1` protocol: their length (4 bytes, big endian), then the message as
+//! `wire` encodes it. A send is done once the stream is written and closed; a connection
+//! that can't be made, or a stream that can't be written, refuses it. A stream's protocol
+//! goes with its first data, without waiting for the other side to agree
+//! (`swarm_config`): opening a stream costs no round trip.
+//!
+//! Connected peers tell each other where they listen (libp2p's identify protocol), so a
+//! node learns the addresses of peers that dialled it. A node publishes where it can be
+//! reached (`local_addresses`): the addresses it announces (`--libp2p-announce`, e.g. a
+//! public address forwarded to it), or else those it listens on, loopback left out.
+//!
+//! A node nobody can dial (behind NAT, or a closed port) can still be reached through a
+//! relay: a node of the swarm that relays (`--libp2p-relay`), on which it reserves a slot
+//! (`--libp2p-relays`); its address through the relay (`<relay>/p2p-circuit/p2p/<it>`)
+//! is published with the others. Over a relayed connection, both sides then try to
+//! connect directly (libp2p's DCUtR, hole punching); the relay carries the traffic until
+//! that works, or for good if it doesn't, within its limits (`RELAY_CIRCUIT_DURATION`,
+//! `RELAY_CIRCUIT_BYTES`; a circuit that ends is made again on the next send).
+//!
+//! Of several connections to one peer (both dialled at once, or AutoNAT dialled back),
+//! all but one are closed a few seconds later (`DUPLICATE_GRACE`, `Book::duplicates`),
+//! by the side with the smaller peer ID.
+//!
+//! A relay says so where it publishes its addresses: next to each, the same address
+//! ending in `/p2p-circuit` (where to reserve a slot), which other nodes find in the
+//! member list. A node that needs a relay and has none configured picks up to
+//! `AUTO_RELAYS` of those (`relay_check`). It needs one when it has no address others
+//! could dial, or when AutoNAT says it can't be dialled: AutoNAT asks connected peers
+//! (nodes of the swarm) to dial it back. Once AutoNAT says it can be dialled, it lets
+//! the relays it picked go.
+//!
+//! With `--libp2p-mdns`, a node finds libp2p peers on its LAN (multicast DNS). Their
+//! LAN addresses are noted, and those that turn out to speak continuum (identify says
+//! so) are seeds while the node joins (`discovered_seeds`): nodes on one LAN find each
+//! other without configured seeds.
+//!
+//! It uses public libp2p infrastructure only with `--libp2p-public`: then it joins the
+//! IPFS DHT (Kademlia, through the IPFS bootstrap nodes), where it finds addresses of
+//! peers the member list doesn't have, and public peers that relay; those are picked
+//! after members that relay. Public relays keep circuits short (by default 2 minutes and
+//! 128 KiB): enough to punch holes through NATs, not to carry a swarm's traffic. Without
+//! it, relays are those configured, or members that say they relay.
+
+use std::collections::HashMap;
+use std::path::Path;
+use std::str::FromStr;
+use std::sync::{Arc, Mutex, MutexGuard};
+use std::time::Duration;
+
+use ::libp2p::multiaddr::Protocol;
+use ::libp2p::swarm::dial_opts::{DialOpts, PeerCondition};
+use ::libp2p::swarm::{ConnectionId, SwarmEvent};
+use ::libp2p::{Multiaddr, PeerId, StreamProtocol, Swarm, identity::Keypair};
+use anyhow::{Context, Result, anyhow, bail};
+use async_trait::async_trait;
+use futures::{AsyncReadExt, AsyncWriteExt, StreamExt};
+use tokio::sync::{Semaphore, mpsc, oneshot, watch};
+use tracing::{debug, info, trace, warn};
+
+use crate::locks::MutexExt;
+use crate::transport::stats::{
+ Connections, DialFailures, HolePunches, Libp2pStats, Reachability, RelaySlots, TransportStats,
+};
+use crate::transport::{
+ Connection, EncodedMessage, HandlerFn, MessageHandler, ParsedMessage, Transport, hand_on_probe,
+ is_probe, wire,
+};
+
+const PROTOCOL: StreamProtocol = StreamProtocol::new("/continuum/1");
+/// Sends taking this long are logged with where the time went (at debug).
+const SLOW_SEND: Duration = Duration::from_millis(5);
+/// The largest message read: anyone who can connect can send anything, so this bounds
+/// what one stream costs. Sync messages above 16 chunks travel as blobs.
+const MAX_MESSAGE: usize = 128 << 20;
+/// How long a connection may take to come up.
+const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
+/// A connection nobody uses is closed after this; neighbours keep theirs busy.
+const IDLE_TIMEOUT: Duration = Duration::from_secs(300);
+/// How long a stream may take to open and to carry a message, before what its size adds
+/// (`stream_timeout`).
+const STREAM_TIMEOUT: Duration = Duration::from_secs(10);
+/// The slowest a message is expected to go, in bytes per second.
+const MIN_RATE: u64 = 1 << 20;
+/// Inbound streams read at once.
+const READS: usize = 32;
+/// Addresses kept per peer, and peers with addresses at most.
+const ADDRESSES_PER_PEER: usize = 8;
+const MAX_PEERS: usize = 4096;
+
+/// A relayed connection (circuit) through this node lasts at most this long and carries
+/// at most this much: generous, since the swarm's own relays may carry sync traffic for
+/// good where hole punching fails (libp2p's defaults, 2 minutes and 128 KiB, are for
+/// the punching only).
+const RELAY_CIRCUIT_DURATION: Duration = Duration::from_secs(3600);
+const RELAY_CIRCUIT_BYTES: u64 = 1 << 30;
+/// Circuits through this node at once, and per peer.
+const RELAY_CIRCUITS: usize = 64;
+const RELAY_CIRCUITS_PER_PEER: usize = 8;
+/// A reservation on a relay that ended (the relay restarted, say) is made again after
+/// this.
+const RELAY_RETRY: Duration = Duration::from_secs(30);
+/// Relays picked among the members, at most, and how often a node checks whether it
+/// needs them.
+const AUTO_RELAYS: usize = 2;
+const RELAY_CHECK: Duration = Duration::from_secs(30);
+/// A relay that lost the slot before granting it (it doesn't answer, or another peer
+/// answers at its address: a relay restarted without its libp2p key) isn't picked again
+/// for this long.
+const RELAY_PAUSE: Duration = Duration::from_secs(600);
+/// A peer nobody mentioned for this long (the member list, identify, mDNS, a
+/// connection) is forgotten, unless connected: mostly old peer IDs of nodes restarted
+/// without their libp2p key, whose new ones the member list has since. Members are
+/// mentioned every few seconds (`swarm::topology`). Checked every `PRUNE_EVERY`.
+const STALE_AFTER: Duration = Duration::from_secs(600);
+const PRUNE_EVERY: Duration = Duration::from_secs(60);
+/// Several connections to one peer (both dialled at once, or AutoNAT dialled back to
+/// see whether it can) are cut to one (`Book::duplicates`) this long after the last one
+/// opened: long enough for what it was dialled for. Looked for every `DEDUPE_EVERY`.
+const DUPLICATE_GRACE: Duration = Duration::from_secs(10);
+const DEDUPE_EVERY: Duration = Duration::from_secs(5);
+
+/// How long a message of `bytes` may take on a stream.
+fn stream_timeout(bytes: usize) -> Duration {
+ STREAM_TIMEOUT + Duration::from_millis(bytes as u64 * 1000 / MIN_RATE)
+}
+
+/// What a libp2p transport is made with.
+pub(crate) struct Libp2pConfig {
+ pub(crate) key: Keypair,
+ /// Multiaddrs to listen on.
+ pub(crate) listen: Vec<Multiaddr>,
+ /// Multiaddrs to publish instead of the listen addresses, if any.
+ pub(crate) announce: Vec<Multiaddr>,
+ /// Relays to reserve a slot on (multiaddrs; without the relay's peer ID, a connection
+ /// finds it out).
+ pub(crate) relays: Vec<Multiaddr>,
+ /// Relay for other nodes.
+ pub(crate) relay: bool,
+ /// Find peers on the LAN (mDNS).
+ pub(crate) mdns: bool,
+ /// Use public infrastructure: the IPFS DHT, public relays.
+ pub(crate) public: bool,
+ /// libp2p's in-memory transport instead of TCP and QUIC: for tests.
+ pub(crate) memory: bool,
+}
+
+impl Libp2pConfig {
+ /// The key in `path` (32 raw bytes, an Ed25519 secret key), or a new one each start
+ /// without a file: the peer ID only says where a node is, so a new one costs nothing
+ /// but the WHO that finds it again (`transport::identity`).
+ pub(crate) fn load_key(path: Option<&Path>) -> Result<Keypair> {
+ let Some(path) = path else {
+ return Ok(Keypair::generate_ed25519());
+ };
+ let mut bytes = std::fs::read(path)
+ .with_context(|| format!("Reading the libp2p key from {}", path.display()))?;
+ crate::swarm::auth::warn_if_readable_by_others(path, "libp2p key");
+ if bytes.len() != 32 {
+ bail!(
+ "The libp2p key in {} has {} bytes; it has 32",
+ path.display(),
+ bytes.len()
+ );
+ }
+ Keypair::ed25519_from_bytes(&mut bytes)
+ .map_err(|error| anyhow!("The libp2p key in {}: {error}", path.display()))
+ }
+}
+
+enum Command {
+ /// Connect to `peer` (at `addresses`) unless connected.
+ Connect {
+ peer: PeerId,
+ addresses: Vec<Multiaddr>,
+ done: oneshot::Sender<Result<(), String>>,
+ },
+ /// Connect to `address` to find out which peer is there.
+ Identify {
+ address: Multiaddr,
+ done: oneshot::Sender<Result<PeerId, String>>,
+ },
+ /// See whether relays among the members are needed now (one was just learned of).
+ CheckRelays,
+ /// Reserve a slot on the relay at `relay` (again). `configured`: the address as
+ /// configured, for later attempts (without a peer ID, the relay may since have
+ /// another).
+ Reserve {
+ relay: Multiaddr,
+ configured: Multiaddr,
+ },
+}
+
+/// Where peers are, and where we listen.
+#[derive(Default)]
+struct Book {
+ addresses: HashMap<PeerId, Vec<Multiaddr>>,
+ listening: Vec<Multiaddr>,
+ /// `Libp2pConfig::announce`.
+ announced: Vec<Multiaddr>,
+ /// `Libp2pConfig::relay`: this node relays, and says so.
+ serving: bool,
+ /// Members that say they relay, and where (with the relay's peer ID).
+ relays: HashMap<PeerId, Multiaddr>,
+ /// With `Libp2pConfig::public`: other peers that relay (identify says so), and
+ /// where.
+ public_relays: HashMap<PeerId, Multiaddr>,
+ /// Relays picked that never granted a slot, and when (`RELAY_PAUSE`).
+ failed_relays: HashMap<PeerId, tokio::time::Instant>,
+ /// Relays that granted us a slot we hold.
+ slots: std::collections::HashSet<PeerId>,
+ /// Open connections: to whom, whether through a relay, and the remote address.
+ connections: HashMap<ConnectionId, (PeerId, bool, Multiaddr)>,
+ /// What AutoNAT says, and the address it confirmed while public.
+ reachability: Reachability,
+ public_address: Option<Multiaddr>,
+ /// Since start (`stats`).
+ counts: Counts,
+ /// When each peer was last mentioned (`STALE_AFTER`).
+ seen: HashMap<PeerId, tokio::time::Instant>,
+ /// Peers found on the LAN (mDNS) not known yet to speak continuum.
+ lan: std::collections::HashSet<PeerId>,
+ /// Peers that speak continuum (identify says so).
+ continuum: std::collections::HashSet<PeerId>,
+ /// Peers found on the LAN that speak continuum, with their address: seeds not
+ /// handed out yet (`discovered_seeds`).
+ discovered: Vec<Multiaddr>,
+}
+
+/// What `Libp2pStats` counts since start.
+#[derive(Default)]
+struct Counts {
+ slots_granted: u64,
+ slots_lost: u64,
+ slots_not_granted: u64,
+ hole_punches: HolePunches,
+ dial_failures: DialFailures,
+}
+
+/// Counts a dial that failed with `error` (not one that wasn't needed).
+fn count_dial_failure(failures: &mut DialFailures, error: &::libp2p::swarm::DialError) {
+ use ::libp2p::swarm::DialError;
+ match error {
+ DialError::DialPeerConditionFalse(_) => {}
+ DialError::Transport(_) => failures.unreachable += 1,
+ DialError::WrongPeerId { .. } => failures.wrong_peer_id += 1,
+ DialError::NoAddresses => failures.no_addresses += 1,
+ _ => failures.other += 1,
+ }
+}
+
+impl Book {
+ /// How we are connected to `peer`: directly if any connection is direct.
+ fn connection(&self, peer: &PeerId) -> Option<Connection> {
+ let mut found = None;
+ for (connected, relayed, _) in self.connections.values() {
+ if connected == peer {
+ if !relayed {
+ return Some(Connection::Direct);
+ }
+ found = Some(Connection::Relayed);
+ }
+ }
+ found
+ }
+
+ /// The connections to `peer` to close, of several: all but one, the oldest direct
+ /// one (the oldest relayed one if none is direct).
+ fn duplicates(&self, peer: &PeerId) -> Vec<ConnectionId> {
+ let mut connections: Vec<(bool, ConnectionId)> = self
+ .connections
+ .iter()
+ .filter(|(_, (connected, _, _))| connected == peer)
+ .map(|(id, (_, relayed, _))| (*relayed, *id))
+ .collect();
+ // Direct (false) first, then the oldest.
+ connections.sort();
+ connections.into_iter().skip(1).map(|(_, id)| id).collect()
+ }
+
+ /// The remote address of a connection to `peer`: a direct one if there is one.
+ fn location(&self, peer: &PeerId) -> Option<&Multiaddr> {
+ let mut found = None;
+ for (connected, relayed, address) in self.connections.values() {
+ if connected == peer {
+ if !relayed {
+ return Some(address);
+ }
+ found = Some(address);
+ }
+ }
+ found
+ }
+
+ fn stats(&self) -> Libp2pStats {
+ let mut connections = Connections::default();
+ for (_, relayed, _) in self.connections.values() {
+ if *relayed {
+ connections.relayed += 1;
+ } else {
+ connections.direct += 1;
+ }
+ }
+ Libp2pStats {
+ reachability: self.reachability,
+ public_address: self.public_address.as_ref().map(Multiaddr::to_string),
+ connections,
+ relay_slots: RelaySlots {
+ held: self.slots.len(),
+ paused: self
+ .failed_relays
+ .values()
+ .filter(|at| at.elapsed() < RELAY_PAUSE)
+ .count(),
+ granted: self.counts.slots_granted,
+ lost: self.counts.slots_lost,
+ not_granted: self.counts.slots_not_granted,
+ },
+ hole_punches: self.counts.hole_punches,
+ dial_failures: self.counts.dial_failures,
+ }
+ }
+
+ /// `peer` was mentioned just now.
+ fn touch(&mut self, peer: PeerId) {
+ self.seen.insert(peer, tokio::time::Instant::now());
+ }
+
+ /// Forgets the peers not mentioned within `STALE_AFTER` before `now`, but those
+ /// `connected`. Returns how many.
+ fn prune(&mut self, now: tokio::time::Instant, connected: impl Fn(&PeerId) -> bool) -> usize {
+ let stale: Vec<PeerId> = self
+ .addresses
+ .keys()
+ .chain(self.relays.keys())
+ .chain(self.public_relays.keys())
+ .chain(self.continuum.iter())
+ .chain(self.lan.iter())
+ .filter(|peer| {
+ !connected(peer)
+ && self
+ .seen
+ .get(peer)
+ .is_none_or(|at| now.saturating_duration_since(*at) >= STALE_AFTER)
+ })
+ .copied()
+ .collect::<std::collections::HashSet<_>>()
+ .into_iter()
+ .collect();
+ for peer in &stale {
+ self.addresses.remove(peer);
+ self.relays.remove(peer);
+ self.public_relays.remove(peer);
+ self.continuum.remove(peer);
+ self.lan.remove(peer);
+ self.failed_relays.remove(peer);
+ self.seen.remove(peer);
+ }
+ stale.len()
+ }
+
+ /// `peer`, found on the LAN, speaks continuum: a seed, at the address we know first.
+ fn discover(&mut self, peer: PeerId) {
+ let Some(address) = self.addresses.get(&peer).and_then(|known| known.first()) else {
+ return;
+ };
+ if let Ok(address) = address.clone().with_p2p(peer)
+ && !self.discovered.contains(&address)
+ && self.discovered.len() < MAX_PEERS
+ {
+ self.discovered.push(address);
+ }
+ }
+
+ /// Where others could dial us directly: the announced addresses, or else those we
+ /// listen on (`published`), without the peer ID.
+ fn direct(&self) -> Vec<Multiaddr> {
+ if !self.announced.is_empty() {
+ return self.announced.clone();
+ }
+ let direct: Vec<Multiaddr> = self
+ .listening
+ .iter()
+ .filter(|address| !is_relayed(address))
+ .cloned()
+ .collect();
+ published(&direct)
+ }
+}
+
+/// Whether `address` only reaches this host (or this link): no use to other hosts, and
+/// dialled by one, it reaches that host itself.
+fn is_local(address: &Multiaddr) -> bool {
+ match address.iter().next() {
+ Some(Protocol::Ip4(ip)) => ip.is_loopback() || ip.is_link_local() || ip.is_unspecified(),
+ Some(Protocol::Ip6(ip)) => {
+ ip.is_loopback() || ip.is_unicast_link_local() || ip.is_unspecified()
+ }
+ _ => false,
+ }
+}
+
+/// The addresses of `all` worth publishing: all but local ones, unless they're all
+/// local (a test on one host).
+fn published(all: &[Multiaddr]) -> Vec<Multiaddr> {
+ let reachable: Vec<Multiaddr> = all
+ .iter()
+ .filter(|address| !is_local(address))
+ .cloned()
+ .collect();
+ if reachable.is_empty() {
+ all.to_vec()
+ } else {
+ reachable
+ }
+}
+
+impl Book {
+ /// Whether all of `addresses` are known of `peer` (`remember`).
+ fn knows(&self, peer: PeerId, addresses: &[Multiaddr]) -> bool {
+ let Some(known) = self.addresses.get(&peer) else {
+ return false;
+ };
+ addresses.iter().all(|address| {
+ let mut address = address.clone();
+ if let Some(Protocol::P2p(_)) = address.iter().last() {
+ address.pop();
+ }
+ known.contains(&address)
+ })
+ }
+
+ /// `peer` is at `address` (without its `/p2p` part): kept first.
+ fn remember(&mut self, peer: PeerId, mut address: Multiaddr) {
+ if let Some(Protocol::P2p(_)) = address.iter().last() {
+ address.pop();
+ }
+ if !self.addresses.contains_key(&peer) && self.addresses.len() >= MAX_PEERS {
+ return;
+ }
+ self.touch(peer);
+ let addresses = self.addresses.entry(peer).or_default();
+ addresses.retain(|known| *known != address);
+ addresses.insert(0, address);
+ addresses.truncate(ADDRESSES_PER_PEER);
+ }
+}
+
+/// Whether `address` goes through a relay.
+fn is_relayed(address: &Multiaddr) -> bool {
+ address
+ .iter()
+ .any(|protocol| matches!(protocol, Protocol::P2pCircuit))
+}
+
+/// The relay `address` goes through, if it goes through one: the peer before
+/// `/p2p-circuit`.
+fn relay_of(address: &Multiaddr) -> Option<PeerId> {
+ let mut relay = None;
+ for protocol in address.iter() {
+ match protocol {
+ Protocol::P2p(peer) => relay = Some(peer),
+ Protocol::P2pCircuit => return relay,
+ _ => {}
+ }
+ }
+ None
+}
+
+/// The addresses of `known` this node can dial: not those through itself as the relay
+/// (a relay reaches the nodes it relays for only while they are connected to it).
+fn dialable(known: &[Multiaddr], local: PeerId) -> Vec<Multiaddr> {
+ known
+ .iter()
+ .filter(|address| relay_of(address) != Some(local))
+ .cloned()
+ .collect()
+}
+
+/// `address` with the peer ID it ends in, if it ends in one.
+fn split_peer(address: &Multiaddr) -> Option<PeerId> {
+ match address.iter().last() {
+ Some(Protocol::P2p(peer)) => Some(peer),
+ _ => None,
+ }
+}
+
+pub(crate) struct Libp2pTransport {
+ local_id: String,
+ peer_id: PeerId,
+ control: libp2p_stream::Control,
+ commands: mpsc::UnboundedSender<Command>,
+ book: Arc<Mutex<Book>>,
+ incoming: tokio::sync::Mutex<Option<libp2p_stream::IncomingStreams>>,
+ stopped: watch::Receiver<bool>,
+}
+
+fn lock(book: &Mutex<Book>) -> MutexGuard<'_, Book> {
+ book.lock_or_recover()
+}
+
+impl Libp2pTransport {
+ /// Builds the swarm, listens, and runs it in a task of its own.
+ pub(crate) fn start(config: Libp2pConfig) -> Result<Self> {
+ let peer_id = config.key.public().to_peer_id();
+ let features = Features::of(&config);
+ let mut swarm = if config.memory {
+ memory_swarm(config.key, features)?
+ } else {
+ network_swarm(config.key, features)?
+ };
+ for address in config.listen {
+ swarm
+ .listen_on(address.clone())
+ .with_context(|| format!("Listening on {address}"))?;
+ }
+ for address in &config.announce {
+ // Identify tells peers these.
+ swarm.add_external_address(address.clone());
+ }
+ let mut control = swarm.behaviour().stream.new_control();
+ let incoming = control
+ .accept(PROTOCOL)
+ .map_err(|error| anyhow!("Accepting {PROTOCOL}: {error}"))?;
+ let configured_relays = !config.relays.is_empty();
+ let book = Arc::new(Mutex::new(Book {
+ announced: config.announce,
+ serving: config.relay,
+ ..Book::default()
+ }));
+ let (commands, receiver) = mpsc::unbounded_channel();
+ for relay in config.relays {
+ let _ = commands.send(Command::Reserve {
+ configured: relay.clone(),
+ relay,
+ });
+ }
+ let (stopping, stopped) = watch::channel(false);
+ tokio::spawn(run(
+ swarm,
+ receiver,
+ commands.clone(),
+ Arc::clone(&book),
+ Relaying {
+ serving: config.relay,
+ configured: configured_relays,
+ public: config.public,
+ local: peer_id,
+ },
+ stopping,
+ ));
+ info!(peer_id = %peer_id, "Transport started");
+ Ok(Self {
+ local_id: peer_id.to_string(),
+ peer_id,
+ control,
+ commands,
+ book,
+ incoming: tokio::sync::Mutex::new(Some(incoming)),
+ stopped,
+ })
+ }
+
+ /// Connected to `peer`, if it can be had in time.
+ async fn connect(&self, peer: PeerId) -> Result<()> {
+ let addresses = lock(&self.book)
+ .addresses
+ .get(&peer)
+ .map(|known| dialable(known, self.peer_id))
+ .unwrap_or_default();
+ let (done, connected) = oneshot::channel();
+ self.commands
+ .send(Command::Connect {
+ peer,
+ addresses,
+ done,
+ })
+ .map_err(|_| anyhow!("the libp2p swarm stopped"))?;
+ match tokio::time::timeout(CONNECT_TIMEOUT, connected).await {
+ Ok(Ok(Ok(()))) => Ok(()),
+ Ok(Ok(Err(error))) => bail!("couldn't connect to {peer}: {error}"),
+ Ok(Err(_)) => bail!("the libp2p swarm stopped"),
+ Err(_) => bail!("connecting to {peer} took longer than {CONNECT_TIMEOUT:?}"),
+ }
+ }
+}
+
+/// The protocol version identify says (informational: nothing checks it).
+const IDENTIFY_VERSION: &str = "/continuum/1";
+
+#[derive(::libp2p::swarm::NetworkBehaviour)]
+struct Behaviour {
+ stream: libp2p_stream::Behaviour,
+ identify: ::libp2p::identify::Behaviour,
+ relay_client: ::libp2p::relay::client::Behaviour,
+ /// Only with `Libp2pConfig::relay`.
+ relay: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::relay::Behaviour>,
+ dcutr: ::libp2p::dcutr::Behaviour,
+ autonat: ::libp2p::autonat::Behaviour,
+ /// Only with `Libp2pConfig::mdns`.
+ mdns: ::libp2p::swarm::behaviour::toggle::Toggle<::libp2p::mdns::tokio::Behaviour>,
+ /// Only with `Libp2pConfig::public`: the IPFS DHT.
+ kad: ::libp2p::swarm::behaviour::toggle::Toggle<
+ ::libp2p::kad::Behaviour<::libp2p::kad::store::MemoryStore>,
+ >,
+}
+
+/// What the behaviour has besides what every node has.
+#[derive(Clone, Copy)]
+struct Features {
+ relay: bool,
+ mdns: bool,
+ public: bool,
+}
+
+impl Features {
+ fn of(config: &Libp2pConfig) -> Self {
+ Self {
+ relay: config.relay,
+ mdns: config.mdns,
+ public: config.public,
+ }
+ }
+}
+
+/// The IPFS DHT's protocol, and its bootstrap nodes (all at `IPFS_BOOTSTRAP_ADDRESS`).
+const IPFS_DHT: StreamProtocol = StreamProtocol::new("/ipfs/kad/1.0.0");
+const IPFS_BOOTSTRAP_ADDRESS: &str = "/dnsaddr/bootstrap.libp2p.io";
+const IPFS_BOOTSTRAP_PEERS: [&str; 4] = [
+ "QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
+ "QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
+ "QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
+ "QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
+];
+/// How often a node on the DHT refreshes its routing table.
+const DHT_BOOTSTRAP: Duration = Duration::from_secs(300);
+
+impl Behaviour {
+ fn new(
+ key: &Keypair,
+ relay_client: ::libp2p::relay::client::Behaviour,
+ features: Features,
+ ) -> std::io::Result<Self> {
+ let Features {
+ relay,
+ mdns,
+ public,
+ } = features;
+ let peer_id = key.public().to_peer_id();
+ let kad = public.then(|| {
+ let mut dht = ::libp2p::kad::Behaviour::with_config(
+ peer_id,
+ ::libp2p::kad::store::MemoryStore::new(peer_id),
+ ::libp2p::kad::Config::new(IPFS_DHT),
+ );
+ let bootstrap: Multiaddr = IPFS_BOOTSTRAP_ADDRESS.parse().expect("a multiaddr");
+ for peer in IPFS_BOOTSTRAP_PEERS {
+ let peer: PeerId = peer.parse().expect("a peer ID");
+ let _ = dht.add_address(&peer, bootstrap.clone());
+ }
+ dht
+ });
+ let identify = ::libp2p::identify::Config::new(IDENTIFY_VERSION.to_owned(), key.public())
+ .with_agent_version(format!("continuum/{}", env!("CARGO_PKG_VERSION")));
+ let relay = relay.then(|| {
+ ::libp2p::relay::Behaviour::new(
+ peer_id,
+ ::libp2p::relay::Config {
+ max_circuits: RELAY_CIRCUITS,
+ max_circuits_per_peer: RELAY_CIRCUITS_PER_PEER,
+ max_circuit_duration: RELAY_CIRCUIT_DURATION,
+ max_circuit_bytes: RELAY_CIRCUIT_BYTES,
+ ..Default::default()
+ },
+ )
+ });
+ let mdns = match mdns {
+ true => Some(::libp2p::mdns::tokio::Behaviour::new(
+ ::libp2p::mdns::Config::default(),
+ peer_id,
+ )?),
+ false => None,
+ };
+ Ok(Self {
+ stream: libp2p_stream::Behaviour::new(),
+ identify: ::libp2p::identify::Behaviour::new(identify),
+ relay_client,
+ relay: relay.into(),
+ dcutr: ::libp2p::dcutr::Behaviour::new(peer_id),
+ autonat: ::libp2p::autonat::Behaviour::new(
+ peer_id,
+ ::libp2p::autonat::Config {
+ // Swarms on private networks (a LAN, WireGuard) count too.
+ only_global_ips: false,
+ ..Default::default()
+ },
+ ),
+ mdns: mdns.into(),
+ kad: kad.into(),
+ })
+ }
+}
+
+fn network_swarm(key: Keypair, features: Features) -> Result<Swarm<Behaviour>> {
+ Ok(::libp2p::SwarmBuilder::with_existing_identity(key)
+ .with_tokio()
+ .with_tcp(
+ ::libp2p::tcp::Config::default(),
+ ::libp2p::noise::Config::new,
+ ::libp2p::yamux::Config::default,
+ )?
+ .with_quic()
+ .with_dns()?
+ .with_relay_client(
+ ::libp2p::noise::Config::new,
+ ::libp2p::yamux::Config::default,
+ )?
+ .with_behaviour(|key, relay_client| {
+ Behaviour::new(key, relay_client, features)
+ .map_err(|error| Box::new(error) as Box<dyn std::error::Error + Send + Sync>)
+ })?
+ .with_swarm_config(|config| swarm_config(config, features))
+ .build())
+}
+
+/// The swarm's settings. Streams are opened without waiting for the other side to agree
+/// on the protocol (`V1Lazy`): the protocol and the first data go together, saving a
+/// round trip per message (one stream each, module docs). That is safe only towards
+/// peers sure to speak the protocol, so not with public libp2p infrastructure
+/// (`Libp2pConfig::public`), whose nodes speak only some: there a protocol they don't
+/// would fail late, and unclearly.
+fn swarm_config(config: ::libp2p::swarm::Config, features: Features) -> ::libp2p::swarm::Config {
+ let config = config.with_idle_connection_timeout(IDLE_TIMEOUT);
+ if features.public {
+ config
+ } else {
+ config.with_substream_upgrade_protocol_override(::libp2p::core::upgrade::Version::V1Lazy)
+ }
+}
+
+fn memory_swarm(key: Keypair, features: Features) -> Result<Swarm<Behaviour>> {
+ use ::libp2p::Transport as _;
+ use ::libp2p::core::transport::{MemoryTransport, OrTransport};
+ use ::libp2p::core::upgrade::Version;
+
+ let peer_id = key.public().to_peer_id();
+ let (relay_transport, relay_client) = ::libp2p::relay::client::new(peer_id);
+ let transport = OrTransport::new(relay_transport, MemoryTransport::default())
+ .boxed()
+ .upgrade(Version::V1)
+ .authenticate(::libp2p::noise::Config::new(&key)?)
+ .multiplex(::libp2p::yamux::Config::default())
+ .timeout(CONNECT_TIMEOUT)
+ .boxed();
+ Ok(Swarm::new(
+ transport,
+ Behaviour::new(&key, relay_client, features)?,
+ peer_id,
+ swarm_config(::libp2p::swarm::Config::with_tokio_executor(), features),
+ ))
+}
+
+/// Drives the swarm: dials what `commands` ask for and notes where we listen, until the
+/// transport is dropped.
+/// How a node relates to relays.
+struct Relaying {
+ /// It relays for others.
+ serving: bool,
+ /// Relays to use are configured: it doesn't pick any.
+ configured: bool,
+ /// Public relays may be picked, after members.
+ public: bool,
+ local: PeerId,
+}
+
+/// The relays to pick among `offers`, if this node needs relays, keeping `using`: a
+/// node needs them when it has no address others could dial (`direct`), or AutoNAT says
+/// it can't be dialled; up to `AUTO_RELAYS`, in an order of its own (the lowest hash of
+/// both peer IDs), so that not every node picks the same.
+fn relay_check(
+ local: PeerId,
+ direct: bool,
+ nat: &::libp2p::autonat::NatStatus,
+ offers: &HashMap<PeerId, Multiaddr>,
+ using: &[PeerId],
+) -> Vec<(PeerId, Multiaddr)> {
+ use ::libp2p::autonat::NatStatus;
+ let needed = match nat {
+ NatStatus::Private => true,
+ NatStatus::Public(_) => false,
+ NatStatus::Unknown => !direct,
+ };
+ if !needed || using.len() >= AUTO_RELAYS {
+ return Vec::new();
+ }
+ let score = |relay: &PeerId| {
+ use std::hash::{Hash, Hasher};
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
+ (local, relay).hash(&mut hasher);
+ hasher.finish()
+ };
+ let mut candidates: Vec<(PeerId, Multiaddr)> = offers
+ .iter()
+ .filter(|(relay, _)| **relay != local && !using.contains(relay))
+ .map(|(relay, address)| (*relay, address.clone()))
+ .collect();
+ candidates.sort_by_key(|(relay, _)| score(relay));
+ candidates.truncate(AUTO_RELAYS - using.len());
+ candidates
+}
+
+/// Reserves slots on the relays `relay_check` picks among the members, and lets those
+/// picked go once AutoNAT says this node can be dialled.
+fn pick_relays(
+ swarm: &mut Swarm<Behaviour>,
+ book: &Mutex<Book>,
+ relaying: &Relaying,
+ nat: &::libp2p::autonat::NatStatus,
+ picked: &mut HashMap<::libp2p::core::transport::ListenerId, PeerId>,
+) {
+ if nat.is_public() && !picked.is_empty() {
+ info!("Reachable directly (AutoNAT); letting the relays picked go");
+ let mut book = lock(book);
+ for (listener, relay) in std::mem::take(picked) {
+ swarm.remove_listener(listener);
+ book.slots.remove(&relay);
+ }
+ return;
+ }
+ let (direct, offers, public) = {
+ let mut book = lock(book);
+ book.failed_relays
+ .retain(|_, at| at.elapsed() < RELAY_PAUSE);
+ let failed = book.failed_relays.clone();
+ let usable = |offers: &HashMap<PeerId, Multiaddr>| -> HashMap<PeerId, Multiaddr> {
+ offers
+ .iter()
+ .filter(|(relay, _)| !failed.contains_key(relay))
+ .map(|(relay, address)| (*relay, address.clone()))
+ .collect()
+ };
+ let public = if relaying.public {
+ usable(&book.public_relays)
+ } else {
+ HashMap::new()
+ };
+ (!book.direct().is_empty(), usable(&book.relays), public)
+ };
+ let mut using: Vec<PeerId> = picked.values().copied().collect();
+ // Members first, then public ones for what's left.
+ let mut wanted = relay_check(relaying.local, direct, nat, &offers, &using);
+ using.extend(wanted.iter().map(|(relay, _)| *relay));
+ wanted.extend(relay_check(relaying.local, direct, nat, &public, &using));
+ for (relay, address) in wanted {
+ match swarm.listen_on(address.clone().with(Protocol::P2pCircuit)) {
+ Ok(listener) => {
+ info!(relay = %relay, address = %address, "Picked a member as relay");
+ picked.insert(listener, relay);
+ }
+ Err(error) => {
+ debug!(relay = %relay, error = %error, "Couldn't use a member as relay");
+ }
+ }
+ }
+}
+
+/// Asks the DHT (with `--libp2p-public`) where `peer` is, for the next attempt.
+fn find_on_dht(swarm: &mut Swarm<Behaviour>, peer: PeerId) {
+ if let Some(dht) = swarm.behaviour_mut().kad.as_mut() {
+ dht.get_closest_peers(peer);
+ }
+}
+
+/// `again`: to schedule commands.
+async fn run(
+ mut swarm: Swarm<Behaviour>,
+ mut commands: mpsc::UnboundedReceiver<Command>,
+ again: mpsc::UnboundedSender<Command>,
+ book: Arc<Mutex<Book>>,
+ relaying: Relaying,
+ stopping: watch::Sender<bool>,
+) {
+ let serving = relaying.serving;
+ let mut nat = ::libp2p::autonat::NatStatus::Unknown;
+ // Relays picked among the members, by listener.
+ let mut picked: HashMap<::libp2p::core::transport::ListenerId, PeerId> = HashMap::new();
+ let mut check = tokio::time::interval(RELAY_CHECK);
+ check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ let mut dht_bootstrap = tokio::time::interval(DHT_BOOTSTRAP);
+ dht_bootstrap.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ // Relays we listen through, by listener: as configured, and their peer.
+ let mut reservations: HashMap<::libp2p::core::transport::ListenerId, (Multiaddr, PeerId)> =
+ HashMap::new();
+ let reserve_later = |configured: Multiaddr| {
+ let again = again.clone();
+ tokio::spawn(async move {
+ tokio::time::sleep(RELAY_RETRY).await;
+ let _ = again.send(Command::Reserve {
+ relay: configured.clone(),
+ configured,
+ });
+ });
+ };
+ let mut connecting: HashMap<PeerId, Vec<oneshot::Sender<Result<(), String>>>> = HashMap::new();
+ let mut identifying: HashMap<
+ ConnectionId,
+ (Multiaddr, oneshot::Sender<Result<PeerId, String>>),
+ > = HashMap::new();
+ let mut prune = tokio::time::interval(PRUNE_EVERY);
+ prune.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ // Peers that may have several connections, and when to cut them to one. Only the
+ // side with the smaller peer ID does that, so the two never close different ones.
+ let mut dedupe_due: HashMap<PeerId, tokio::time::Instant> = HashMap::new();
+ let mut dedupe = tokio::time::interval(DEDUPE_EVERY);
+ dedupe.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ loop {
+ tokio::select! {
+ _ = dedupe.tick(), if !dedupe_due.is_empty() => {
+ let now = tokio::time::Instant::now();
+ let due: Vec<PeerId> = dedupe_due
+ .iter()
+ .filter(|(_, at)| **at <= now)
+ .map(|(peer, _)| *peer)
+ .collect();
+ for peer in due {
+ dedupe_due.remove(&peer);
+ let duplicates = lock(&book).duplicates(&peer);
+ for connection in duplicates {
+ debug!(peer = %peer, connection = %connection, "Closing a duplicate connection");
+ swarm.close_connection(connection);
+ }
+ }
+ }
+ _ = prune.tick() => {
+ let forgotten = lock(&book).prune(tokio::time::Instant::now(), |peer| swarm.is_connected(peer));
+ if forgotten > 0 {
+ debug!(forgotten, "Forgot peers nobody mentions any more");
+ }
+ }
+ _ = dht_bootstrap.tick(), if relaying.public => {
+ if let Some(dht) = swarm.behaviour_mut().kad.as_mut()
+ && let Err(error) = dht.bootstrap()
+ {
+ warn!(error = %error, "Couldn't join the IPFS DHT");
+ }
+ }
+ _ = check.tick(), if !relaying.configured => {
+ pick_relays(&mut swarm, &book, &relaying, &nat, &mut picked);
+ }
+ command = commands.recv() => match command {
+ None => break,
+ Some(Command::CheckRelays) => {
+ if !relaying.configured {
+ pick_relays(&mut swarm, &book, &relaying, &nat, &mut picked);
+ }
+ }
+ Some(Command::Connect { peer, addresses, done }) => {
+ if swarm.is_connected(&peer) {
+ let _ = done.send(Ok(()));
+ continue;
+ }
+ let waiting = connecting.entry(peer).or_default();
+ waiting.push(done);
+ if waiting.len() > 1 {
+ continue; // Already dialing.
+ }
+ let mut opts = DialOpts::peer_id(peer)
+ .addresses(addresses)
+ .condition(PeerCondition::DisconnectedAndNotDialing);
+ if relaying.public {
+ // Addresses the DHT knows too.
+ opts = opts.extend_addresses_through_behaviour();
+ }
+ if let Err(error) = swarm.dial(opts.build()) {
+ count_dial_failure(&mut lock(&book).counts.dial_failures, &error);
+ for done in connecting.remove(&peer).into_iter().flatten() {
+ let _ = done.send(Err(error.to_string()));
+ }
+ find_on_dht(&mut swarm, peer);
+ }
+ }
+ Some(Command::Reserve { relay, configured }) if split_peer(&relay).is_none() => {
+ // Which peer relays there first.
+ let again = again.clone();
+ tokio::spawn(async move {
+ let (done, identified) = oneshot::channel();
+ if again
+ .send(Command::Identify { address: relay.clone(), done })
+ .is_err()
+ {
+ return;
+ }
+ match tokio::time::timeout(CONNECT_TIMEOUT, identified).await {
+ Ok(Ok(Ok(peer))) => {
+ if let Ok(relay) = relay.with_p2p(peer) {
+ let _ = again.send(Command::Reserve { relay, configured });
+ }
+ }
+ _ => {
+ warn!(relay = %relay, "Couldn't reach a relay; trying again later");
+ tokio::time::sleep(RELAY_RETRY).await;
+ let _ = again.send(Command::Reserve { relay, configured });
+ }
+ }
+ });
+ }
+ Some(Command::Reserve { relay, configured }) => {
+ let Some(relay_peer) = split_peer(&relay) else {
+ continue; // The arm above takes those.
+ };
+ match swarm.listen_on(relay.clone().with(Protocol::P2pCircuit)) {
+ Ok(listener) => {
+ debug!(relay = %relay, "Reserving a slot on a relay");
+ reservations.insert(listener, (configured, relay_peer));
+ }
+ Err(error) => {
+ warn!(relay = %relay, error = %error, "Couldn't use a relay; trying again later");
+ reserve_later(configured);
+ }
+ }
+ }
+ Some(Command::Identify { address, done }) => {
+ let opts = DialOpts::unknown_peer_id().address(address.clone()).build();
+ let id = opts.connection_id();
+ match swarm.dial(opts) {
+ Ok(()) => {
+ identifying.insert(id, (address, done));
+ }
+ Err(error) => {
+ let _ = done.send(Err(error.to_string()));
+ }
+ }
+ }
+ },
+ event = swarm.select_next_some() => match event {
+ SwarmEvent::ConnectionEstablished { peer_id, connection_id, endpoint, .. } => {
+ {
+ let mut book = lock(&book);
+ book.touch(peer_id);
+ let relayed = endpoint.is_relayed();
+ book.connections.insert(
+ connection_id,
+ (peer_id, relayed, endpoint.get_remote_address().clone()),
+ );
+ if relaying.local < peer_id
+ && book.connections.values().filter(|(connected, _, _)| *connected == peer_id).count() > 1
+ {
+ dedupe_due.insert(peer_id, tokio::time::Instant::now() + DUPLICATE_GRACE);
+ }
+ debug!(
+ peer = %peer_id,
+ connection = %connection_id,
+ dialled = endpoint.is_dialer(),
+ address = %endpoint.get_remote_address(),
+ relayed,
+ "Connection opened"
+ );
+ }
+ for done in connecting.remove(&peer_id).into_iter().flatten() {
+ let _ = done.send(Ok(()));
+ }
+ if let Some((address, done)) = identifying.remove(&connection_id) {
+ lock(&book).remember(peer_id, address);
+ let _ = done.send(Ok(peer_id));
+ }
+ }
+ SwarmEvent::ConnectionClosed { peer_id, connection_id, cause, .. } => {
+ lock(&book).connections.remove(&connection_id);
+ debug!(
+ peer = %peer_id,
+ connection = %connection_id,
+ cause = ?cause,
+ "Connection closed"
+ );
+ }
+ SwarmEvent::OutgoingConnectionError { connection_id, peer_id, error } => {
+ count_dial_failure(&mut lock(&book).counts.dial_failures, &error);
+ if let Some((_, done)) = identifying.remove(&connection_id) {
+ let _ = done.send(Err(error.to_string()));
+ }
+ if let Some(peer) = peer_id
+ && !swarm.is_connected(&peer)
+ {
+ if let Some(waiting) = connecting.remove(&peer) {
+ for done in waiting {
+ let _ = done.send(Err(error.to_string()));
+ }
+ find_on_dht(&mut swarm, peer);
+ }
+ }
+ }
+ SwarmEvent::NewListenAddr { address, .. } => {
+ info!(address = %address, "Listening");
+ // A relay tells those reserving where it is: its external addresses.
+ if serving && !is_local(&address) && !is_relayed(&address) {
+ swarm.add_external_address(address.clone());
+ }
+ let mut book = lock(&book);
+ if !book.listening.contains(&address) {
+ book.listening.push(address);
+ }
+ }
+ SwarmEvent::ExpiredListenAddr { address, .. } => {
+ info!(address = %address, "No longer listening");
+ if serving {
+ swarm.remove_external_address(&address);
+ }
+ lock(&book).listening.retain(|known| *known != address);
+ }
+ SwarmEvent::ListenerClosed { listener_id, addresses, reason } => {
+ lock(&book)
+ .listening
+ .retain(|known| !addresses.contains(known));
+ if let Some((relay, relay_peer)) = reservations.remove(&listener_id) {
+ {
+ let mut book = lock(&book);
+ if book.slots.remove(&relay_peer) {
+ book.counts.slots_lost += 1;
+ } else {
+ book.counts.slots_not_granted += 1;
+ }
+ }
+ warn!(relay = %relay, reason = ?reason, "Lost the slot on a relay; reserving again later");
+ reserve_later(relay);
+ }
+ // A relay picked among the members: the next check picks again, another
+ // one if this one never granted the slot.
+ if let Some(relay) = picked.remove(&listener_id) {
+ let lost = {
+ let mut book = lock(&book);
+ let lost = book.slots.remove(&relay);
+ if lost {
+ book.counts.slots_lost += 1;
+ } else {
+ book.counts.slots_not_granted += 1;
+ book.failed_relays.insert(relay, tokio::time::Instant::now());
+ }
+ lost
+ };
+ if lost {
+ debug!(relay = %relay, reason = ?reason, "Lost the slot on a member relaying");
+ } else {
+ info!(
+ relay = %relay,
+ pause = ?RELAY_PAUSE,
+ "A relay didn't grant a slot (unreachable, or another peer at its address); picking others for a while"
+ );
+ }
+ }
+ }
+ SwarmEvent::ListenerError { error, .. } => {
+ warn!(error = %error, "Listener failed");
+ }
+ SwarmEvent::Behaviour(BehaviourEvent::RelayClient(
+ ::libp2p::relay::client::Event::ReservationReqAccepted { relay_peer_id, renewal, .. },
+ )) => {
+ {
+ let mut book = lock(&book);
+ book.slots.insert(relay_peer_id);
+ book.failed_relays.remove(&relay_peer_id);
+ if !renewal {
+ book.counts.slots_granted += 1;
+ }
+ }
+ if !renewal {
+ info!(relay = %relay_peer_id, "Reachable through a relay");
+ }
+ }
+ SwarmEvent::Behaviour(BehaviourEvent::Autonat(
+ ::libp2p::autonat::Event::StatusChanged { old, new },
+ )) => {
+ info!(old = ?old, new = ?new, "Reachability (AutoNAT)");
+ {
+ use ::libp2p::autonat::NatStatus;
+ let mut book = lock(&book);
+ (book.reachability, book.public_address) = match &new {
+ NatStatus::Unknown => (Reachability::Unknown, None),
+ NatStatus::Private => (Reachability::Private, None),
+ NatStatus::Public(address) => {
+ (Reachability::Public, Some(address.clone()))
+ }
+ };
+ }
+ nat = new;
+ }
+ SwarmEvent::Behaviour(BehaviourEvent::Relay(event)) => {
+ debug!(event = ?event, "Relay event");
+ }
+ SwarmEvent::Behaviour(BehaviourEvent::Dcutr(event)) => match event.result {
+ Ok(_) => {
+ lock(&book).counts.hole_punches.succeeded += 1;
+ info!(peer = %event.remote_peer_id, "Connected directly to a peer behind a relay");
+ }
+ Err(error) => {
+ lock(&book).counts.hole_punches.failed += 1;
+ debug!(peer = %event.remote_peer_id, error = %error, "Couldn't connect directly to a peer behind a relay; the relay carries on");
+ }
+ },
+ SwarmEvent::Behaviour(BehaviourEvent::Identify(
+ ::libp2p::identify::Event::Received { connection_id, peer_id, info },
+ )) => {
+ let mut book = lock(&book);
+ let listen = published(&info.listen_addrs);
+ // Every new connection identifies again: logged when it says something new.
+ if book.knows(peer_id, &listen) {
+ trace!(peer = %peer_id, connection = %connection_id, "Peer identified again");
+ } else {
+ debug!(
+ peer = %peer_id,
+ connection = %connection_id,
+ listen = ?info.listen_addrs,
+ observed = %info.observed_addr,
+ agent = %info.agent_version,
+ "Peer identified"
+ );
+ }
+ // Last first: `remember` puts each in front.
+ for address in listen.into_iter().rev() {
+ book.remember(peer_id, address);
+ }
+ if relaying.public
+ && info.protocols.contains(&::libp2p::relay::HOP_PROTOCOL_NAME)
+ && (book.public_relays.len() < MAX_PEERS
+ || book.public_relays.contains_key(&peer_id))
+ && let Some(address) = book
+ .addresses
+ .get(&peer_id)
+ .and_then(|known| known.first())
+ .and_then(|address| address.clone().with_p2p(peer_id).ok())
+ {
+ book.public_relays.insert(peer_id, address);
+ }
+ if info.protocols.contains(&PROTOCOL) {
+ book.continuum.insert(peer_id);
+ if book.lan.remove(&peer_id) {
+ book.discover(peer_id);
+ }
+ }
+ }
+ SwarmEvent::Behaviour(BehaviourEvent::Mdns(
+ ::libp2p::mdns::Event::Discovered(found),
+ )) => {
+ for (peer, address) in found {
+ if peer == relaying.local {
+ continue;
+ }
+ debug!(peer = %peer, address = %address, "Peer on the LAN");
+ let continuum = {
+ let mut book = lock(&book);
+ book.remember(peer, address.clone());
+ book.continuum.contains(&peer)
+ };
+ if continuum {
+ lock(&book).discover(peer);
+ } else if lock(&book).lan.len() < MAX_PEERS {
+ // Connected, identify says whether it speaks continuum.
+ lock(&book).lan.insert(peer);
+ let _ = swarm.dial(
+ DialOpts::peer_id(peer)
+ .addresses(vec![address])
+ .condition(PeerCondition::DisconnectedAndNotDialing)
+ .build(),
+ );
+ }
+ }
+ }
+ _ => {}
+ },
+ }
+ }
+ let _ = stopping.send(true);
+}
+
+/// One message from an inbound stream.
+async fn read_message(mut stream: ::libp2p::Stream) -> Result<Vec<u8>> {
+ let mut length = [0u8; 4];
+ tokio::time::timeout(STREAM_TIMEOUT, stream.read_exact(&mut length))
+ .await
+ .map_err(|_| anyhow!("no length within {STREAM_TIMEOUT:?}"))??;
+ let length = u32::from_be_bytes(length) as usize;
+ if length > MAX_MESSAGE {
+ bail!("a message of {length} bytes; at most {MAX_MESSAGE} are read");
+ }
+ let mut body = vec![0u8; length];
+ tokio::time::timeout(stream_timeout(length), stream.read_exact(&mut body))
+ .await
+ .map_err(|_| {
+ anyhow!(
+ "{length} bytes took longer than {:?}",
+ stream_timeout(length)
+ )
+ })??;
+ Ok(body)
+}
+
+#[async_trait]
+impl Transport for Libp2pTransport {
+ fn local_id(&self) -> &str {
+ &self.local_id
+ }
+
+ fn is_peer_id(&self, id: &str) -> bool {
+ PeerId::from_str(id).is_ok()
+ }
+
+ /// The longest a message may take on a stream.
+ fn delivery_limit(&self) -> Duration {
+ stream_timeout(MAX_MESSAGE)
+ }
+
+ /// Where we can be reached, each with our peer ID: the announced addresses, or else
+ /// those we listen on (`published`), and those through relays. A relay adds each
+ /// direct one again, ending in `/p2p-circuit`: where to reserve a slot.
+ fn local_addresses(&self) -> Vec<String> {
+ let book = lock(&self.book);
+ let direct: Vec<Multiaddr> = book
+ .direct()
+ .into_iter()
+ .filter_map(|address| address.with_p2p(self.peer_id).ok())
+ .collect();
+ let relayed = book
+ .listening
+ .iter()
+ .filter(|address| is_relayed(address))
+ .filter_map(|address| address.clone().with_p2p(self.peer_id).ok());
+ let offers: Vec<Multiaddr> = if book.serving {
+ direct
+ .iter()
+ .map(|address| address.clone().with(Protocol::P2pCircuit))
+ .collect()
+ } else {
+ Vec::new()
+ };
+ direct
+ .into_iter()
+ .chain(relayed)
+ .chain(offers)
+ .map(|address| address.to_string())
+ .collect()
+ }
+
+ /// A peer ID, or a multiaddr ending in one (noted as where that peer is).
+ fn locate(&self, target: &str) -> Option<String> {
+ if let Ok(peer) = PeerId::from_str(target) {
+ return Some(peer.to_string());
+ }
+ let mut address = Multiaddr::from_str(target).ok()?;
+ // A relay saying where to reserve a slot: `<address>/p2p/<relay>/p2p-circuit`.
+ if let Some(Protocol::P2pCircuit) = address.iter().last() {
+ address.pop();
+ let relay = split_peer(&address)?;
+ if relay != self.peer_id {
+ let new = {
+ let mut book = lock(&self.book);
+ let new = book.relays.get(&relay) != Some(&address);
+ if new && (book.relays.len() < MAX_PEERS || book.relays.contains_key(&relay)) {
+ book.relays.insert(relay, address.clone());
+ }
+ book.remember(relay, address);
+ new
+ };
+ if new {
+ let _ = self.commands.send(Command::CheckRelays);
+ }
+ }
+ return Some(relay.to_string());
+ }
+ let peer = split_peer(&address)?;
+ if peer != self.peer_id {
+ lock(&self.book).remember(peer, address);
+ }
+ Some(peer.to_string())
+ }
+
+ fn connection(&self, target: &str) -> Option<Connection> {
+ let peer = PeerId::from_str(target).ok()?;
+ lock(&self.book).connection(&peer)
+ }
+
+ fn location(&self, target: &str) -> Option<String> {
+ let peer = PeerId::from_str(target).ok()?;
+ let address = lock(&self.book).location(&peer)?.clone();
+ Some(
+ address
+ .with_p2p(peer)
+ .unwrap_or_else(|address| address)
+ .to_string(),
+ )
+ }
+
+ fn stats(&self) -> TransportStats {
+ TransportStats {
+ libp2p: Some(lock(&self.book).stats()),
+ ..TransportStats::default()
+ }
+ }
+
+ /// Nodes found on the LAN (mDNS) that speak continuum.
+ fn discovered_seeds(&self) -> Vec<String> {
+ std::mem::take(&mut lock(&self.book).discovered)
+ .iter()
+ .map(Multiaddr::to_string)
+ .collect()
+ }
+
+ /// Where `target` is, each address with its peer ID.
+ fn locations(&self, target: &str) -> Vec<String> {
+ let Ok(peer) = PeerId::from_str(target) else {
+ return Vec::new();
+ };
+ lock(&self.book)
+ .addresses
+ .get(&peer)
+ .into_iter()
+ .flatten()
+ .filter_map(|address| address.clone().with_p2p(peer).ok())
+ .map(|address| address.to_string())
+ .collect()
+ }
+
+ /// A multiaddr ending in a peer ID is that peer; one without is connected to, to
+ /// find out. A bare peer ID resolves if this transport knows where it is.
+ async fn resolve_seed(&self, seed: &str) -> Result<String> {
+ if let Ok(peer) = PeerId::from_str(seed) {
+ if lock(&self.book).addresses.contains_key(&peer) {
+ return Ok(peer.to_string());
+ }
+ bail!("no address known for libp2p peer {peer}; give a multiaddr");
+ }
+ let address =
+ Multiaddr::from_str(seed).with_context(|| format!("{seed} is no multiaddr"))?;
+ if let Some(peer) = self.locate(seed) {
+ return Ok(peer);
+ }
+ let (done, identified) = oneshot::channel();
+ self.commands
+ .send(Command::Identify { address, done })
+ .map_err(|_| anyhow!("the libp2p swarm stopped"))?;
+ match tokio::time::timeout(CONNECT_TIMEOUT, identified).await {
+ Ok(Ok(Ok(peer))) => Ok(peer.to_string()),
+ Ok(Ok(Err(error))) => bail!("couldn't connect to {seed}: {error}"),
+ Ok(Err(_)) => bail!("the libp2p swarm stopped"),
+ Err(_) => bail!("connecting to {seed} took longer than {CONNECT_TIMEOUT:?}"),
+ }
+ }
+
+ fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
+ let body = wire::encode(message)?;
+ Ok(EncodedMessage {
+ peer: peer.to_owned(),
+ address: peer.to_owned(),
+ payload_len: body.len(),
+ body: body.into(),
+ kind: message.kind(),
+ message: None,
+ })
+ }
+
+ async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> {
+ let peer = PeerId::from_str(&message.peer)
+ .with_context(|| format!("{} is no libp2p peer ID", message.peer))?;
+ let started = std::time::Instant::now();
+ self.connect(peer).await?;
+ let connected = started.elapsed();
+ let mut control = self.control.clone();
+ let body = message.body.clone();
+ let write = async move {
+ let mut stream = control.open_stream(peer, PROTOCOL).await?;
+ let opened = started.elapsed();
+ stream.write_all(&(body.len() as u32).to_be_bytes()).await?;
+ stream.write_all(&body).await?;
+ stream.close().await?;
+ Ok::<_, anyhow::Error>(opened)
+ };
+ let timeout = stream_timeout(message.body.len());
+ let opened = tokio::time::timeout(timeout, write)
+ .await
+ .map_err(|_| anyhow!("sending to {peer} took longer than {timeout:?}"))?
+ .with_context(|| format!("Sending to {peer}"))?;
+ let took = started.elapsed();
+ if took >= SLOW_SEND {
+ // Where a send's time goes: the swarm task (connecting), the stream's protocol
+ // negotiation (opening), or the connection (writing).
+ debug!(
+ peer = %peer,
+ kind = ?message.kind,
+ bytes = message.body.len(),
+ connect_ms = connected.as_secs_f64() * 1000.0,
+ open_ms = (opened - connected).as_secs_f64() * 1000.0,
+ write_ms = (took - opened).as_secs_f64() * 1000.0,
+ "Slow send"
+ );
+ }
+ Ok(())
+ }
+
+ /// Reads inbound streams (`READS` at once) and handles their messages one at a
+ /// time, in the order they were read; PINGs and PONGs at once (`is_probe`).
+ async fn start_consumer(&self, handler: MessageHandler) -> Result<()> {
+ let Some(mut incoming) = self.incoming.lock().await.take() else {
+ bail!("libp2p consumer already started");
+ };
+ let handler: Arc<HandlerFn> = Arc::from(handler);
+ let (sender, mut received) = mpsc::channel::<(PeerId, ParsedMessage, usize)>(READS);
+ let reads = Arc::new(Semaphore::new(READS));
+ let probes = Arc::clone(&handler);
+ tokio::spawn(async move {
+ while let Some((peer, stream)) = incoming.next().await {
+ let Ok(permit) = Arc::clone(&reads).acquire_owned().await else {
+ break;
+ };
+ let sender = sender.clone();
+ let probes = Arc::clone(&probes);
+ tokio::spawn(async move {
+ match read_message(stream).await {
+ Ok(body) => match wire::decode(&body) {
+ Ok(message) if is_probe(&message) => {
+ hand_on_probe(&probes, peer.to_string(), message, body.len());
+ }
+ Ok(message) => {
+ let _ = sender.send((peer, message, body.len())).await;
+ }
+ Err(error) => {
+ debug!(peer = %peer, error = %error, "Dropping a message that doesn't decode");
+ }
+ },
+ Err(error) => {
+ debug!(peer = %peer, error = %format!("{error:#}"), "Dropping an unreadable stream");
+ }
+ }
+ drop(permit);
+ });
+ }
+ });
+ while let Some((peer, message, payload_len)) = received.recv().await {
+ if let Err(error) = handler(peer.to_string(), message, payload_len).await {
+ warn!(peer = %peer, error = %format!("{error:#}"), "Failed to process inbound message");
+ }
+ }
+ Ok(())
+ }
+
+ async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> {
+ let mut stopped = self.stopped.clone();
+ let _ = stopped.wait_for(|stopped| *stopped).await;
+ Err(std::io::Error::other("the libp2p swarm stopped"))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::atomic::{AtomicU64, Ordering};
+
+ use super::*;
+ use crate::document::replica::Epoch;
+ use crate::transport::{HeadsSummary, Status};
+
+ /// Memory ports are process-wide.
+ static NEXT_PORT: AtomicU64 = AtomicU64::new(40_000);
+
+ /// A node on the in-memory transport, listening on a port of its own (or nowhere),
+ /// and its address with its peer ID (or just its peer ID).
+ fn memory_node_with(
+ listen: bool,
+ relays: Vec<Multiaddr>,
+ relay: bool,
+ ) -> (Libp2pTransport, Multiaddr) {
+ let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed);
+ let key = Keypair::generate_ed25519();
+ let address: Multiaddr = format!("/memory/{port}").parse().unwrap();
+ let peer = key.public().to_peer_id();
+ let transport = Libp2pTransport::start(Libp2pConfig {
+ key,
+ listen: if listen {
+ vec![address.clone()]
+ } else {
+ Vec::new()
+ },
+ announce: Vec::new(),
+ relays,
+ relay,
+ mdns: false,
+ public: false,
+ memory: true,
+ })
+ .unwrap();
+ let address = if listen {
+ address.with_p2p(peer).unwrap()
+ } else {
+ Multiaddr::empty().with(Protocol::P2p(peer))
+ };
+ (transport, address)
+ }
+
+ fn memory_node() -> (Libp2pTransport, Multiaddr) {
+ memory_node_with(true, Vec::new(), false)
+ }
+
+ fn status() -> ParsedMessage {
+ ParsedMessage::Status(Status {
+ heads: HeadsSummary::of(Vec::new()),
+ epoch: Epoch::GENESIS,
+ epoch_seq: 3,
+ ready: true,
+ syncs_with_you: false,
+ })
+ }
+
+ /// Starts `transport`'s consumer, forwarding what it receives.
+ fn received(
+ transport: Arc<Libp2pTransport>,
+ ) -> mpsc::UnboundedReceiver<(String, ParsedMessage)> {
+ let (sender, receiver) = mpsc::unbounded_channel();
+ tokio::spawn(async move {
+ transport
+ .start_consumer(Box::new(move |from, message, _| {
+ let sender = sender.clone();
+ Box::pin(async move {
+ let _ = sender.send((from, message));
+ Ok(())
+ })
+ }))
+ .await
+ });
+ receiver
+ }
+
+ #[tokio::test]
+ async fn messages_both_ways_over_one_connection() {
+ let (a, a_address) = memory_node();
+ let (b, _) = memory_node();
+ let (a, b) = (Arc::new(a), Arc::new(b));
+ let mut at_a = received(Arc::clone(&a));
+ let mut at_b = received(Arc::clone(&b));
+
+ assert_eq!(
+ b.resolve_seed(&a_address.to_string()).await.unwrap(),
+ a.local_id()
+ );
+ b.send_message(a.local_id(), &status()).await.unwrap();
+ let (from, message) = at_a.recv().await.unwrap();
+ assert_eq!(from, b.local_id());
+ assert!(matches!(message, ParsedMessage::Status(status) if status.epoch_seq == 3));
+
+ // a knows no address of b, but b's connection is there.
+ a.send_message(b.local_id(), &status()).await.unwrap();
+ let (from, _) = at_b.recv().await.unwrap();
+ assert_eq!(from, a.local_id());
+
+ assert_eq!(a.connection(b.local_id()), Some(Connection::Direct));
+ assert_eq!(b.connection(a.local_id()), Some(Connection::Direct));
+ let stats = a.stats().libp2p.unwrap();
+ assert!(stats.connections.direct >= 1);
+ assert_eq!(stats.connections.relayed, 0);
+ }
+
+ #[tokio::test]
+ async fn a_seed_without_a_peer_id_is_connected_to_to_find_out() {
+ let (a, a_address) = memory_node();
+ let (b, _) = memory_node();
+ let mut bare = a_address.clone();
+ bare.pop();
+ assert_eq!(
+ b.resolve_seed(&bare.to_string()).await.unwrap(),
+ a.local_id()
+ );
+ // And b now knows where a is.
+ assert_eq!(b.locations(a.local_id()), [a_address.to_string()]);
+ }
+
+ #[tokio::test]
+ async fn listen_addresses_carry_the_peer_id() {
+ let (a, a_address) = memory_node();
+ let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
+ while a.local_addresses().is_empty() && tokio::time::Instant::now() < deadline {
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ assert_eq!(a.local_addresses(), [a_address.to_string()]);
+ assert_eq!(
+ a.locate(&a_address.to_string()).as_deref(),
+ Some(a.local_id())
+ );
+ }
+
+ /// Waits up to 5 s for `condition`.
+ async fn eventually(mut condition: impl FnMut() -> bool) -> bool {
+ let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
+ while !condition() {
+ if tokio::time::Instant::now() >= deadline {
+ return false;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ true
+ }
+
+ /// A node learns where a peer that dialled it listens (identify), and can then
+ /// pass it on.
+ #[tokio::test]
+ async fn a_dialled_node_learns_where_the_dialler_listens() {
+ let (a, a_address) = memory_node();
+ let (b, b_address) = memory_node();
+ let (a, b) = (Arc::new(a), Arc::new(b));
+ let _at_a = received(Arc::clone(&a));
+ b.resolve_seed(&a_address.to_string()).await.unwrap();
+ b.send_message(a.local_id(), &status()).await.unwrap();
+ let b_id = b.local_id().to_owned();
+ assert!(
+ eventually(|| a.locations(&b_id) == [b_address.to_string()]).await,
+ "a knows {:?} of b",
+ a.locations(&b_id)
+ );
+ }
+
+ #[tokio::test]
+ async fn announced_addresses_are_published_instead() {
+ let key = Keypair::generate_ed25519();
+ let peer = key.public().to_peer_id();
+ let port = NEXT_PORT.fetch_add(1, Ordering::Relaxed);
+ let announced: Multiaddr = "/ip4/203.0.113.7/tcp/9661".parse().unwrap();
+ let transport = Libp2pTransport::start(Libp2pConfig {
+ key,
+ listen: vec![format!("/memory/{port}").parse().unwrap()],
+ announce: vec![announced.clone()],
+ relays: Vec::new(),
+ relay: false,
+ mdns: false,
+ public: false,
+ memory: true,
+ })
+ .unwrap();
+ assert_eq!(
+ transport.local_addresses(),
+ [announced.with_p2p(peer).unwrap().to_string()]
+ );
+ }
+
+ #[test]
+ fn local_addresses_are_left_out_unless_there_are_only_local_ones() {
+ let parse = |text: &str| text.parse::<Multiaddr>().unwrap();
+ let loopback = parse("/ip4/127.0.0.1/tcp/1");
+ let link = parse("/ip6/fe80::1/tcp/1");
+ let lan = parse("/ip4/192.168.1.2/tcp/1");
+ let wireguard = parse("/ip6/fd00::2/tcp/1");
+ assert_eq!(
+ published(&[
+ loopback.clone(),
+ link.clone(),
+ lan.clone(),
+ wireguard.clone()
+ ]),
+ [lan, wireguard]
+ );
+ assert_eq!(
+ published(&[loopback.clone(), link.clone()]),
+ [loopback, link]
+ );
+ }
+
+ /// b listens nowhere but through the relay r; a reaches it there.
+ #[tokio::test]
+ async fn a_node_nobody_can_dial_is_reached_through_a_relay() {
+ let (_r, r_address) = memory_node_with(true, Vec::new(), true);
+ let (b, _) = memory_node_with(false, vec![r_address.clone()], false);
+ let b = Arc::new(b);
+ let mut at_b = received(Arc::clone(&b));
+ assert!(
+ eventually(|| !b.local_addresses().is_empty()).await,
+ "b got no slot on the relay"
+ );
+ let relayed = b.local_addresses()[0].clone();
+ assert!(
+ relayed.starts_with(&format!("{r_address}/p2p-circuit/")),
+ "{relayed}"
+ );
+
+ assert!(eventually(|| b.stats().libp2p.unwrap().relay_slots.held == 1).await);
+ let slots = b.stats().libp2p.unwrap().relay_slots;
+ assert_eq!((slots.granted, slots.not_granted), (1, 0));
+
+ let (a, _) = memory_node();
+ assert_eq!(a.resolve_seed(&relayed).await.unwrap(), b.local_id());
+ a.send_message(b.local_id(), &status()).await.unwrap();
+ let (from, _) = at_b.recv().await.unwrap();
+ assert_eq!(from, a.local_id());
+ assert!(a.connection(b.local_id()).is_some());
+ }
+
+ fn peer() -> PeerId {
+ Keypair::generate_ed25519().public().to_peer_id()
+ }
+
+ #[test]
+ fn relays_are_picked_when_needed() {
+ use ::libp2p::autonat::NatStatus;
+ let local = peer();
+ let offers: HashMap<PeerId, Multiaddr> = (0..4)
+ .map(|port| (peer(), format!("/memory/{port}").parse().unwrap()))
+ .collect();
+ let check = |direct, nat: &NatStatus, using: &[PeerId]| {
+ relay_check(local, direct, nat, &offers, using)
+ };
+ // Dialable, or not known yet with an address to dial: none.
+ assert!(check(true, &NatStatus::Public(Multiaddr::empty()), &[]).is_empty());
+ assert!(check(false, &NatStatus::Public(Multiaddr::empty()), &[]).is_empty());
+ assert!(check(true, &NatStatus::Unknown, &[]).is_empty());
+ // No address to dial, or AutoNAT says it can't be dialled: some.
+ assert_eq!(check(false, &NatStatus::Unknown, &[]).len(), AUTO_RELAYS);
+ let picked = check(true, &NatStatus::Private, &[]);
+ assert_eq!(picked.len(), AUTO_RELAYS);
+ // One in use: the next in the same order.
+ let more = check(true, &NatStatus::Private, &[picked[0].0]);
+ assert_eq!(more.len(), AUTO_RELAYS - 1);
+ assert_eq!(more[0].0, picked[1].0);
+ let all: Vec<PeerId> = picked.iter().map(|(relay, _)| *relay).collect();
+ assert!(check(true, &NatStatus::Private, &all).is_empty());
+ // Never itself.
+ let only_us = HashMap::from([(local, Multiaddr::empty())]);
+ assert!(relay_check(local, false, &NatStatus::Private, &only_us, &[]).is_empty());
+ }
+
+ /// A relay says so in its addresses; b, which nobody can dial, learns of it (as from
+ /// the member list) and becomes reachable through it.
+ #[tokio::test]
+ async fn a_member_that_relays_is_picked() {
+ let (r, _) = memory_node_with(true, Vec::new(), true);
+ assert!(eventually(|| !r.local_addresses().is_empty()).await);
+ let offer = r
+ .local_addresses()
+ .into_iter()
+ .find(|address| address.ends_with("/p2p-circuit"))
+ .expect("the relay offers itself");
+ let (b, _) = memory_node_with(false, Vec::new(), false);
+ assert_eq!(b.locate(&offer).as_deref(), Some(r.local_id()));
+ assert!(
+ eventually(|| b
+ .local_addresses()
+ .iter()
+ .any(|address| address.contains("/p2p-circuit/")))
+ .await,
+ "b isn't reachable through r: {:?}",
+ b.local_addresses()
+ );
+ }
+
+ /// A peer on the LAN that speaks continuum becomes a seed once, at its address.
+ #[test]
+ fn lan_peers_become_seeds_once() {
+ let mut book = Book::default();
+ let lan = peer();
+ let address: Multiaddr = "/ip4/192.168.1.5/tcp/9661".parse().unwrap();
+ book.discover(lan); // No address known: nothing.
+ assert!(book.discovered.is_empty());
+ book.remember(lan, address.clone());
+ book.discover(lan);
+ book.discover(lan);
+ assert_eq!(book.discovered, [address.with_p2p(lan).unwrap()]);
+ }
+
+ #[test]
+ fn known_addresses() {
+ let mut book = Book::default();
+ let (relay, us) = (peer(), peer());
+ let direct: Multiaddr = "/ip6/fd00::1/tcp/9661".parse().unwrap();
+ let relayed: Multiaddr = format!("/ip6/fd00::2/tcp/9661/p2p/{relay}/p2p-circuit/p2p/{us}")
+ .parse()
+ .unwrap();
+ assert!(!book.knows(us, &[direct.clone()]));
+ book.remember(us, relayed.clone());
+ book.remember(us, direct.clone());
+ // In any order, with or without the peer ID.
+ assert!(book.knows(us, &[relayed.clone(), direct.clone()]));
+ assert!(book.knows(us, &[direct.clone().with_p2p(us).unwrap()]));
+ assert!(!book.knows(us, &["/ip6/fd00::3/tcp/9661".parse().unwrap()]));
+ }
+
+ /// Peers nobody mentions any more are forgotten, but connected ones.
+ #[test]
+ fn stale_peers_are_forgotten() {
+ let start = tokio::time::Instant::now();
+ let (old, current, connected) = (peer(), peer(), peer());
+ let address: Multiaddr = "/ip6/fd00::1/tcp/9661".parse().unwrap();
+ let mut book = Book::default();
+ for peer in [old, current, connected] {
+ book.remember(peer, address.clone());
+ }
+ book.relays.insert(old, address.clone());
+ book.continuum.insert(old);
+ let later = start + STALE_AFTER;
+ // `current` was mentioned again since.
+ book.seen.insert(current, later);
+ assert_eq!(book.prune(later, |peer| *peer == connected), 1);
+ assert!(!book.addresses.contains_key(&old));
+ assert!(!book.relays.contains_key(&old) && !book.continuum.contains(&old));
+ assert!(book.addresses.contains_key(&current));
+ assert!(book.addresses.contains_key(&connected));
+ assert_eq!(book.prune(later, |_| false), 1); // `connected`, once it isn't.
+ }
+
+ /// A relay doesn't dial the nodes it relays for through itself.
+ #[test]
+ fn no_circuits_through_ourselves() {
+ let (us, other_relay, target) = (peer(), peer(), peer());
+ let through = |relay: PeerId| -> Multiaddr {
+ format!("/ip6/fd00::1/tcp/9661/p2p/{relay}/p2p-circuit")
+ .parse()
+ .unwrap()
+ };
+ let direct: Multiaddr = "/ip6/fd00::2/tcp/9661".parse().unwrap();
+ assert_eq!(relay_of(&through(us)), Some(us));
+ assert_eq!(relay_of(&direct), None);
+ assert_eq!(relay_of(&direct.clone().with_p2p(target).unwrap()), None);
+ assert_eq!(
+ dialable(&[through(us), through(other_relay), direct.clone()], us),
+ [through(other_relay), direct]
+ );
+ }
+
+ #[tokio::test]
+ async fn unknown_peers_are_refused() {
+ let (b, _) = memory_node();
+ let stranger = Keypair::generate_ed25519()
+ .public()
+ .to_peer_id()
+ .to_string();
+ assert!(b.send_message(&stranger, &status()).await.is_err());
+ assert!(b.resolve_seed(&stranger).await.is_err());
+ assert_eq!(b.connection(&stranger), None);
+ }
+
+ #[test]
+ fn a_direct_connection_wins_over_a_relayed_one() {
+ let mut book = Book::default();
+ let (relayed_only, both) = (peer(), peer());
+ let mut id = 0;
+ let mut connect = |book: &mut Book, peer: PeerId, relayed: bool| {
+ id += 1;
+ let address: Multiaddr = format!("/ip4/10.0.0.{id}/tcp/1").parse().unwrap();
+ book.connections
+ .insert(ConnectionId::new_unchecked(id), (peer, relayed, address));
+ };
+ connect(&mut book, relayed_only, true);
+ connect(&mut book, both, true);
+ connect(&mut book, both, false);
+ assert_eq!(book.connection(&relayed_only), Some(Connection::Relayed));
+ assert_eq!(book.connection(&both), Some(Connection::Direct));
+ assert_eq!(book.connection(&peer()), None);
+ assert_eq!(
+ book.location(&both).map(|address| address.to_string()),
+ Some("/ip4/10.0.0.3/tcp/1".to_owned())
+ );
+ // Of several, the oldest direct one stays; a relayed one only without a direct.
+ connect(&mut book, both, false);
+ assert_eq!(
+ book.duplicates(&both),
+ [
+ ConnectionId::new_unchecked(4),
+ ConnectionId::new_unchecked(2)
+ ]
+ );
+ assert!(book.duplicates(&relayed_only).is_empty());
+ let connections = book.stats().connections;
+ assert_eq!((connections.direct, connections.relayed), (1, 2));
+ }
+}
diff --git a/src/transport/mod.rs b/src/transport/mod.rs
index 35e3499..25ad56d 100644
--- a/src/transport/mod.rs
+++ b/src/transport/mod.rs
@@ -2,62 +2,102 @@ use anyhow::Result;
use async_trait::async_trait;
use automerge::ChangeHash;
use automerge::sync::Message;
+use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::boxed::Box;
use std::future::Future;
use std::pin::Pin;
+use std::sync::atomic::{AtomicUsize, Ordering};
-pub use crate::document::replica::Epoch;
+pub(crate) use crate::document::replica::Epoch;
-pub mod firewall;
-pub mod mycelium;
+// Used by the transports that don't authenticate their peers themselves (`tor`).
+#[cfg_attr(not(feature = "tor"), allow(dead_code))]
+pub(crate) mod challenge;
+pub(crate) mod firewall;
+pub(crate) mod identity;
+#[cfg(feature = "libp2p")]
+pub(crate) mod libp2p;
+pub(crate) mod mycelium;
+pub(crate) mod paths;
#[cfg(test)]
pub(crate) mod sim;
-pub mod traffic;
+pub(crate) mod stats;
+#[cfg(feature = "tor")]
+pub(crate) mod tor;
+pub(crate) mod traffic;
+pub(crate) mod wire;
/// Random identifier of one run of a node (from start to exit).
-pub type RunId = [u8; 16];
+pub(crate) type RunId = [u8; 16];
-/// Most heads listed in a keepalive; more are sent as a digest. A head takes 34 bytes on
-/// the wire, so 32 of them keep a keepalive within one Mycelium chunk (1300 bytes; see
-/// the size test in `mycelium`). There are rarely more heads than nodes that wrote
-/// concurrently since the last merging write.
-pub const MAX_LISTED_HEADS: usize = 32;
+/// Most heads listed in a keepalive; more are sent as a digest. A head takes 34 bytes
+/// on the wire, so 32 of them keep a keepalive within one small packet (a Mycelium
+/// chunk: 1300 bytes; see the size test in `mycelium`). There are rarely more heads
+/// than nodes that wrote concurrently since the last merging write.
+pub(crate) const MAX_LISTED_HEADS: usize = 32;
/// A document's heads, compactly: the heads themselves while there are few, otherwise a
-/// digest of them. Two summaries are equal exactly when the heads are, whatever form
-/// each took (nodes with different limits still agree).
+/// digest of them with some of them. Two summaries are equal exactly when the heads are,
+/// whatever form each took (nodes with different limits still agree).
#[derive(Debug, Clone)]
-pub enum HeadsSummary {
+pub(crate) enum HeadsSummary {
/// Sorted.
Heads(Vec<ChangeHash>),
- /// SHA-256 over the sorted heads.
- Digest([u8; 32]),
+ Digest {
+ /// SHA-256 over the sorted heads.
+ digest: [u8; 32],
+ /// `MAX_LISTED_HEADS` of the heads, another window of them each time: a receiver
+ /// pulls those it lacks, and over a few keepalives every head is listed, instead
+ /// of starting the sync over to find out what it lacks.
+ sample: Vec<ChangeHash>,
+ },
}
+/// Where the next sample starts (`HeadsSummary::of`).
+static SAMPLE_OFFSET: AtomicUsize = AtomicUsize::new(0);
+
impl HeadsSummary {
- pub fn of(mut heads: Vec<ChangeHash>) -> Self {
+ pub(crate) fn of(mut heads: Vec<ChangeHash>) -> Self {
heads.sort_unstable();
if heads.len() <= MAX_LISTED_HEADS {
- Self::Heads(heads)
- } else {
- Self::Digest(digest_of(&heads))
+ return Self::Heads(heads);
+ }
+ let start = SAMPLE_OFFSET.fetch_add(MAX_LISTED_HEADS, Ordering::Relaxed) % heads.len();
+ let sample = heads
+ .iter()
+ .cycle()
+ .skip(start)
+ .take(MAX_LISTED_HEADS)
+ .copied()
+ .collect();
+ Self::Digest {
+ digest: digest_of(&heads),
+ sample,
}
}
/// The digest of the heads, computed if they are listed.
- pub fn digest(&self) -> [u8; 32] {
+ pub(crate) fn digest(&self) -> [u8; 32] {
match self {
Self::Heads(heads) => digest_of(heads),
- Self::Digest(digest) => *digest,
+ Self::Digest { digest, .. } => *digest,
}
}
- /// The heads, if listed.
- pub fn heads(&self) -> Option<&[ChangeHash]> {
+ /// The heads, if all are listed.
+ pub(crate) fn heads(&self) -> Option<&[ChangeHash]> {
match self {
Self::Heads(heads) => Some(heads),
- Self::Digest(_) => None,
+ Self::Digest { .. } => None,
+ }
+ }
+
+ /// The heads listed: all of them, or a sample.
+ pub(crate) fn listed(&self) -> &[ChangeHash] {
+ match self {
+ Self::Heads(heads) => heads,
+ Self::Digest { sample, .. } => sample,
}
}
}
@@ -82,79 +122,117 @@ fn digest_of(sorted_heads: &[ChangeHash]) -> [u8; 32] {
hasher.finalize().into()
}
-/// PING and PONG. Addressed to one peer: `syncs_with_you` is about the recipient.
-#[derive(Debug, Clone)]
-pub struct Keepalive {
+/// STATUS and STATUS_REPLY. Addressed to one peer: `syncs_with_you` is about the recipient.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct Status {
/// Only comparable between nodes on the same epoch.
- pub heads: HeadsSummary,
- pub epoch: Epoch,
+ #[serde(with = "crate::transport::wire::codec::heads")]
+ pub(crate) heads: HeadsSummary,
+ #[serde(with = "crate::transport::wire::codec::epoch")]
+ pub(crate) epoch: Epoch,
/// See `SyncEnvelope::epoch_seq`.
- pub epoch_seq: u32,
+ pub(crate) epoch_seq: u32,
/// The sender is ready (same meaning as `SyncEnvelope::ready`).
- pub ready: bool,
+ pub(crate) ready: bool,
/// The sender would sync with the recipient: counts it as an active member on its
/// own epoch.
- pub syncs_with_you: bool,
+ pub(crate) syncs_with_you: bool,
}
-#[derive(Debug)]
-pub enum ParsedMessage {
+/// Cheap to clone: large parts are shared (`bytes::Bytes`).
+#[derive(Debug, Clone)]
+pub(crate) enum ParsedMessage {
Hello(HelloRequest),
HelloReply(HelloReply),
Sync(SyncEnvelope),
- Ping(Keepalive),
- Pong(Keepalive),
+ Status(Status),
+ StatusReply(Status),
Blob(BlobMessage),
+ /// Asks the recipient who it is, saying who the sender is (`transport::identity`).
+ Who(IdentityProof),
+ /// Answers a WHO.
+ Me(IdentityProof),
+ /// Asks for a PONG with this nonce, over the same path (`transport::paths`).
+ Ping(u64),
+ /// Answers a PING.
+ Pong(u64),
+}
+
+/// A node's proof that it holds the key of its node ID (`identity`), sending from its
+/// transport address to the recipient's: a signature over both addresses
+/// (`transport::identity::signed`). WHO and ME end at the identity layer; the rest of
+/// the node never sees them.
+#[derive(Clone, Serialize, Deserialize)]
+pub(crate) struct IdentityProof {
+ /// The sender's public node key.
+ #[serde(with = "crate::transport::wire::codec::bytes32")]
+ pub(crate) key: [u8; 32],
+ #[serde(with = "crate::transport::wire::codec::bytes64")]
+ pub(crate) signature: [u8; 64],
+}
+
+impl std::fmt::Debug for IdentityProof {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("IdentityProof")
+ .field("key", &Hex(&self.key))
+ .finish_non_exhaustive()
+ }
}
/// Content-addressed bytes that any node holding them can serve, in chunks
/// (`swarm::blob`).
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum BlobId {
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub(crate) enum BlobId {
/// An Automerge change, by its hash: e.g. an epoch's snapshot change.
- Change(ChangeHash),
+ Change(#[serde(with = "crate::transport::wire::codec::change_hash")] ChangeHash),
/// Bytes by their BLAKE3 hash: e.g. the content of a remake.
- Content([u8; 32]),
+ Content(#[serde(with = "crate::transport::wire::codec::bytes32")] [u8; 32]),
+ /// The changes of a `DeltaSpec`, by its ID (`DeltaSpec::id`). The requests carry
+ /// the spec, so any node holding its `to` can serve it.
+ Delta(#[serde(with = "crate::transport::wire::codec::bytes32")] [u8; 32]),
}
impl std::fmt::Display for BlobId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
BlobId::Change(hash) => write!(f, "change:{hash}"),
- BlobId::Content(hash) => {
- write!(f, "content:")?;
- hash.iter().try_for_each(|byte| write!(f, "{byte:02x}"))
- }
+ BlobId::Content(hash) => write!(f, "content:{}", Hex(hash)),
+ BlobId::Delta(id) => write!(f, "delta:{}", Hex(id)),
}
}
}
/// A blob request or one of its answers. Each is a message of its own, so no message
/// is larger than one chunk.
-#[derive(Debug, Clone)]
-pub struct BlobMessage {
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct BlobMessage {
/// The sender's epoch (every message carries it).
- pub epoch: Epoch,
+ #[serde(with = "crate::transport::wire::codec::epoch")]
+ pub(crate) epoch: Epoch,
/// See `SyncEnvelope::epoch_seq`.
- pub epoch_seq: u32,
- pub id: BlobId,
- pub body: BlobBody,
+ pub(crate) epoch_seq: u32,
+ pub(crate) id: BlobId,
+ pub(crate) body: BlobBody,
}
-#[derive(Clone)]
-pub enum BlobBody {
- /// Asks for these chunks; for the manifest if empty.
+#[derive(Clone, Serialize, Deserialize)]
+pub(crate) enum BlobBody {
+ /// Asks for these chunks; for the manifest if empty. `delta`: the spec of a
+ /// `BlobId::Delta`, which the ID alone doesn't say.
Request {
chunks: Vec<u32>,
+ delta: Option<DeltaSpec>,
},
/// Size and per-chunk BLAKE3 hashes; the last chunk may be shorter.
Manifest {
size: u64,
chunk_size: u32,
+ #[serde(with = "crate::transport::wire::codec::hashes32")]
hashes: Vec<[u8; 32]>,
},
Chunk {
index: u32,
+ #[serde(with = "crate::transport::wire::codec::shared_bytes")]
data: bytes::Bytes,
},
/// The sender doesn't have the blob.
@@ -165,9 +243,11 @@ pub enum BlobBody {
impl std::fmt::Debug for BlobBody {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
- BlobBody::Request { chunks } => {
- f.debug_struct("Request").field("chunks", chunks).finish()
- }
+ BlobBody::Request { chunks, delta } => f
+ .debug_struct("Request")
+ .field("chunks", chunks)
+ .field("delta", &delta.is_some())
+ .finish(),
BlobBody::Manifest {
size,
chunk_size,
@@ -190,125 +270,439 @@ impl std::fmt::Debug for BlobBody {
/// What a message is, without its contents.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum MessageKind {
+pub(crate) enum MessageKind {
Hello,
HelloReply,
Sync,
+ Status,
+ StatusReply,
+ Blob,
+ Who,
+ Me,
Ping,
Pong,
- Blob,
}
impl ParsedMessage {
- pub fn kind(&self) -> MessageKind {
+ pub(crate) fn kind(&self) -> MessageKind {
match self {
ParsedMessage::Hello(_) => MessageKind::Hello,
ParsedMessage::HelloReply(_) => MessageKind::HelloReply,
ParsedMessage::Sync(_) => MessageKind::Sync,
+ ParsedMessage::Status(_) => MessageKind::Status,
+ ParsedMessage::StatusReply(_) => MessageKind::StatusReply,
+ ParsedMessage::Blob(_) => MessageKind::Blob,
+ ParsedMessage::Who(_) => MessageKind::Who,
+ ParsedMessage::Me(_) => MessageKind::Me,
ParsedMessage::Ping(_) => MessageKind::Ping,
ParsedMessage::Pong(_) => MessageKind::Pong,
- ParsedMessage::Blob(_) => MessageKind::Blob,
- }
- }
-
- /// Every message carries the sender's epoch.
- pub fn epoch(&self) -> Epoch {
- match self {
- ParsedMessage::Hello(hello) => hello.epoch,
- ParsedMessage::HelloReply(reply) => reply.epoch,
- ParsedMessage::Sync(envelope) => envelope.epoch,
- ParsedMessage::Ping(keepalive) | ParsedMessage::Pong(keepalive) => keepalive.epoch,
- ParsedMessage::Blob(blob) => blob.epoch,
}
}
- /// `SyncEnvelope::epoch_seq`, as every message carries it.
- pub fn epoch_seq(&self) -> u32 {
+ /// The sender's epoch and `SyncEnvelope::epoch_seq`, as every message carries them
+ /// but WHO, ME, PING and PONG (which the node never sees).
+ pub(crate) fn epoch_stamp(&self) -> Option<(Epoch, u32)> {
match self {
- ParsedMessage::Hello(hello) => hello.epoch_seq,
- ParsedMessage::HelloReply(reply) => reply.epoch_seq,
- ParsedMessage::Sync(envelope) => envelope.epoch_seq,
- ParsedMessage::Ping(keepalive) | ParsedMessage::Pong(keepalive) => keepalive.epoch_seq,
- ParsedMessage::Blob(blob) => blob.epoch_seq,
+ ParsedMessage::Hello(hello) => Some((hello.epoch, hello.epoch_seq)),
+ ParsedMessage::HelloReply(reply) => Some((reply.epoch, reply.epoch_seq)),
+ ParsedMessage::Sync(envelope) => Some((envelope.epoch, envelope.epoch_seq)),
+ ParsedMessage::Status(status) | ParsedMessage::StatusReply(status) => {
+ Some((status.epoch, status.epoch_seq))
+ }
+ ParsedMessage::Blob(blob) => Some((blob.epoch, blob.epoch_seq)),
+ ParsedMessage::Who(_)
+ | ParsedMessage::Me(_)
+ | ParsedMessage::Ping(_)
+ | ParsedMessage::Pong(_) => None,
}
}
}
-#[derive(Debug, Clone)]
-pub struct HelloRequest {
+#[derive(Clone, Serialize, Deserialize)]
+pub(crate) struct HelloRequest {
/// The sender's swarm, if it knows it (a new node doesn't yet). A receiver in
/// another swarm doesn't answer.
- pub swarm_id: Option<String>,
- pub epoch: Epoch,
+ pub(crate) swarm_id: Option<String>,
+ #[serde(with = "crate::transport::wire::codec::epoch")]
+ pub(crate) epoch: Epoch,
/// See `SyncEnvelope::epoch_seq`.
- pub epoch_seq: u32,
+ pub(crate) epoch_seq: u32,
/// Proof of the swarm key (`swarm::auth`); empty without one.
- pub auth: Vec<u8>,
+ #[serde(with = "serde_bytes")]
+ pub(crate) auth: Vec<u8>,
/// The sender is ready (same meaning as `SyncEnvelope::ready`).
- pub ready: bool,
+ pub(crate) ready: bool,
/// Random per run of the sender; the reply echoes it.
- pub run_id: RunId,
+ #[serde(with = "crate::transport::wire::codec::run_id")]
+ pub(crate) run_id: RunId,
+ /// The handshake's number: the sender numbers the handshakes it starts in a run,
+ /// with all peers, from 1. A retry carries the same number, so the recipient resets
+ /// its sync session only for a new handshake, not for every resend or late copy.
+ pub(crate) handshake: u32,
}
-#[derive(Debug, Clone)]
-pub struct HelloReply {
- pub swarm_id: String,
- pub peers: Vec<String>,
- pub epoch: Epoch,
+/// A peer a HELLO_REPLY lists: who it is, and where the sender reaches it.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) struct ListedPeer {
+ pub(crate) id: String,
+ /// As `Transport::addresses_of` gives them; hints for the recipient.
+ pub(crate) addresses: Vec<String>,
+}
+
+/// Where messages to a peer go now (`Transport::destination`).
+#[derive(Debug, Clone, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct Destination {
+ /// As the identity layer keeps it: `<scheme>:<transport identity>`.
+ pub(crate) address: String,
+ /// The peer proved itself there; else it is only said to be there, and sends wait
+ /// for its proof.
+ pub(crate) proven: bool,
+ /// The transport's connection to it, if the transport has connections.
+ pub(crate) connection: Option<Connection>,
+ /// Every address it proved itself at, best first, as the paths layer measures them
+ /// (`transport::paths`).
+ pub(crate) paths: Vec<stats::PathInfo>,
+}
+
+/// How a transport with connections (libp2p) is connected to a peer.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) enum Connection {
+ Direct,
+ /// Only through a relay.
+ Relayed,
+}
+
+impl Connection {
+ pub(crate) fn as_str(self) -> &'static str {
+ match self {
+ Self::Direct => "direct",
+ Self::Relayed => "relayed",
+ }
+ }
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub(crate) struct HelloReply {
+ pub(crate) swarm_id: String,
+ pub(crate) peers: Vec<ListedPeer>,
+ #[serde(with = "crate::transport::wire::codec::epoch")]
+ pub(crate) epoch: Epoch,
/// See `SyncEnvelope::epoch_seq`.
- pub epoch_seq: u32,
+ pub(crate) epoch_seq: u32,
/// Proof of the swarm key (`swarm::auth`); empty without one.
- pub auth: Vec<u8>,
+ #[serde(with = "serde_bytes")]
+ pub(crate) auth: Vec<u8>,
/// The sender is ready.
- pub ready: bool,
+ pub(crate) ready: bool,
/// The `run_id` of the HELLO answered. A reply to a HELLO from an earlier run of the
/// recipient (delivered late, or queued while it was down) is ignored.
- pub run_id: RunId,
+ #[serde(with = "crate::transport::wire::codec::run_id")]
+ pub(crate) run_id: RunId,
/// How long before this HELLO the sender had last heard from the recipient, if ever
/// (in milliseconds): tells a restarted node how long it was away.
- pub away_ms: Option<u64>,
+ pub(crate) away_ms: Option<u64>,
/// The sender's own run (as its HELLOs carry it): a new one means it restarted.
- pub sender_run_id: RunId,
+ #[serde(with = "crate::transport::wire::codec::run_id")]
+ pub(crate) sender_run_id: RunId,
}
-#[derive(Debug, Clone)]
-pub struct SyncEnvelope {
- pub swarm_id: String,
+/// Bytes shown as lowercase hex, e.g. run IDs in logs.
+pub(crate) struct Hex<'a>(pub(crate) &'a [u8]);
+
+impl std::fmt::Display for Hex<'_> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ self.0.iter().try_for_each(|byte| write!(f, "{byte:02x}"))
+ }
+}
+
+impl std::fmt::Debug for Hex<'_> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ std::fmt::Display::fmt(self, f)
+ }
+}
+
+/// Whether a handshake carries a swarm-key proof, not the proof: logged messages don't
+/// hand out tags, and stay short.
+fn auth_shown(auth: &[u8]) -> &'static str {
+ if auth.is_empty() { "none" } else { "present" }
+}
+
+impl std::fmt::Debug for HelloRequest {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("HelloRequest")
+ .field("swarm_id", &self.swarm_id)
+ .field("epoch", &self.epoch)
+ .field("epoch_seq", &self.epoch_seq)
+ .field("auth", &auth_shown(&self.auth))
+ .field("ready", &self.ready)
+ .field("run_id", &Hex(&self.run_id))
+ .field("handshake", &self.handshake)
+ .finish()
+ }
+}
+
+impl std::fmt::Debug for HelloReply {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("HelloReply")
+ .field("swarm_id", &self.swarm_id)
+ .field("peers", &self.peers.len())
+ .field("epoch", &self.epoch)
+ .field("epoch_seq", &self.epoch_seq)
+ .field("auth", &auth_shown(&self.auth))
+ .field("ready", &self.ready)
+ .field("run_id", &Hex(&self.run_id))
+ .field("away_ms", &self.away_ms)
+ .field("sender_run_id", &Hex(&self.sender_run_id))
+ .finish()
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct SyncEnvelope {
+ pub(crate) swarm_id: String,
/// Epoch of the document the message was generated from.
- pub epoch: Epoch,
+ #[serde(with = "crate::transport::wire::codec::epoch")]
+ pub(crate) epoch: Epoch,
/// How many times the sender changed epoch in its current run, when it sent this
/// (`Replica::epoch_seq`). Messages may arrive late and out of order: the receiver
/// ignores an epoch older than the newest it has seen in the same session.
- pub epoch_seq: u32,
- pub message: Message,
+ pub(crate) epoch_seq: u32,
+ pub(crate) body: SyncBody,
/// The sender was ready when it generated the message: its document is the swarm's,
/// not one it is still filling (joining, rejoining). Only such a peer can tell a
/// catching-up node that it has everything; an empty document says nothing else.
- pub ready: bool,
+ pub(crate) ready: bool,
+}
+
+/// What a sync envelope carries.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) enum SyncBody {
+ /// An Automerge sync message.
+ Message(#[serde(with = "crate::transport::wire::codec::sync_message")] SyncPayload),
+ /// Instead of a message too large to send in one piece: fetch these changes as a
+ /// blob (`BlobId::Delta`), then sync on.
+ Delta(DeltaSpec),
+}
+
+/// The changes in the history of `to` but not of `from`: what a peer at `from` lacks of
+/// a document at `to`. Every node holding `to` produces the same bytes for it
+/// (`swarm::blob::delta_bytes`).
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) struct DeltaSpec {
+ /// Sorted.
+ #[serde(with = "crate::transport::wire::codec::change_hashes")]
+ pub(crate) from: Vec<ChangeHash>,
+ /// Sorted.
+ #[serde(with = "crate::transport::wire::codec::change_hashes")]
+ pub(crate) to: Vec<ChangeHash>,
+}
+
+impl DeltaSpec {
+ pub(crate) fn new(mut from: Vec<ChangeHash>, mut to: Vec<ChangeHash>) -> Self {
+ from.sort_unstable();
+ from.dedup();
+ to.sort_unstable();
+ to.dedup();
+ Self { from, to }
+ }
+
+ /// Names the spec as a blob: SHA-256 over both lists.
+ pub(crate) fn id(&self) -> [u8; 32] {
+ let mut hasher = Sha256::new();
+ hasher.update(b"continuum/delta/1");
+ for heads in [&self.from, &self.to] {
+ hasher.update((heads.len() as u32).to_be_bytes());
+ for head in heads {
+ hasher.update(head.0);
+ }
+ }
+ hasher.finalize().into()
+ }
+}
+
+/// An Automerge sync message in Automerge's own encoding. Decoded only where it is
+/// applied (`decode`), after the sender proved to be a member and off the consumer: a
+/// large message from anyone else costs nothing but its bytes.
+#[derive(Clone)]
+pub(crate) struct SyncPayload(pub(crate) bytes::Bytes);
+
+impl SyncPayload {
+ pub(crate) fn encode(message: Message) -> Self {
+ Self(message.encode().into())
+ }
+
+ pub(crate) fn decode(&self) -> Result<Message> {
+ Message::decode(&self.0)
+ .map_err(|error| anyhow::anyhow!("Automerge decode failed: {error:?}"))
+ }
+}
+
+/// Sizes instead of contents: logged messages stay short.
+impl std::fmt::Debug for SyncPayload {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "SyncPayload({} bytes)", self.0.len())
+ }
}
/// A message serialized for the wire. Cloning is cheap (shared buffer).
#[derive(Clone, Debug)]
-pub struct EncodedMessage {
- pub peer: String,
- pub body: bytes::Bytes,
- pub kind: MessageKind,
+pub(crate) struct EncodedMessage {
+ pub(crate) peer: String,
+ /// Where the transport sends it: `peer` itself for a transport whose identities are
+ /// addresses, the peer's address under the identity layer (`transport::identity`).
+ pub(crate) address: String,
+ pub(crate) body: bytes::Bytes,
+ pub(crate) kind: MessageKind,
/// Bytes the message takes on the wire (without the transport's own framing).
- pub payload_len: usize,
+ pub(crate) payload_len: usize,
+ /// The message itself, under the paths layer (`transport::paths`): to encode it
+ /// again for another transport when a send there fails. None from a transport.
+ pub(crate) message: Option<std::sync::Arc<ParsedMessage>>,
}
/// Called with the sender, the message and its `payload_len` (as in `EncodedMessage`).
-pub type HandlerFn = dyn Fn(String, ParsedMessage, usize) -> Pin<Box<dyn Future<Output = Result<()>> + Send>>
+pub(crate) type HandlerFn = dyn Fn(String, ParsedMessage, usize) -> Pin<Box<dyn Future<Output = Result<()>> + Send>>
+ Send
+ Sync
+ 'static;
-pub type MessageHandler = Box<HandlerFn>;
+pub(crate) type MessageHandler = Box<HandlerFn>;
+
+/// Whether `message` is a PING or PONG. The paths layer times them, so a transport's
+/// consumer hands them on as soon as they are read (`hand_on_probe`), not after the
+/// messages read before them are handled (which can take a while: sync).
+pub(crate) fn is_probe(message: &ParsedMessage) -> bool {
+ matches!(message, ParsedMessage::Ping(_) | ParsedMessage::Pong(_))
+}
+
+/// Hands a PING or PONG (`is_probe`) to `handler` in a task of its own.
+pub(crate) fn hand_on_probe(
+ handler: &std::sync::Arc<HandlerFn>,
+ sender: String,
+ message: ParsedMessage,
+ payload_len: usize,
+) {
+ let handler = std::sync::Arc::clone(handler);
+ tokio::spawn(async move {
+ if let Err(error) = handler(sender.clone(), message, payload_len).await {
+ tracing::warn!(peer = %sender, error = %format!("{error:#}"), "Failed to process inbound message");
+ }
+ });
+}
#[async_trait]
-pub trait Transport: Send + Sync {
+pub(crate) trait Transport: Send + Sync {
/// This node's identity on the transport (for Mycelium: its public key).
fn local_id(&self) -> &str;
+ /// Whether `id` can be a node's identity on this transport (for Mycelium: a public
+ /// key). Identities other peers name to us (in a HELLO_REPLY) are checked with it.
+ fn is_peer_id(&self, id: &str) -> bool;
+
+ /// How long the transport tries to deliver a message before dropping it: after that,
+ /// a message sent is delivered or lost for good.
+ fn delivery_limit(&self) -> std::time::Duration;
+
+ /// Where this node can be reached, as addresses other nodes can use: published in
+ /// the member list (`swarm::members`). By default its identity, for a transport
+ /// whose identities are addresses (Mycelium: a public key).
+ fn local_addresses(&self) -> Vec<String> {
+ vec![self.local_id().to_owned()]
+ }
+
+ /// For a transport under the identity layer: the transport identity at `target`, an
+ /// address of this transport without its scheme, noting where it is if `target` says
+ /// more than who (as a libp2p multiaddr ending in the peer ID does). None if
+ /// `target` isn't one of this transport's. By default `target` itself, for a
+ /// transport whose identities are addresses.
+ fn locate(&self, target: &str) -> Option<String> {
+ self.is_peer_id(target).then(|| target.to_owned())
+ }
+
+ /// For a transport under the identity layer: where the node with transport identity
+ /// `target` is, as addresses others can use (`locate` reads them). By default
+ /// `target` itself, as for `locate`.
+ fn locations(&self, target: &str) -> Vec<String> {
+ vec![target.to_owned()]
+ }
+
+ /// Where `peer` can be reached, as far as this node knows: passed on to others (in
+ /// a HELLO_REPLY). By default its identity, as for `local_addresses`.
+ fn addresses_of(&self, peer: &str) -> Vec<String> {
+ vec![peer.to_owned()]
+ }
+
+ /// The transports this node is on, as the schemes of their addresses
+ /// (`swarm::topology`), whether or not it has an address on each (one that only
+ /// connects out has none). By default the empty scheme, for a transport whose
+ /// identities are addresses.
+ fn transports(&self) -> Vec<String> {
+ vec![String::new()]
+ }
+
+ /// Seeds the transport found by itself since last asked (libp2p: nodes on the LAN),
+ /// for a node that is joining. By default none.
+ fn discovered_seeds(&self) -> Vec<String> {
+ Vec::new()
+ }
+
+ /// Whether this node has a way to reach `peer` at all: an address of it on one of
+ /// its transports. Members it can't reach aren't its neighbours (`swarm::topology`).
+ /// By default yes, as for `local_addresses`.
+ fn can_reach(&self, peer: &str) -> bool {
+ let _ = peer;
+ true
+ }
+
+ /// Where others say `peer` can be reached (the member list, a HELLO_REPLY): hints,
+ /// to be checked before they are relied on. By default nothing to learn, as for
+ /// `local_addresses`.
+ fn learn_addresses(&self, peer: &str, addresses: &[String]) {
+ let _ = (peer, addresses);
+ }
+
+ /// Where messages to `peer` go now. None if nowhere, and by default, for a
+ /// transport whose identities are addresses (`peer` is where they go).
+ fn destination(&self, peer: &str) -> Option<Destination> {
+ let _ = peer;
+ None
+ }
+
+ /// Where messages to `target` (a transport identity) go now, as an address that
+ /// names `target` too: the remote address of the connection in use, say. By default
+ /// nowhere but `target` itself (Mycelium).
+ fn location(&self, target: &str) -> Option<String> {
+ let _ = target;
+ None
+ }
+
+ /// How this node is connected to `target` (a transport identity), if it is. By
+ /// default not at all, for a transport without connections (Mycelium).
+ fn connection(&self, target: &str) -> Option<Connection> {
+ let _ = target;
+ None
+ }
+
+ /// The smoothed round-trip time of the path messages to `peer` go over, if
+ /// measured (`transport::paths`). By default none.
+ fn rtt(&self, peer: &str) -> Option<std::time::Duration> {
+ let _ = peer;
+ None
+ }
+
+ /// Whether the transport answers PINGs with PONGs itself (the paths layer,
+ /// `transport::paths`); if not (a bare transport, in tests), the node does. By
+ /// default not.
+ fn answers_pings(&self) -> bool {
+ false
+ }
+
+ /// Figures about the transport itself (`stats`). By default none.
+ fn stats(&self) -> stats::TransportStats {
+ stats::TransportStats::default()
+ }
+
/// Resolves a seed (anything that leads to a node, e.g. an address) to that node's
/// identity, as `local_id` gives it on that node.
async fn resolve_seed(&self, seed: &str) -> Result<String>;
@@ -347,7 +741,16 @@ mod tests {
let mut shuffled = heads.clone();
rng.shuffle(&mut shuffled);
assert_eq!(summary, HeadsSummary::of(shuffled));
- assert_eq!(summary, HeadsSummary::Digest(summary.digest()));
+ assert_eq!(
+ summary,
+ HeadsSummary::Digest {
+ digest: summary.digest(),
+ sample: Vec::new(),
+ }
+ );
+ // A sample is some of the heads.
+ assert!(summary.listed().iter().all(|head| heads.contains(head)));
+ assert_eq!(summary.listed().len(), n.min(MAX_LISTED_HEADS));
let mut other = heads.clone();
match rng.below(3) {
@@ -364,4 +767,22 @@ mod tests {
assert_ne!(summary, HeadsSummary::of(other));
});
}
+
+ /// Successive samples cover every head.
+ #[test]
+ fn samples_rotate_over_all_heads() {
+ let heads: Vec<ChangeHash> = (0..100u8).map(|n| ChangeHash([n; 32])).collect();
+ let mut seen = std::collections::HashSet::new();
+ for _ in 0..heads.len().div_ceil(MAX_LISTED_HEADS) + 1 {
+ seen.extend(HeadsSummary::of(heads.clone()).listed().iter().copied());
+ }
+ // Other tests sample concurrently and move the window too, so allow a few more.
+ for _ in 0..100 {
+ if seen.len() == heads.len() {
+ break;
+ }
+ seen.extend(HeadsSummary::of(heads.clone()).listed().iter().copied());
+ }
+ assert_eq!(seen.len(), heads.len());
+ }
}
diff --git a/src/transport/mycelium.rs b/src/transport/mycelium.rs
deleted file mode 100644
index efab475..0000000
--- a/src/transport/mycelium.rs
+++ /dev/null
@@ -1,1276 +0,0 @@
-use anyhow::{Context, Result, anyhow, bail};
-use async_trait::async_trait;
-use automerge::ChangeHash;
-use base64::engine::{Engine, general_purpose::STANDARD as BASE64};
-use reqwest::StatusCode;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
-use serde_bytes::{ByteBuf, Bytes};
-use std::borrow::Cow;
-use std::boxed::Box;
-use std::fs::File;
-use std::io::Write;
-use std::net::TcpListener;
-use std::os::unix::io::FromRawFd;
-use std::path::PathBuf;
-use std::process::Stdio;
-use std::time::Duration;
-use thiserror::Error;
-use tokio::process::{Child, Command};
-use url::Url;
-
-use crate::transport::EncodedMessage;
-use crate::transport::{BlobBody, BlobId, BlobMessage};
-use crate::transport::{Epoch, HelloReply, HelloRequest, SyncEnvelope};
-use crate::transport::{HeadsSummary, Keepalive, MAX_LISTED_HEADS};
-use crate::transport::{MessageHandler, ParsedMessage, Transport};
-
-const MYCELIUM_POLL_TIMEOUT_SECS: u64 = 300;
-
-const TOPIC_PREFIX: &str = "continuum.v1.";
-const TOPIC_HELLO: &str = "continuum.v1.hello";
-const TOPIC_HELLO_REPLY: &str = "continuum.v1.hello-reply";
-const TOPIC_SYNC: &str = "continuum.v1.sync";
-const TOPIC_PING: &str = "continuum.v1.ping";
-const TOPIC_PONG: &str = "continuum.v1.pong";
-const TOPIC_BLOB: &str = "continuum.v1.blob";
-
-const EMPTY_QUEUE_DELAY: Duration = Duration::from_millis(25);
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct InboundMessage {
- id: String,
- src_pk: String,
- topic: String,
- payload: String,
-}
-
-// Deserialize only for debugging
-#[derive(Debug, Serialize, Deserialize)]
-struct OutboundMessage {
- dst: Destination,
- topic: String,
- payload: String,
-}
-
-// Deserialize only for debugging
-#[derive(Debug, Serialize, Deserialize)]
-struct Destination {
- pk: String,
-}
-
-/// Represents where a 32-byte cryptographic key originates.
-#[derive(Debug, Clone)]
-pub enum KeySource {
- Raw([u8; 32]),
- Path(PathBuf),
-}
-
-/// Atomic structure ensuring private network credentials are always handled together.
-#[derive(Debug, Clone)]
-pub struct PrivateNetworkConfig {
- pub name: String,
- pub key: KeySource,
-}
-
-/// Comprehensive parameters required to spawn or target an instance.
-#[derive(Debug, Clone)]
-pub struct MyceliumSpawnConfig {
- pub rest_url_opt: Option<Url>,
- pub key_file_opt: Option<PathBuf>,
- pub network_key_file: Option<PathBuf>,
- pub network_name: String,
- pub tcp_port: Option<u16>,
- pub api_port: Option<u16>,
- pub rpc_port: Option<u16>,
- pub discovery_port: Option<u16>,
- pub peers: Vec<String>,
-}
-
-impl Default for MyceliumSpawnConfig {
- fn default() -> Self {
- Self {
- rest_url_opt: None,
- key_file_opt: None,
- network_key_file: None,
- network_name: "something".to_string(),
- tcp_port: None,
- api_port: None,
- rpc_port: None,
- discovery_port: None,
- peers: vec!["tls://127.0.0.1:9651".to_string()],
- }
- }
-}
-
-pub enum MyceliumRuntime {
- Supervised(Box<SupervisedMycelium>),
- External,
-}
-
-pub struct MyceliumSetup {
- pub runtime: MyceliumRuntime,
- pub rest_url: Url,
-}
-
-impl MyceliumSetup {
- /// Evaluates the configuration payload and initializes the chosen execution strategy.
- pub async fn bootstrap(config: MyceliumSpawnConfig) -> Result<Self> {
- // TODO warn for all ignored options if rest_url_opt is set
- if let Some(url) = config.rest_url_opt {
- tracing::info!(
- "Configured to interface with an external standalone Mycelium instance at: {}",
- url
- );
- return Ok(Self {
- runtime: MyceliumRuntime::External,
- rest_url: url,
- });
- }
-
- tracing::info!(
- "Configured to initialize an embedded user-space Mycelium background worker"
- );
-
- let key = match config.key_file_opt {
- Some(key_file) => KeySource::Path(key_file),
- None => {
- let mut bytes = [0u8; 32];
- getrandom::fill(&mut bytes)?;
- KeySource::Raw(bytes)
- }
- };
-
- let private_net_config = config
- .network_key_file
- .map(|net_key_path| PrivateNetworkConfig {
- name: config.network_name.clone(),
- key: KeySource::Path(net_key_path),
- });
-
- let supervised_node = SupervisedMycelium::spawn(
- key,
- private_net_config,
- config.tcp_port,
- config.api_port,
- config.rpc_port,
- config.discovery_port,
- &config.peers,
- )
- .await
- .context("Failed to securely bootstrap embedded Mycelium background worker")?;
-
- let resolved_url = Url::parse(&format!(
- "http://127.0.0.1:{}",
- supervised_node.config.api_port
- ))?;
-
- Ok(Self {
- runtime: MyceliumRuntime::Supervised(Box::new(supervised_node)),
- rest_url: resolved_url,
- })
- }
-
- /// Blocks until the background daemon drops out. If running externally, hangs open indefinitely.
- pub async fn wait_for_exit(&mut self) -> std::io::Result<std::process::ExitStatus> {
- match &mut self.runtime {
- MyceliumRuntime::Supervised(node) => node.wait_for_exit().await,
- MyceliumRuntime::External => {
- std::future::pending::<std::io::Result<std::process::ExitStatus>>().await
- }
- }
- }
-}
-
-/// The part of Mycelium's `/api/v1/admin` answer we use.
-#[derive(Deserialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct MyceliumAdminInfo {
- pub node_pubkey: String,
-}
-
-#[derive(Debug, Clone)]
-pub struct MyceliumNodeConfig {
- pub tcp_port: u16,
- pub api_port: u16,
- pub rpc_port: u16,
- pub discovery_port: Option<u16>,
- pub private_network: Option<String>,
-}
-
-pub struct SupervisedMycelium {
- child: Child,
- pub config: MyceliumNodeConfig,
- _key_file_handle: Option<File>,
- _net_key_file_handle: Option<File>,
-}
-
-impl SupervisedMycelium {
- async fn spawn(
- node_key: KeySource,
- private_network: Option<PrivateNetworkConfig>,
- explicit_tcp: Option<u16>,
- explicit_api: Option<u16>,
- explicit_rpc: Option<u16>,
- explicit_discovery_port: Option<u16>,
- peers: &[String],
- ) -> Result<Self> {
- // Linux only: keys are handed over in memory (memfd, never on disk), and the
- // child dies with us (PR_SET_PDEATHSIG). Elsewhere, run myceliumd yourself.
- if cfg!(not(target_os = "linux")) {
- bail!(
- "Supervising Mycelium is only supported on Linux; run myceliumd yourself \
- and pass --mycelium-rest-url"
- );
- }
-
- let tcp_port = match explicit_tcp {
- Some(p) => p,
- None => Self::find_available_port().context("Failed to allocate TCP wire port")?,
- };
- let api_port = match explicit_api {
- Some(p) => p,
- None => Self::find_available_port().context("Failed to allocate API port")?,
- };
- let rpc_port = match explicit_rpc {
- Some(p) => p,
- None => Self::find_available_port().context("Failed to allocate JSON-RPC port")?,
- };
- let discovery_port = match explicit_discovery_port {
- Some(0) => Some(
- Self::find_available_port().context("Failed to allocate peer discovery port")?,
- ),
- p => p,
- };
-
- let config = MyceliumNodeConfig {
- tcp_port,
- api_port,
- rpc_port,
- discovery_port,
- private_network: private_network.as_ref().map(|pn| pn.name.clone()),
- };
-
- let parent_pid = std::process::id();
-
- let (key_file_handle, key_file_path, node_fd) =
- Self::resolve_key_strategy("mycelium_node_key", node_key, parent_pid)?;
-
- let (net_key_file_handle, net_key_file_path, net_fd) = match private_network {
- Some(net_cfg) => {
- let (h, p, fd) =
- Self::resolve_key_strategy("mycelium_net_key", net_cfg.key, parent_pid)?;
- (h, Some(p), fd)
- }
- None => (None, None, None),
- };
-
- let api_addr = format!("127.0.0.1:{}", config.api_port);
- let rpc_addr = format!("127.0.0.1:{}", config.rpc_port);
-
- let mut cmd = Command::new("mycelium");
-
- let fds_to_inherit: Vec<i32> = [node_fd, net_fd].iter().filter_map(|&x| x).collect();
- // The child must never outlive us. An orphaned Mycelium keeps running with our
- // key; the next one we start then shares its address, and replies (including
- // Mycelium's own message ACKs) go to whichever instance the routers picked,
- // leaving our messages pending forever. Drop only runs on a clean shutdown, so
- // let the kernel signal the child when we die, however we die.
- let parent_pid = std::process::id() as libc::pid_t;
- unsafe {
- cmd.pre_exec(move || {
- for &fd in &fds_to_inherit {
- libc::fcntl(fd, libc::F_SETFD, 0);
- }
- // Own process group: a Ctrl-C in the terminal goes to continuum only,
- // which then stops Mycelium itself, instead of both exiting at once (which
- // shows up as "Mycelium runtime process exited").
- if libc::setpgid(0, 0) != 0 {
- return Err(std::io::Error::last_os_error());
- }
- #[cfg(target_os = "linux")]
- {
- if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) != 0 {
- return Err(std::io::Error::last_os_error());
- }
- // We may have died between fork and prctl.
- if libc::getppid() != parent_pid {
- libc::_exit(1);
- }
- }
- Ok(())
- });
- }
-
- cmd.arg("--key-file")
- .arg(&key_file_path)
- .arg("--tcp-listen-port")
- .arg(config.tcp_port.to_string())
- .arg("--disable-quic")
- .arg("--api-addr")
- .arg(&api_addr)
- .arg("--jsonrpc-addr")
- .arg(&rpc_addr)
- .arg("--no-tun");
-
- if let Some(ref discovery_port) = config.discovery_port {
- cmd.arg("--peer-discovery-port")
- .arg(discovery_port.to_string());
- } else {
- cmd.arg("--disable-peer-discovery");
- }
-
- if let Some(ref net_name) = config.private_network {
- cmd.arg("--network-name").arg(net_name);
- }
- if let Some(ref net_key_path) = net_key_file_path {
- cmd.arg("--network-key-file").arg(net_key_path);
- }
-
- if !peers.is_empty() {
- cmd.arg("--peers").arg(peers.join(" "));
- }
-
- cmd.env("RUST_LOG", "info");
-
- cmd.stdout(Stdio::piped());
- cmd.stderr(Stdio::piped());
-
- let mut child = cmd
- .spawn()
- .context("Failed to execute mycelium process daemon")?;
-
- // 1. FIXED: Pass-through streaming for child process STDOUT
- if let Some(mut stdout) = child.stdout.take() {
- tokio::spawn(async move {
- let mut host_stdout = tokio::io::stdout();
- let _ = tokio::io::copy(&mut stdout, &mut host_stdout).await;
- });
- }
-
- // 2. FIXED: Pass-through streaming for child process STDERR
- if let Some(mut stderr) = child.stderr.take() {
- tokio::spawn(async move {
- let mut host_stderr = tokio::io::stderr();
- let _ = tokio::io::copy(&mut stderr, &mut host_stderr).await;
- });
- }
-
- let node = Self {
- child,
- config,
- _key_file_handle: key_file_handle,
- _net_key_file_handle: net_key_file_handle,
- };
-
- Ok(node)
- }
-
- fn resolve_key_strategy(
- desc_tag: &str,
- source: KeySource,
- parent_pid: u32,
- ) -> Result<(Option<File>, PathBuf, Option<i32>)> {
- match source {
- KeySource::Raw(raw_bytes) => {
- let (handle, fd) = Self::create_ephemeral_file(desc_tag, raw_bytes)?;
- let pseudo_path = PathBuf::from(format!("/proc/{}/fd/{}", parent_pid, fd));
- Ok((Some(handle), pseudo_path, Some(fd)))
- }
- KeySource::Path(existing_path) => Ok((None, existing_path, None)),
- }
- }
-
- fn find_available_port() -> Result<u16> {
- let listener = TcpListener::bind("127.0.0.1:0")?;
- let addr = listener.local_addr()?;
- Ok(addr.port())
- }
-
- fn create_ephemeral_file(desc_tag: &str, raw_key: [u8; 32]) -> Result<(File, i32)> {
- #[cfg(target_os = "linux")]
- {
- let memfd_name = std::ffi::CString::new(desc_tag)?;
- let fd = unsafe { libc::memfd_create(memfd_name.as_ptr(), 0) };
- if fd < 0 {
- return Err(anyhow!(
- "memfd_create failed natively inside Linux kernel for tag: {}",
- desc_tag
- ));
- }
- let mut file = unsafe { File::from_raw_fd(fd) };
- file.write_all(&raw_key)?;
- file.flush()?;
- Ok((file, fd))
- }
-
- // Unreachable: `spawn` refuses to run elsewhere.
- #[cfg(not(target_os = "linux"))]
- {
- let _ = (desc_tag, raw_key);
- bail!("In-memory key files need Linux")
- }
- }
-
- pub async fn wait_for_exit(&mut self) -> std::io::Result<std::process::ExitStatus> {
- self.child.wait().await
- }
-}
-
-impl Drop for SupervisedMycelium {
- fn drop(&mut self) {
- // The memfd key files go away with their handles.
- let _ = self.child.start_kill();
- }
-}
-
-fn is_pubkey(s: &str) -> bool {
- s.len() == 64 && s.chars().all(|c| c.is_ascii_hexdigit())
-}
-
-pub struct MyceliumTransport {
- // Wrap your zero-allocation lifecycle supervisor directly inside the driver plane
- pub setup: tokio::sync::Mutex<MyceliumSetup>,
- pub client: reqwest::Client,
- pub poll_client: reqwest::Client,
- pub messages_endpoint: Url,
- pubkey_basepath: Url,
- local_id: String,
-}
-
-impl MyceliumTransport {
- pub async fn connect(setup: MyceliumSetup) -> Result<Self> {
- let rest_url = setup.rest_url.clone();
-
- let client = reqwest::Client::builder()
- .timeout(Duration::from_secs(10))
- .build()?;
- let poll_client = reqwest::Client::builder()
- .timeout(Duration::from_secs(MYCELIUM_POLL_TIMEOUT_SECS + 30))
- .build()?;
-
- let admin: MyceliumAdminInfo = client
- .get(rest_url.join("/api/v1/admin")?)
- .send()
- .await?
- .error_for_status()?
- .json()
- .await
- .context("Querying local Mycelium identity")?;
-
- let transport = Self {
- setup: tokio::sync::Mutex::new(setup),
- client,
- poll_client,
- messages_endpoint: rest_url.join("/api/v1/messages")?,
- pubkey_basepath: rest_url.join("/api/v1/pubkey/")?,
- local_id: admin.node_pubkey,
- };
- transport.discard_backlog().await;
- Ok(transport)
- }
-
- /// Drops what Mycelium queued for us before we started. A Mycelium that outlives us
- /// (an external one, or across our restart) keeps accepting messages meant for us:
- /// HELLO retries, keepalives, sync messages of sessions that ended with our previous
- /// run. Processing them would only replay stale handshakes. Everything that still
- /// matters is resent: peers retry their HELLOs, and sync starts over after the
- /// handshake. Runs before we send anything, so no answer to us is dropped.
- ///
- /// Bounded in time, not count: the backlog is whatever piled up while we were away,
- /// which a local loop drains in well under a second even for large swarms. Whatever
- /// is still arriving after that is current traffic (peers that don't know we're
- /// back), and it is processed normally, exactly as without this step.
- async fn discard_backlog(&self) {
- const DRAIN_FOR: Duration = Duration::from_secs(2);
- let deadline = std::time::Instant::now() + DRAIN_FOR;
- let mut discarded = 0u64;
- while std::time::Instant::now() < deadline {
- let response = self
- .client
- .get(self.messages_endpoint.clone())
- .query(&[("timeout", 0)])
- .query(&[("topic", encode_topic(TOPIC_PREFIX))])
- .send()
- .await;
- match response {
- Ok(response) if response.status() == StatusCode::NO_CONTENT => break,
- Ok(response) if response.status().is_success() => discarded += 1,
- Ok(response) => {
- tracing::warn!(status = %response.status(), "Couldn't drain the Mycelium inbox");
- break;
- }
- Err(error) => {
- tracing::warn!(error = %error, "Couldn't drain the Mycelium inbox");
- break;
- }
- }
- }
- if discarded > 0 {
- tracing::info!(
- discarded,
- "Discarded messages Mycelium queued for us before we started"
- );
- }
- }
-
- async fn lookup_pubkey(&self, ip: &std::net::Ipv6Addr) -> Result<String> {
- #[derive(Deserialize)]
- struct PublicKeyResponse {
- #[serde(rename = "publicKey")]
- public_key: String,
- }
-
- let response: PublicKeyResponse = self
- .client
- .get(self.pubkey_basepath.join(&ip.to_string())?)
- .send()
- .await?
- .error_for_status()?
- .json()
- .await?;
- Ok(response.public_key)
- }
-}
-
-#[async_trait]
-impl Transport for MyceliumTransport {
- fn local_id(&self) -> &str {
- &self.local_id
- }
-
- async fn resolve_seed(&self, hint: &str) -> Result<String> {
- if is_pubkey(hint) {
- return Ok(hint.to_owned());
- }
-
- if let Ok(ip) = hint.parse::<std::net::Ipv6Addr>() {
- return self.lookup_pubkey(&ip).await;
- }
-
- let ip = tokio::time::timeout(Duration::from_secs(10), tokio::net::lookup_host((hint, 0)))
- .await
- .context("DNS lookup timed out")??
- .find_map(|addr| match addr.ip() {
- std::net::IpAddr::V6(ip) => Some(ip),
- _ => None,
- })
- .ok_or_else(|| anyhow!("{hint}: no AAAA record found"))?;
-
- self.lookup_pubkey(&ip).await
- }
-
- fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- encode_message(peer, message)
- }
-
- async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> {
- tracing::trace!(payload_bytes = message.body.len(), "Sending message");
- let response = self
- .client
- .post(self.messages_endpoint.clone())
- .header(reqwest::header::CONTENT_TYPE, "application/json")
- .body(message.body.clone())
- .send()
- .await?;
-
- let status = response.status();
- if !status.is_success() {
- tracing::error!(status = %status, "Mycelium rejected outbound message");
- return Err(TransportError::MyceliumRejected {
- peer: message.peer.clone(),
- status,
- }
- .into());
- }
-
- // Mycelium's message ID, for GET /api/v1/messages/status/{id} when chasing
- // a message that never arrived.
- if tracing::enabled!(tracing::Level::DEBUG) {
- let Ok(body) = response
- .bytes()
- .await
- .map_err(anyhow::Error::from)
- .and_then(|bytes| Ok(serde_json::from_slice::<serde_json::Value>(&bytes)?))
- else {
- tracing::debug!(to = %message.peer, "Sent message (couldn't decode response from Mycelium REST API)");
- return Ok(());
- };
- let Ok(outbound) = serde_json::from_slice::<OutboundMessage>(&message.body) else {
- tracing::debug!(to = %message.peer, body_len = %message.body.len(), "Sent message (couldn't parse outbound message)");
- return Ok(());
- };
- let Ok(parsed) = decode_payload(&outbound.topic, &outbound.payload) else {
- tracing::debug!(to = %message.peer, message = ?outbound, "Sent message (couldn't decode outbound message)");
- return Ok(());
- };
- let Some(id) = body.get("id").and_then(|id| id.as_str()) else {
- tracing::debug!(to = %message.peer, message = ?parsed, "Sent message (couldn't read id from Mycelium REST API response)");
- return Ok(());
- };
- tracing::debug!(to = %message.peer, message = ?parsed, id = %id, "Sent message");
- }
- Ok(())
- }
-
- async fn start_consumer(&self, handler: MessageHandler) -> Result<()> {
- tracing::info!(
- endpoint = self.messages_endpoint.as_str(),
- "Starting Mycelium message consumer"
- );
-
- loop {
- tracing::trace!("Polling Mycelium inbox");
-
- let response = match self
- .poll_client
- .get(self.messages_endpoint.clone())
- .query(&[("timeout", MYCELIUM_POLL_TIMEOUT_SECS)])
- .query(&[("topic", encode_topic(TOPIC_PREFIX))])
- .send()
- .await
- {
- Ok(resp) => resp,
- Err(error) => {
- tracing::warn!(error = %error, "Mycelium inbox poll failed");
- tokio::time::sleep(Duration::from_millis(250)).await;
- continue;
- }
- };
-
- let status = response.status();
-
- if status == StatusCode::NO_CONTENT {
- tracing::trace!("Mycelium inbox is empty");
- tokio::time::sleep(EMPTY_QUEUE_DELAY).await;
- continue;
- }
-
- if !status.is_success() {
- tracing::warn!(status = %status, "Mycelium inbox returned unsuccessful HTTP status");
- tokio::time::sleep(Duration::from_millis(500)).await;
- continue;
- }
-
- let raw_bytes = match response.bytes().await {
- Ok(bytes) => bytes,
- Err(error) => {
- tracing::warn!(error = %error, "Failed to read raw Mycelium response stream");
- tokio::time::sleep(Duration::from_millis(100)).await;
- continue;
- }
- };
-
- let inbound_message = match serde_json::from_slice::<InboundMessage>(&raw_bytes) {
- Ok(msg) => msg,
- Err(error) => {
- tracing::error!(
- error = %error,
- raw_payload = %String::from_utf8_lossy(&raw_bytes),
- "Failed to decode inbound Mycelium message structure"
- );
- tokio::time::sleep(Duration::from_millis(500)).await;
- continue;
- }
- };
-
- match parse_message(&inbound_message) {
- Ok((sender, parsed)) => {
- tracing::debug!(
- from = %inbound_message.src_pk,
- message = ?parsed,
- id = %inbound_message.id,
- //payload_bytes = raw_bytes.len(),
- "Received message"
- );
- let payload_len = base64_decoded_len(&inbound_message.payload);
- if let Err(error) = handler(sender, parsed, payload_len).await {
- tracing::warn!(error = %format!("{error:#}"), "Failed to process inbound message");
- }
- }
- Err(transport_error) => {
- tracing::error!(
- error = %transport_error,
- topic = %inbound_message.topic,
- "Dropping poisoned packet: parsing failed gracefully"
- );
- // No sleep required: we successfully popped a malformed packet and want to process the next one immediately
- }
- }
- }
- }
-
- async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> {
- let mut guard = self.setup.lock().await;
- guard.wait_for_exit().await
- }
-}
-
-/// A message as the Mycelium REST API takes it (the JSON body of an outbound message).
-/// Also the simulator's wire format (`transport::sim`), so tests go through the same
-/// encoding.
-pub(crate) fn encode_message(peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- let (outbound, payload_len) = format_message(peer, message)?;
- Ok(EncodedMessage {
- peer: peer.to_owned(),
- body: serde_json::to_vec(&outbound)?.into(),
- kind: message.kind(),
- payload_len,
- })
-}
-
-/// The message in a body made by [`encode_message`].
-#[cfg(test)]
-pub(crate) fn decode_message(body: &[u8]) -> Result<ParsedMessage> {
- let outbound: OutboundMessage = serde_json::from_slice(body)?;
- Ok(decode_payload(&outbound.topic, &outbound.payload)?)
-}
-
-fn encode_topic(topic: &str) -> String {
- BASE64.encode(topic.as_bytes())
-}
-
-fn decode_topic(topic: &str) -> Result<String, TransportError> {
- let bytes = BASE64.decode(topic)?;
- Ok(String::from_utf8(bytes)?)
-}
-
-fn encode_payload(payload: &Vec<u8>) -> String {
- BASE64.encode(payload)
-}
-
-/// Length of what padded base64 `encoded` decodes to.
-fn base64_decoded_len(encoded: &str) -> usize {
- let padding = encoded.bytes().rev().take_while(|&b| b == b'=').count();
- (encoded.len() / 4 * 3).saturating_sub(padding)
-}
-
-fn decode_payload(topic_encoded: &str, payload: &str) -> Result<ParsedMessage, TransportError> {
- let topic: &str = &decode_topic(topic_encoded)?;
- let bytes = BASE64.decode(payload)?;
- match topic {
- TOPIC_HELLO => {
- let msg = bytes_to_hello_request(&bytes)?;
- Ok(ParsedMessage::Hello(msg))
- }
- TOPIC_HELLO_REPLY => {
- let msg = bytes_to_hello_reply(&bytes)?;
- Ok(ParsedMessage::HelloReply(msg))
- }
- TOPIC_SYNC => {
- let msg = bytes_to_sync_envelope(&bytes)?;
- Ok(ParsedMessage::Sync(msg))
- }
- TOPIC_PING => Ok(ParsedMessage::Ping(bytes_to_keepalive(&bytes)?)),
- TOPIC_PONG => Ok(ParsedMessage::Pong(bytes_to_keepalive(&bytes)?)),
- TOPIC_BLOB => Ok(ParsedMessage::Blob(bytes_to_blob(&bytes)?)),
- t => Err(TransportError::UnknownTopic(t.to_string())),
- }
-}
-
-fn parse_message(message: &InboundMessage) -> Result<(String, ParsedMessage), TransportError> {
- let sender = message.src_pk.clone();
-
- if sender.is_empty() {
- return Err(TransportError::NoSender);
- }
-
- let parsed = decode_payload(&message.topic, &message.payload)?;
-
- Ok((sender, parsed))
-}
-
-/// The message for Mycelium's API, and the length of its payload.
-fn format_message(
- recipient: &str,
- message: &ParsedMessage,
-) -> Result<(OutboundMessage, usize), TransportError> {
- let (topic, payload_bytes) = match message {
- ParsedMessage::Hello(req) => (TOPIC_HELLO, hello_request_to_bytes(req)?),
- ParsedMessage::HelloReply(res) => (TOPIC_HELLO_REPLY, hello_reply_to_bytes(res)?),
- ParsedMessage::Sync(env) => (TOPIC_SYNC, sync_envelope_to_bytes(env)?),
- ParsedMessage::Ping(k) => (TOPIC_PING, keepalive_to_bytes(k)?),
- ParsedMessage::Pong(k) => (TOPIC_PONG, keepalive_to_bytes(k)?),
- ParsedMessage::Blob(blob) => (TOPIC_BLOB, blob_to_bytes(blob)?),
- };
- let outbound = OutboundMessage {
- dst: Destination {
- pk: recipient.to_owned(),
- },
- topic: encode_topic(topic),
- payload: encode_payload(&payload_bytes),
- };
- Ok((outbound, payload_bytes.len()))
-}
-
-// =====================================================================
-// 1. WIRE SHADOWS
-// =====================================================================
-//
-// rmp_serde encodes structs as arrays. Fields added later are appended at the end and
-// marked #[serde(default)], so a message from a node that predates them still decodes
-// (an absent epoch is the genesis epoch).
-
-/// Epoch on the wire. The default (generation 0, empty id) is the genesis epoch.
-#[derive(Serialize, Deserialize, Default)]
-struct EpochShadow {
- generation: u64,
- id: ByteBuf,
- /// The sender's epoch changes in its current run (`SyncEnvelope::epoch_seq`).
- #[serde(default)]
- seq: u32,
-}
-
-fn epoch_to_shadow(epoch: &Epoch, seq: u32) -> EpochShadow {
- EpochShadow {
- generation: epoch.generation,
- id: ByteBuf::from(epoch.id.to_vec()),
- seq,
- }
-}
-
-fn epoch_from_shadow(shadow: &EpochShadow) -> Result<Epoch, TransportError> {
- if shadow.generation == 0 && shadow.id.is_empty() {
- return Ok(Epoch::GENESIS);
- }
- let id = shadow
- .id
- .as_slice()
- .try_into()
- .map_err(|_| TransportError::InvalidPayload("epoch id must be 32 bytes".into()))?;
- Ok(Epoch {
- generation: shadow.generation,
- id,
- })
-}
-
-/// Either `heads` (listed) or `digest` (32 bytes, `heads` empty) is set.
-#[derive(Serialize, Deserialize)]
-struct KeepaliveShadow {
- heads: Vec<ByteBuf>,
- digest: ByteBuf,
- epoch: EpochShadow,
- ready: bool,
- syncs_with_you: bool,
-}
-
-fn keepalive_to_bytes(msg: &Keepalive) -> Result<Vec<u8>, rmp_serde::encode::Error> {
- let (heads, digest) = match &msg.heads {
- HeadsSummary::Heads(heads) => (
- heads
- .iter()
- .map(|head| ByteBuf::from(head.0.to_vec()))
- .collect(),
- ByteBuf::new(),
- ),
- HeadsSummary::Digest(digest) => (Vec::new(), ByteBuf::from(digest.to_vec())),
- };
- rmp_serde::to_vec(&KeepaliveShadow {
- heads,
- digest,
- epoch: epoch_to_shadow(&msg.epoch, msg.epoch_seq),
- ready: msg.ready,
- syncs_with_you: msg.syncs_with_you,
- })
-}
-
-fn bytes_to_keepalive(bytes: &[u8]) -> Result<Keepalive, TransportError> {
- let shadow: KeepaliveShadow = rmp_serde::from_slice(bytes)?;
- let heads = if shadow.digest.is_empty() {
- if shadow.heads.len() > MAX_LISTED_HEADS {
- return Err(TransportError::InvalidPayload(format!(
- "more than {MAX_LISTED_HEADS} heads listed"
- )));
- }
- let heads = shadow
- .heads
- .iter()
- .map(|head| ChangeHash::try_from(head.as_slice()))
- .collect::<Result<Vec<_>, _>>()
- .map_err(|_| TransportError::InvalidPayload("a head must be 32 bytes".into()))?;
- HeadsSummary::Heads(heads)
- } else {
- let digest =
- shadow.digest.as_slice().try_into().map_err(|_| {
- TransportError::InvalidPayload("heads digest must be 32 bytes".into())
- })?;
- HeadsSummary::Digest(digest)
- };
- Ok(Keepalive {
- heads,
- epoch: epoch_from_shadow(&shadow.epoch)?,
- epoch_seq: shadow.epoch.seq,
- ready: shadow.ready,
- syncs_with_you: shadow.syncs_with_you,
- })
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct HelloRequestShadow {
- swarm_id: Option<String>,
- epoch: EpochShadow,
- auth: ByteBuf,
- ready: bool,
- run_id: ByteBuf,
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct HelloReplyShadow<'a> {
- // Cow allows zero-copy references when saving, but creates an owned String when loading
- pub swarm_id: Cow<'a, str>,
- #[serde(default)]
- pub peers: Vec<String>,
- #[serde(default)]
- epoch: EpochShadow,
- #[serde(default)]
- auth: ByteBuf,
- ready: bool,
- run_id: ByteBuf,
- away_ms: Option<u64>,
- sender_run_id: ByteBuf,
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct SyncEnvelopeShadow<'a> {
- pub swarm_id: Cow<'a, str>,
- #[serde(with = "automerge_msg_codec")]
- pub message: automerge::sync::Message,
- #[serde(default)]
- epoch: EpochShadow,
- /// Absent from nodes that predate it: never counts as ready.
- #[serde(default)]
- ready: bool,
-}
-
-// =====================================================================
-// 2. CUSTOM CODEC
-// =====================================================================
-mod automerge_msg_codec {
- use super::*;
- use automerge::sync::Message;
-
- pub fn serialize<S>(msg: &Message, serializer: S) -> Result<S::Ok, S::Error>
- where
- S: Serializer,
- {
- let bytes = msg.clone().encode();
- Bytes::new(&bytes).serialize(serializer)
- }
-
- pub fn deserialize<'de, D>(deserializer: D) -> Result<Message, D::Error>
- where
- D: Deserializer<'de>,
- {
- let byte_buf = ByteBuf::deserialize(deserializer)?;
- Message::decode(&byte_buf)
- .map_err(|e| serde::de::Error::custom(format!("Automerge decode failed: {:?}", e)))
- }
-}
-
-// =====================================================================
-// 3. CLEAN CONVERSION INTERFACE
-// =====================================================================
-
-// --- HelloRequest ---
-pub fn hello_request_to_bytes(msg: &HelloRequest) -> Result<Vec<u8>, rmp_serde::encode::Error> {
- rmp_serde::to_vec(&HelloRequestShadow {
- swarm_id: msg.swarm_id.clone(),
- epoch: epoch_to_shadow(&msg.epoch, msg.epoch_seq),
- auth: ByteBuf::from(msg.auth.clone()),
- ready: msg.ready,
- run_id: ByteBuf::from(msg.run_id.to_vec()),
- })
-}
-pub fn bytes_to_hello_request(bytes: &[u8]) -> Result<HelloRequest, TransportError> {
- let shadow: HelloRequestShadow = rmp_serde::from_slice(bytes)?;
- Ok(HelloRequest {
- swarm_id: shadow.swarm_id,
- epoch: epoch_from_shadow(&shadow.epoch)?,
- epoch_seq: shadow.epoch.seq,
- auth: shadow.auth.into_vec(),
- ready: shadow.ready,
- run_id: run_id_from_shadow(&shadow.run_id)?,
- })
-}
-
-fn run_id_from_shadow(bytes: &ByteBuf) -> Result<crate::transport::RunId, TransportError> {
- bytes
- .as_slice()
- .try_into()
- .map_err(|_| TransportError::InvalidPayload("run id must be 16 bytes".into()))
-}
-
-// --- HelloReply ---
-pub fn hello_reply_to_bytes(msg: &HelloReply) -> Result<Vec<u8>, rmp_serde::encode::Error> {
- let shadow = HelloReplyShadow {
- swarm_id: Cow::Borrowed(&msg.swarm_id),
- peers: msg.peers.clone(), // Vec must be cloned for serialization wrapper matching
- epoch: epoch_to_shadow(&msg.epoch, msg.epoch_seq),
- auth: ByteBuf::from(msg.auth.clone()),
- ready: msg.ready,
- run_id: ByteBuf::from(msg.run_id.to_vec()),
- away_ms: msg.away_ms,
- sender_run_id: ByteBuf::from(msg.sender_run_id.to_vec()),
- };
- rmp_serde::to_vec(&shadow)
-}
-pub fn bytes_to_hello_reply(bytes: &[u8]) -> Result<HelloReply, TransportError> {
- let shadow: HelloReplyShadow = rmp_serde::from_slice(bytes)?;
- Ok(HelloReply {
- swarm_id: shadow.swarm_id.into_owned(),
- peers: shadow.peers,
- epoch: epoch_from_shadow(&shadow.epoch)?,
- epoch_seq: shadow.epoch.seq,
- auth: shadow.auth.into_vec(),
- ready: shadow.ready,
- run_id: run_id_from_shadow(&shadow.run_id)?,
- away_ms: shadow.away_ms,
- sender_run_id: run_id_from_shadow(&shadow.sender_run_id)?,
- })
-}
-
-// --- SyncEnvelope ---
-pub fn sync_envelope_to_bytes(msg: &SyncEnvelope) -> Result<Vec<u8>, rmp_serde::encode::Error> {
- let shadow = SyncEnvelopeShadow {
- swarm_id: Cow::Borrowed(&msg.swarm_id),
- message: msg.message.clone(), // Clone required by automerge_msg_codec ownership rules
- epoch: epoch_to_shadow(&msg.epoch, msg.epoch_seq),
- ready: msg.ready,
- };
- rmp_serde::to_vec(&shadow)
-}
-pub fn bytes_to_sync_envelope(bytes: &[u8]) -> Result<SyncEnvelope, TransportError> {
- let shadow: SyncEnvelopeShadow = rmp_serde::from_slice(bytes)?;
- Ok(SyncEnvelope {
- swarm_id: shadow.swarm_id.into_owned(),
- epoch: epoch_from_shadow(&shadow.epoch)?,
- epoch_seq: shadow.epoch.seq,
- message: shadow.message,
- ready: shadow.ready,
- })
-}
-
-// --- BlobMessage ---
-#[derive(Serialize, Deserialize)]
-enum BlobIdShadow {
- Change(ByteBuf),
- Content(ByteBuf),
-}
-
-#[derive(Serialize, Deserialize)]
-enum BlobBodyShadow {
- Request {
- chunks: Vec<u32>,
- },
- Manifest {
- size: u64,
- chunk_size: u32,
- hashes: Vec<ByteBuf>,
- },
- Chunk {
- index: u32,
- data: ByteBuf,
- },
- Missing,
-}
-
-#[derive(Serialize, Deserialize)]
-struct BlobShadow {
- epoch: EpochShadow,
- id: BlobIdShadow,
- body: BlobBodyShadow,
-}
-
-fn blob_to_bytes(msg: &BlobMessage) -> Result<Vec<u8>, rmp_serde::encode::Error> {
- let id = match msg.id {
- BlobId::Change(hash) => BlobIdShadow::Change(ByteBuf::from(hash.0.to_vec())),
- BlobId::Content(hash) => BlobIdShadow::Content(ByteBuf::from(hash.to_vec())),
- };
- let body = match &msg.body {
- BlobBody::Request { chunks } => BlobBodyShadow::Request {
- chunks: chunks.clone(),
- },
- BlobBody::Manifest {
- size,
- chunk_size,
- hashes,
- } => BlobBodyShadow::Manifest {
- size: *size,
- chunk_size: *chunk_size,
- hashes: hashes
- .iter()
- .map(|hash| ByteBuf::from(hash.to_vec()))
- .collect(),
- },
- BlobBody::Chunk { index, data } => BlobBodyShadow::Chunk {
- index: *index,
- data: ByteBuf::from(data.to_vec()),
- },
- BlobBody::Missing => BlobBodyShadow::Missing,
- };
- rmp_serde::to_vec(&BlobShadow {
- epoch: epoch_to_shadow(&msg.epoch, msg.epoch_seq),
- id,
- body,
- })
-}
-
-fn hash32(bytes: &[u8], what: &str) -> Result<[u8; 32], TransportError> {
- bytes
- .try_into()
- .map_err(|_| TransportError::InvalidPayload(format!("{what} must be 32 bytes")))
-}
-
-fn bytes_to_blob(bytes: &[u8]) -> Result<BlobMessage, TransportError> {
- let shadow: BlobShadow = rmp_serde::from_slice(bytes)?;
- let id = match shadow.id {
- BlobIdShadow::Change(hash) => BlobId::Change(ChangeHash(hash32(&hash, "a change hash")?)),
- BlobIdShadow::Content(hash) => BlobId::Content(hash32(&hash, "a content hash")?),
- };
- let body = match shadow.body {
- BlobBodyShadow::Request { chunks } => BlobBody::Request { chunks },
- BlobBodyShadow::Manifest {
- size,
- chunk_size,
- hashes,
- } => BlobBody::Manifest {
- size,
- chunk_size,
- hashes: hashes
- .iter()
- .map(|hash| hash32(hash, "a chunk hash"))
- .collect::<Result<_, _>>()?,
- },
- BlobBodyShadow::Chunk { index, data } => BlobBody::Chunk {
- index,
- data: data.into_vec().into(),
- },
- BlobBodyShadow::Missing => BlobBody::Missing,
- };
- Ok(BlobMessage {
- epoch: epoch_from_shadow(&shadow.epoch)?,
- epoch_seq: shadow.epoch.seq,
- id,
- body,
- })
-}
-
-#[derive(Error, Debug)]
-pub enum TransportError {
- #[error("Mycelium delivered a message without a sender public key")]
- NoSender,
-
- #[error("Base64 decoding failed: {0}")]
- Base64Decode(#[from] base64::DecodeError),
-
- #[error("UTF-8 decoding failed: {0}")]
- Utf8Decode(#[from] std::string::FromUtf8Error),
-
- #[error("MessagePack serialization failed: {0}")]
- MessagePackEncode(#[from] rmp_serde::encode::Error),
-
- #[error("MessagePack deserialization failed: {0}")]
- MessagePackDecode(#[from] rmp_serde::decode::Error),
-
- #[error("Unknown topic: {0}")]
- UnknownTopic(String),
-
- #[error("Invalid payload: {0}")]
- InvalidPayload(String),
-
- #[error("Network HTTP request failed: {0}")]
- Http(#[from] reqwest::Error),
-
- #[error("Mycelium server rejected outbound message to {peer} (HTTP {status})")]
- MyceliumRejected {
- peer: String,
- status: reqwest::StatusCode,
- },
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- /// Mycelium splits messages into chunks of this size.
- const MYCELIUM_CHUNK: usize = 1300;
-
- #[test]
- fn largest_keepalive_fits_in_one_chunk() {
- let keepalive = Keepalive {
- heads: HeadsSummary::Heads(
- (0..MAX_LISTED_HEADS as u8)
- .map(|n| ChangeHash([n; 32]))
- .collect(),
- ),
- epoch: Epoch {
- generation: u64::MAX,
- id: [0xff; 32],
- },
- epoch_seq: u32::MAX,
- ready: true,
- syncs_with_you: true,
- };
- let payload = keepalive_to_bytes(&keepalive).unwrap();
- // Mycelium's message data: a length byte, the topic, then our payload.
- let size = 1 + TOPIC_PONG.len().max(TOPIC_PING.len()) + payload.len();
- assert!(
- size + 50 <= MYCELIUM_CHUNK,
- "keepalive with {MAX_LISTED_HEADS} heads takes {size} bytes"
- );
- assert_eq!(bytes_to_keepalive(&payload).unwrap().heads, keepalive.heads);
- }
-
- #[test]
- fn blob_messages_round_trip() {
- let epoch = Epoch {
- generation: 3,
- id: [7; 32],
- };
- let id = BlobId::Change(ChangeHash([9; 32]));
- let chunk = BlobMessage {
- epoch,
- epoch_seq: 4,
- id,
- body: BlobBody::Chunk {
- index: 5,
- data: vec![1u8; 64 * 1024].into(),
- },
- };
- let bytes = blob_to_bytes(&chunk).unwrap();
- // A 64 KiB chunk costs only a few bytes of framing.
- assert!(bytes.len() < 64 * 1024 + 128, "{} bytes", bytes.len());
- let decoded = bytes_to_blob(&bytes).unwrap();
- assert_eq!(decoded.id, id);
- assert_eq!(decoded.epoch, epoch);
- assert_eq!(decoded.epoch_seq, 4);
- assert!(
- matches!(decoded.body, BlobBody::Chunk { index: 5, ref data } if data.len() == 64 * 1024)
- );
-
- let manifest = BlobMessage {
- epoch,
- epoch_seq: 4,
- id,
- body: BlobBody::Manifest {
- size: 100,
- chunk_size: 64,
- hashes: vec![[1; 32], [2; 32]],
- },
- };
- let decoded = bytes_to_blob(&blob_to_bytes(&manifest).unwrap()).unwrap();
- assert!(matches!(
- decoded.body,
- BlobBody::Manifest { size: 100, chunk_size: 64, ref hashes } if hashes.len() == 2
- ));
- }
-
- #[test]
- fn listed_heads_equal_their_digest() {
- let heads: Vec<_> = (0..3u8).map(|n| ChangeHash([n; 32])).collect();
- let listed = HeadsSummary::of(heads.clone());
- let digest = HeadsSummary::Digest(listed.digest());
- assert_eq!(listed, digest);
- assert_ne!(listed, HeadsSummary::of(heads[..2].to_vec()));
- }
-}
diff --git a/src/transport/mycelium.rs.orig b/src/transport/mycelium.rs.orig
deleted file mode 100644
index 9d1fb90..0000000
--- a/src/transport/mycelium.rs.orig
+++ /dev/null
@@ -1,975 +0,0 @@
-use anyhow::{Context, Result, anyhow, bail};
-use async_trait::async_trait;
-use automerge::ChangeHash;
-use base64::engine::{Engine, general_purpose::STANDARD as BASE64};
-use reqwest::StatusCode;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
-use serde_bytes::{ByteBuf, Bytes};
-use std::borrow::Cow;
-use std::boxed::Box;
-use std::fs::File;
-use std::io::Write;
-use std::net::TcpListener;
-use std::os::unix::io::FromRawFd;
-use std::path::PathBuf;
-use std::process::Stdio;
-use std::time::Duration;
-use thiserror::Error;
-use tokio::process::{Child, Command};
-use url::Url;
-
-use crate::transport::EncodedMessage;
-use crate::transport::{Epoch, HelloReply, HelloRequest, SyncEnvelope};
-use crate::transport::{HeadsSummary, Keepalive, MAX_LISTED_HEADS};
-use crate::transport::{MessageHandler, ParsedMessage, Transport};
-
-const MYCELIUM_POLL_TIMEOUT_SECS: u64 = 300;
-
-const TOPIC_PREFIX: &str = "continuum.v1.";
-const TOPIC_HELLO: &str = "continuum.v1.hello";
-const TOPIC_HELLO_REPLY: &str = "continuum.v1.hello-reply";
-const TOPIC_SYNC: &str = "continuum.v1.sync";
-const TOPIC_PING: &str = "continuum.v1.ping";
-const TOPIC_PONG: &str = "continuum.v1.pong";
-
-const EMPTY_QUEUE_DELAY: Duration = Duration::from_millis(25);
-
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct InboundMessage {
- id: String,
- src_pk: String,
- topic: String,
- payload: String,
-}
-
-// Deserialize only for debugging
-#[derive(Debug, Serialize, Deserialize)]
-struct OutboundMessage {
- dst: Destination,
- topic: String,
- payload: String,
-}
-
-// Deserialize only for debugging
-#[derive(Debug, Serialize, Deserialize)]
-struct Destination {
- pk: String,
-}
-
-/// Represents where a 32-byte cryptographic key originates.
-#[derive(Debug, Clone)]
-pub enum KeySource {
- Raw([u8; 32]),
- Path(PathBuf),
-}
-
-/// Atomic structure ensuring private network credentials are always handled together.
-#[derive(Debug, Clone)]
-pub struct PrivateNetworkConfig {
- pub name: String,
- pub key: KeySource,
-}
-
-/// Comprehensive parameters required to spawn or target an instance.
-#[derive(Debug, Clone)]
-pub struct MyceliumSpawnConfig {
- pub rest_url_opt: Option<Url>,
- pub key_file_opt: Option<PathBuf>,
- pub network_key_file: Option<PathBuf>,
- pub network_name: String,
- pub tcp_port: Option<u16>,
- pub api_port: Option<u16>,
- pub rpc_port: Option<u16>,
- pub discovery_port: Option<u16>,
- pub peers: Vec<String>,
-}
-
-impl Default for MyceliumSpawnConfig {
- fn default() -> Self {
- Self {
- rest_url_opt: None,
- key_file_opt: None,
- network_key_file: None,
- network_name: "something".to_string(),
- tcp_port: None,
- api_port: None,
- rpc_port: None,
- discovery_port: None,
- peers: vec!["tls://127.0.0.1:9651".to_string()],
- }
- }
-}
-
-pub enum MyceliumRuntime {
- Supervised(Box<SupervisedMycelium>),
- External,
-}
-
-pub struct MyceliumSetup {
- pub runtime: MyceliumRuntime,
- pub rest_url: Url,
-}
-
-impl MyceliumSetup {
- /// Evaluates the configuration payload and initializes the chosen execution strategy.
- pub async fn bootstrap(config: MyceliumSpawnConfig) -> Result<Self> {
- // TODO warn for all ignored options if rest_url_opt is set
- if let Some(url) = config.rest_url_opt {
- tracing::info!(
- "Configured to interface with an external standalone Mycelium instance at: {}",
- url
- );
- return Ok(Self {
- runtime: MyceliumRuntime::External,
- rest_url: url,
- });
- }
-
- tracing::info!(
- "Configured to initialize an embedded user-space Mycelium background worker"
- );
-
- let key = match config.key_file_opt {
- Some(key_file) => KeySource::Path(key_file),
- None => {
- let mut bytes = [0u8; 32];
- getrandom::fill(&mut bytes)?;
- KeySource::Raw(bytes)
- }
- };
-
- let private_net_config = config
- .network_key_file
- .map(|net_key_path| PrivateNetworkConfig {
- name: config.network_name.clone(),
- key: KeySource::Path(net_key_path),
- });
-
- let supervised_node = SupervisedMycelium::spawn(
- key,
- private_net_config,
- config.tcp_port,
- config.api_port,
- config.rpc_port,
- config.discovery_port,
- &config.peers,
- )
- .await
- .context("Failed to securely bootstrap embedded Mycelium background worker")?;
-
- let resolved_url = Url::parse(&format!(
- "http://127.0.0.1:{}",
- supervised_node.config.api_port
- ))?;
-
- Ok(Self {
- runtime: MyceliumRuntime::Supervised(Box::new(supervised_node)),
- rest_url: resolved_url,
- })
- }
-
- /// Blocks until the background daemon drops out. If running externally, hangs open indefinitely.
- pub async fn wait_for_exit(&mut self) -> std::io::Result<std::process::ExitStatus> {
- match &mut self.runtime {
- MyceliumRuntime::Supervised(node) => node.wait_for_exit().await,
- MyceliumRuntime::External => {
- std::future::pending::<std::io::Result<std::process::ExitStatus>>().await
- }
- }
- }
-}
-
-/// The part of Mycelium's `/api/v1/admin` answer we use.
-#[derive(Deserialize, Debug, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct MyceliumAdminInfo {
- pub node_pubkey: String,
-}
-
-#[derive(Debug, Clone)]
-pub struct MyceliumNodeConfig {
- pub tcp_port: u16,
- pub api_port: u16,
- pub rpc_port: u16,
- pub discovery_port: Option<u16>,
- pub private_network: Option<String>,
-}
-
-pub struct SupervisedMycelium {
- child: Child,
- pub config: MyceliumNodeConfig,
- _key_file_handle: Option<File>,
- _net_key_file_handle: Option<File>,
-}
-
-impl SupervisedMycelium {
- async fn spawn(
- node_key: KeySource,
- private_network: Option<PrivateNetworkConfig>,
- explicit_tcp: Option<u16>,
- explicit_api: Option<u16>,
- explicit_rpc: Option<u16>,
- explicit_discovery_port: Option<u16>,
- peers: &[String],
- ) -> Result<Self> {
- // Linux only: keys are handed over in memory (memfd, never on disk), and the
- // child dies with us (PR_SET_PDEATHSIG). Elsewhere, run myceliumd yourself.
- if cfg!(not(target_os = "linux")) {
- bail!(
- "Supervising Mycelium is only supported on Linux; run myceliumd yourself \
- and pass --mycelium-rest-url"
- );
- }
-
- let tcp_port = match explicit_tcp {
- Some(p) => p,
- None => Self::find_available_port().context("Failed to allocate TCP wire port")?,
- };
- let api_port = match explicit_api {
- Some(p) => p,
- None => Self::find_available_port().context("Failed to allocate API port")?,
- };
- let rpc_port = match explicit_rpc {
- Some(p) => p,
- None => Self::find_available_port().context("Failed to allocate JSON-RPC port")?,
- };
- let discovery_port = match explicit_discovery_port {
- Some(0) => Some(
- Self::find_available_port().context("Failed to allocate peer discovery port")?,
- ),
- p => p,
- };
-
- let config = MyceliumNodeConfig {
- tcp_port,
- api_port,
- rpc_port,
- discovery_port,
- private_network: private_network.as_ref().map(|pn| pn.name.clone()),
- };
-
- let parent_pid = std::process::id();
-
- let (key_file_handle, key_file_path, node_fd) =
- Self::resolve_key_strategy("mycelium_node_key", node_key, parent_pid)?;
-
- let (net_key_file_handle, net_key_file_path, net_fd) = match private_network {
- Some(net_cfg) => {
- let (h, p, fd) =
- Self::resolve_key_strategy("mycelium_net_key", net_cfg.key, parent_pid)?;
- (h, Some(p), fd)
- }
- None => (None, None, None),
- };
-
- let api_addr = format!("127.0.0.1:{}", config.api_port);
- let rpc_addr = format!("127.0.0.1:{}", config.rpc_port);
-
- let mut cmd = Command::new("mycelium");
-
- let fds_to_inherit: Vec<i32> = [node_fd, net_fd].iter().filter_map(|&x| x).collect();
- // The child must never outlive us. An orphaned Mycelium keeps running with our
- // key; the next one we start then shares its address, and replies (including
- // Mycelium's own message ACKs) go to whichever instance the routers picked,
- // leaving our messages pending forever. Drop only runs on a clean shutdown, so
- // let the kernel signal the child when we die, however we die.
- let parent_pid = std::process::id() as libc::pid_t;
- unsafe {
- cmd.pre_exec(move || {
- for &fd in &fds_to_inherit {
- libc::fcntl(fd, libc::F_SETFD, 0);
- }
- // Own process group: a Ctrl-C in the terminal goes to continuum only,
- // which then stops Mycelium itself, instead of both exiting at once (which
- // shows up as "Mycelium runtime process exited").
- if libc::setpgid(0, 0) != 0 {
- return Err(std::io::Error::last_os_error());
- }
- #[cfg(target_os = "linux")]
- {
- if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) != 0 {
- return Err(std::io::Error::last_os_error());
- }
- // We may have died between fork and prctl.
- if libc::getppid() != parent_pid {
- libc::_exit(1);
- }
- }
- Ok(())
- });
- }
-
- cmd.arg("--key-file")
- .arg(&key_file_path)
- .arg("--tcp-listen-port")
- .arg(config.tcp_port.to_string())
- .arg("--disable-quic")
- .arg("--api-addr")
- .arg(&api_addr)
- .arg("--jsonrpc-addr")
- .arg(&rpc_addr)
- .arg("--no-tun");
-
- if let Some(ref discovery_port) = config.discovery_port {
- cmd.arg("--peer-discovery-port")
- .arg(discovery_port.to_string());
- } else {
- cmd.arg("--disable-peer-discovery");
- }
-
- if let Some(ref net_name) = config.private_network {
- cmd.arg("--network-name").arg(net_name);
- }
- if let Some(ref net_key_path) = net_key_file_path {
- cmd.arg("--network-key-file").arg(net_key_path);
- }
-
- if !peers.is_empty() {
- cmd.arg("--peers").arg(peers.join(" "));
- }
-
- cmd.env("RUST_LOG", "info");
-
- cmd.stdout(Stdio::piped());
- cmd.stderr(Stdio::piped());
-
- let mut child = cmd
- .spawn()
- .context("Failed to execute mycelium process daemon")?;
-
- // 1. FIXED: Pass-through streaming for child process STDOUT
- if let Some(mut stdout) = child.stdout.take() {
- tokio::spawn(async move {
- let mut host_stdout = tokio::io::stdout();
- let _ = tokio::io::copy(&mut stdout, &mut host_stdout).await;
- });
- }
-
- // 2. FIXED: Pass-through streaming for child process STDERR
- if let Some(mut stderr) = child.stderr.take() {
- tokio::spawn(async move {
- let mut host_stderr = tokio::io::stderr();
- let _ = tokio::io::copy(&mut stderr, &mut host_stderr).await;
- });
- }
-
- let node = Self {
- child,
- config,
- _key_file_handle: key_file_handle,
- _net_key_file_handle: net_key_file_handle,
- };
-
- Ok(node)
- }
-
- fn resolve_key_strategy(
- desc_tag: &str,
- source: KeySource,
- parent_pid: u32,
- ) -> Result<(Option<File>, PathBuf, Option<i32>)> {
- match source {
- KeySource::Raw(raw_bytes) => {
- let (handle, fd) = Self::create_ephemeral_file(desc_tag, raw_bytes)?;
- let pseudo_path = PathBuf::from(format!("/proc/{}/fd/{}", parent_pid, fd));
- Ok((Some(handle), pseudo_path, Some(fd)))
- }
- KeySource::Path(existing_path) => Ok((None, existing_path, None)),
- }
- }
-
- fn find_available_port() -> Result<u16> {
- let listener = TcpListener::bind("127.0.0.1:0")?;
- let addr = listener.local_addr()?;
- Ok(addr.port())
- }
-
- fn create_ephemeral_file(desc_tag: &str, raw_key: [u8; 32]) -> Result<(File, i32)> {
- #[cfg(target_os = "linux")]
- {
- let memfd_name = std::ffi::CString::new(desc_tag)?;
- let fd = unsafe { libc::memfd_create(memfd_name.as_ptr(), 0) };
- if fd < 0 {
- return Err(anyhow!(
- "memfd_create failed natively inside Linux kernel for tag: {}",
- desc_tag
- ));
- }
- let mut file = unsafe { File::from_raw_fd(fd) };
- file.write_all(&raw_key)?;
- file.flush()?;
- Ok((file, fd))
- }
-
- // Unreachable: `spawn` refuses to run elsewhere.
- #[cfg(not(target_os = "linux"))]
- {
- let _ = (desc_tag, raw_key);
- bail!("In-memory key files need Linux")
- }
- }
-
- pub async fn wait_for_exit(&mut self) -> std::io::Result<std::process::ExitStatus> {
- self.child.wait().await
- }
-}
-
-impl Drop for SupervisedMycelium {
- fn drop(&mut self) {
- // The memfd key files go away with their handles.
- let _ = self.child.start_kill();
- }
-}
-
-fn is_pubkey(s: &str) -> bool {
- s.len() == 64 && s.chars().all(|c| c.is_ascii_hexdigit())
-}
-
-pub struct MyceliumTransport {
- // Wrap your zero-allocation lifecycle supervisor directly inside the driver plane
- pub setup: tokio::sync::Mutex<MyceliumSetup>,
- pub client: reqwest::Client,
- pub poll_client: reqwest::Client,
- pub messages_endpoint: Url,
- pubkey_basepath: Url,
- local_id: String,
-}
-
-impl MyceliumTransport {
- pub async fn connect(setup: MyceliumSetup) -> Result<Self> {
- let rest_url = setup.rest_url.clone();
-
- let client = reqwest::Client::builder()
- .timeout(Duration::from_secs(10))
- .build()?;
- let poll_client = reqwest::Client::builder()
- .timeout(Duration::from_secs(MYCELIUM_POLL_TIMEOUT_SECS + 30))
- .build()?;
-
- let admin: MyceliumAdminInfo = client
- .get(rest_url.join("/api/v1/admin")?)
- .send()
- .await?
- .error_for_status()?
- .json()
- .await
- .context("Querying local Mycelium identity")?;
-
- Ok(Self {
- setup: tokio::sync::Mutex::new(setup),
- client,
- poll_client,
- messages_endpoint: rest_url.join("/api/v1/messages")?,
- pubkey_basepath: rest_url.join("/api/v1/pubkey/")?,
- local_id: admin.node_pubkey,
- })
- }
-
- async fn lookup_pubkey(&self, ip: &std::net::Ipv6Addr) -> Result<String> {
- #[derive(Deserialize)]
- struct PublicKeyResponse {
- #[serde(rename = "publicKey")]
- public_key: String,
- }
-
- let response: PublicKeyResponse = self
- .client
- .get(self.pubkey_basepath.join(&ip.to_string())?)
- .send()
- .await?
- .error_for_status()?
- .json()
- .await?;
- Ok(response.public_key)
- }
-}
-
-#[async_trait]
-impl Transport for MyceliumTransport {
- fn local_id(&self) -> &str {
- &self.local_id
- }
-
- async fn resolve_seed(&self, hint: &str) -> Result<String> {
- if is_pubkey(hint) {
- return Ok(hint.to_owned());
- }
-
- if let Ok(ip) = hint.parse::<std::net::Ipv6Addr>() {
- return self.lookup_pubkey(&ip).await;
- }
-
- let ip = tokio::time::timeout(Duration::from_secs(10), tokio::net::lookup_host((hint, 0)))
- .await
- .context("DNS lookup timed out")??
- .find_map(|addr| match addr.ip() {
- std::net::IpAddr::V6(ip) => Some(ip),
- _ => None,
- })
- .ok_or_else(|| anyhow!("{hint}: no AAAA record found"))?;
-
- self.lookup_pubkey(&ip).await
- }
-
- fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- let outbound = format_message(peer, message)?;
- Ok(EncodedMessage {
- peer: peer.to_owned(),
- body: serde_json::to_vec(&outbound)?.into(),
- })
- }
-
- async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> {
- tracing::trace!(payload_bytes = message.body.len(), "Sending message");
- let response = self
- .client
- .post(self.messages_endpoint.clone())
- .header(reqwest::header::CONTENT_TYPE, "application/json")
- .body(message.body.clone())
- .send()
- .await?;
-
- let status = response.status();
- if !status.is_success() {
- tracing::error!(status = %status, "Mycelium rejected outbound message");
- return Err(TransportError::MyceliumRejected {
- peer: message.peer.clone(),
- status,
- }
- .into());
- }
-
- // Mycelium's message ID, for GET /api/v1/messages/status/{id} when chasing
- // a message that never arrived.
- if tracing::enabled!(tracing::Level::DEBUG) {
- let Ok(body) = response
- .bytes()
- .await
- .map_err(anyhow::Error::from)
- .and_then(|bytes| Ok(serde_json::from_slice::<serde_json::Value>(&bytes)?))
- else {
- tracing::debug!(to = %message.peer, "Sent message (couldn't decode response from Mycelium REST API)");
- return Ok(());
- };
- let Ok(outbound) = serde_json::from_slice::<OutboundMessage>(&message.body) else {
- tracing::debug!(to = %message.peer, body_len = %message.body.len(), "Sent message (couldn't parse outbound message)");
- return Ok(());
- };
- let Ok(parsed) = decode_payload(&outbound.topic, &outbound.payload) else {
- tracing::debug!(to = %message.peer, message = ?outbound, "Sent message (couldn't decode outbound message)");
- return Ok(());
- };
- let Some(id) = body.get("id").and_then(|id| id.as_str()) else {
- tracing::debug!(to = %message.peer, message = ?parsed, "Sent message (couldn't read id from Mycelium REST API response)");
- return Ok(());
- };
- tracing::debug!(to = %message.peer, message = ?parsed, id = %id, "Sent message");
- }
- Ok(())
- }
-
- async fn start_consumer(&self, handler: MessageHandler) -> Result<()> {
- tracing::info!(
- endpoint = self.messages_endpoint.as_str(),
- "Starting Mycelium message consumer"
- );
-
- loop {
- tracing::trace!("Polling Mycelium inbox");
-
- let response = match self
- .poll_client
- .get(self.messages_endpoint.clone())
- .query(&[("timeout", MYCELIUM_POLL_TIMEOUT_SECS)])
- .query(&[("topic", encode_topic(TOPIC_PREFIX))])
- .send()
- .await
- {
- Ok(resp) => resp,
- Err(error) => {
- tracing::warn!(error = %error, "Mycelium inbox poll failed");
- tokio::time::sleep(Duration::from_millis(250)).await;
- continue;
- }
- };
-
- let status = response.status();
-
- if status == StatusCode::NO_CONTENT {
- tracing::trace!("Mycelium inbox is empty");
- tokio::time::sleep(EMPTY_QUEUE_DELAY).await;
- continue;
- }
-
- if !status.is_success() {
- tracing::warn!(status = %status, "Mycelium inbox returned unsuccessful HTTP status");
- tokio::time::sleep(Duration::from_millis(500)).await;
- continue;
- }
-
- let raw_bytes = match response.bytes().await {
- Ok(bytes) => bytes,
- Err(error) => {
- tracing::warn!(error = %error, "Failed to read raw Mycelium response stream");
- tokio::time::sleep(Duration::from_millis(100)).await;
- continue;
- }
- };
-
- let inbound_message = match serde_json::from_slice::<InboundMessage>(&raw_bytes) {
- Ok(msg) => msg,
- Err(error) => {
- tracing::error!(
- error = %error,
- raw_payload = %String::from_utf8_lossy(&raw_bytes),
- "Failed to decode inbound Mycelium message structure"
- );
- tokio::time::sleep(Duration::from_millis(500)).await;
- continue;
- }
- };
-
- match parse_message(&inbound_message) {
- Ok((sender, parsed)) => {
- tracing::debug!(
- from = %inbound_message.src_pk,
- message = ?parsed,
- id = %inbound_message.id,
- //payload_bytes = raw_bytes.len(),
- "Received message"
- );
- if let Err(error) = handler(sender, parsed).await {
- tracing::warn!(error = %format!("{error:#}"), "Failed to process inbound message");
- }
- }
- Err(transport_error) => {
- tracing::error!(
- error = %transport_error,
- topic = %inbound_message.topic,
- "Dropping poisoned packet: parsing failed gracefully"
- );
- // No sleep required: we successfully popped a malformed packet and want to process the next one immediately
- }
- }
- }
- }
-
- async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> {
- let mut guard = self.setup.lock().await;
- guard.wait_for_exit().await
- }
-}
-
-fn encode_topic(topic: &str) -> String {
- BASE64.encode(topic.as_bytes())
-}
-
-fn decode_topic(topic: &str) -> Result<String, TransportError> {
- let bytes = BASE64.decode(topic)?;
- Ok(String::from_utf8(bytes)?)
-}
-
-fn encode_payload(payload: &Vec<u8>) -> String {
- BASE64.encode(payload)
-}
-
-fn decode_payload(topic_encoded: &str, payload: &str) -> Result<ParsedMessage, TransportError> {
- let topic: &str = &decode_topic(topic_encoded)?;
- let bytes = BASE64.decode(payload)?;
- match topic {
- TOPIC_HELLO => {
- let msg = bytes_to_hello_request(&bytes)?;
- Ok(ParsedMessage::Hello(msg))
- }
- TOPIC_HELLO_REPLY => {
- let msg = bytes_to_hello_reply(&bytes)?;
- Ok(ParsedMessage::HelloReply(msg))
- }
- TOPIC_SYNC => {
- let msg = bytes_to_sync_envelope(&bytes)?;
- Ok(ParsedMessage::Sync(msg))
- }
- TOPIC_PING => Ok(ParsedMessage::Ping(bytes_to_keepalive(&bytes)?)),
- TOPIC_PONG => Ok(ParsedMessage::Pong(bytes_to_keepalive(&bytes)?)),
- t => Err(TransportError::UnknownTopic(t.to_string())),
- }
-}
-
-fn parse_message(message: &InboundMessage) -> Result<(String, ParsedMessage), TransportError> {
- let sender = message.src_pk.clone();
-
- if sender.is_empty() {
- return Err(TransportError::NoSender);
- }
-
- let parsed = decode_payload(&message.topic, &message.payload)?;
-
- Ok((sender, parsed))
-}
-
-fn format_message(
- recipient: &str,
- message: &ParsedMessage,
-) -> Result<OutboundMessage, TransportError> {
- let (topic, payload_bytes) = match message {
- ParsedMessage::Hello(req) => (TOPIC_HELLO, hello_request_to_bytes(req)?),
- ParsedMessage::HelloReply(res) => (TOPIC_HELLO_REPLY, hello_reply_to_bytes(res)?),
- ParsedMessage::Sync(env) => (TOPIC_SYNC, sync_envelope_to_bytes(env)?),
- ParsedMessage::Ping(k) => (TOPIC_PING, keepalive_to_bytes(k)?),
- ParsedMessage::Pong(k) => (TOPIC_PONG, keepalive_to_bytes(k)?),
- };
- Ok(OutboundMessage {
- dst: Destination {
- pk: recipient.to_owned(),
- },
- topic: encode_topic(topic),
- payload: encode_payload(&payload_bytes),
- })
-}
-
-// =====================================================================
-// 1. WIRE SHADOWS
-// =====================================================================
-//
-// rmp_serde encodes structs as arrays. Fields added later are appended at the end and
-// marked #[serde(default)], so a message from a node that predates them still decodes
-// (an absent epoch is the genesis epoch).
-
-/// Epoch on the wire. The default (generation 0, empty id) is the genesis epoch.
-#[derive(Serialize, Deserialize, Default)]
-struct EpochShadow {
- generation: u64,
- id: ByteBuf,
-}
-
-fn epoch_to_shadow(epoch: &Epoch) -> EpochShadow {
- EpochShadow {
- generation: epoch.generation,
- id: ByteBuf::from(epoch.id.to_vec()),
- }
-}
-
-fn epoch_from_shadow(shadow: EpochShadow) -> Result<Epoch, TransportError> {
- if shadow.generation == 0 && shadow.id.is_empty() {
- return Ok(Epoch::GENESIS);
- }
- let id = shadow
- .id
- .as_slice()
- .try_into()
- .map_err(|_| TransportError::InvalidPayload("epoch id must be 32 bytes".into()))?;
- Ok(Epoch {
- generation: shadow.generation,
- id,
- })
-}
-
-/// Either `heads` (listed) or `digest` (32 bytes, `heads` empty) is set.
-#[derive(Serialize, Deserialize)]
-struct KeepaliveShadow {
- heads: Vec<ByteBuf>,
- digest: ByteBuf,
- epoch: EpochShadow,
- ready: bool,
- syncs_with_you: bool,
-}
-
-fn keepalive_to_bytes(msg: &Keepalive) -> Result<Vec<u8>, rmp_serde::encode::Error> {
- let (heads, digest) = match &msg.heads {
- HeadsSummary::Heads(heads) => (
- heads
- .iter()
- .map(|head| ByteBuf::from(head.0.to_vec()))
- .collect(),
- ByteBuf::new(),
- ),
- HeadsSummary::Digest(digest) => (Vec::new(), ByteBuf::from(digest.to_vec())),
- };
- rmp_serde::to_vec(&KeepaliveShadow {
- heads,
- digest,
- epoch: epoch_to_shadow(&msg.epoch),
- ready: msg.ready,
- syncs_with_you: msg.syncs_with_you,
- })
-}
-
-fn bytes_to_keepalive(bytes: &[u8]) -> Result<Keepalive, TransportError> {
- let shadow: KeepaliveShadow = rmp_serde::from_slice(bytes)?;
- let heads = if shadow.digest.is_empty() {
- if shadow.heads.len() > MAX_LISTED_HEADS {
- return Err(TransportError::InvalidPayload(format!(
- "more than {MAX_LISTED_HEADS} heads listed"
- )));
- }
- let heads = shadow
- .heads
- .iter()
- .map(|head| ChangeHash::try_from(head.as_slice()))
- .collect::<Result<Vec<_>, _>>()
- .map_err(|_| TransportError::InvalidPayload("a head must be 32 bytes".into()))?;
- HeadsSummary::Heads(heads)
- } else {
- let digest =
- shadow.digest.as_slice().try_into().map_err(|_| {
- TransportError::InvalidPayload("heads digest must be 32 bytes".into())
- })?;
- HeadsSummary::Digest(digest)
- };
- Ok(Keepalive {
- heads,
- epoch: epoch_from_shadow(shadow.epoch)?,
- ready: shadow.ready,
- syncs_with_you: shadow.syncs_with_you,
- })
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct HelloRequestShadow {
- #[serde(default)]
- epoch: EpochShadow,
- #[serde(default)]
- auth: ByteBuf,
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct HelloReplyShadow<'a> {
- // Cow allows zero-copy references when saving, but creates an owned String when loading
- pub swarm_id: Cow<'a, str>,
- #[serde(default)]
- pub peers: Vec<String>,
- #[serde(default)]
- epoch: EpochShadow,
- #[serde(default)]
- auth: ByteBuf,
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct SyncEnvelopeShadow<'a> {
- pub swarm_id: Cow<'a, str>,
- #[serde(with = "automerge_msg_codec")]
- pub message: automerge::sync::Message,
- #[serde(default)]
- epoch: EpochShadow,
- /// Absent from nodes that predate it: never counts as ready.
- #[serde(default)]
- ready: bool,
-}
-
-// =====================================================================
-// 2. CUSTOM CODEC
-// =====================================================================
-mod automerge_msg_codec {
- use super::*;
- use automerge::sync::Message;
-
- pub fn serialize<S>(msg: &Message, serializer: S) -> Result<S::Ok, S::Error>
- where
- S: Serializer,
- {
- let bytes = msg.clone().encode();
- Bytes::new(&bytes).serialize(serializer)
- }
-
- pub fn deserialize<'de, D>(deserializer: D) -> Result<Message, D::Error>
- where
- D: Deserializer<'de>,
- {
- let byte_buf = ByteBuf::deserialize(deserializer)?;
- Message::decode(&byte_buf)
- .map_err(|e| serde::de::Error::custom(format!("Automerge decode failed: {:?}", e)))
- }
-}
-
-// =====================================================================
-// 3. CLEAN CONVERSION INTERFACE
-// =====================================================================
-
-// --- HelloRequest ---
-pub fn hello_request_to_bytes(msg: &HelloRequest) -> Result<Vec<u8>, rmp_serde::encode::Error> {
- rmp_serde::to_vec(&HelloRequestShadow {
- epoch: epoch_to_shadow(&msg.epoch),
- auth: ByteBuf::from(msg.auth.clone()),
- })
-}
-pub fn bytes_to_hello_request(bytes: &[u8]) -> Result<HelloRequest, TransportError> {
- let shadow: HelloRequestShadow = rmp_serde::from_slice(bytes)?;
- Ok(HelloRequest {
- epoch: epoch_from_shadow(shadow.epoch)?,
- auth: shadow.auth.into_vec(),
- })
-}
-
-// --- HelloReply ---
-pub fn hello_reply_to_bytes(msg: &HelloReply) -> Result<Vec<u8>, rmp_serde::encode::Error> {
- let shadow = HelloReplyShadow {
- swarm_id: Cow::Borrowed(&msg.swarm_id),
- peers: msg.peers.clone(), // Vec must be cloned for serialization wrapper matching
- epoch: epoch_to_shadow(&msg.epoch),
- auth: ByteBuf::from(msg.auth.clone()),
- };
- rmp_serde::to_vec(&shadow)
-}
-pub fn bytes_to_hello_reply(bytes: &[u8]) -> Result<HelloReply, TransportError> {
- let shadow: HelloReplyShadow = rmp_serde::from_slice(bytes)?;
- Ok(HelloReply {
- swarm_id: shadow.swarm_id.into_owned(),
- peers: shadow.peers,
- epoch: epoch_from_shadow(shadow.epoch)?,
- auth: shadow.auth.into_vec(),
- })
-}
-
-// --- SyncEnvelope ---
-pub fn sync_envelope_to_bytes(msg: &SyncEnvelope) -> Result<Vec<u8>, rmp_serde::encode::Error> {
- let shadow = SyncEnvelopeShadow {
- swarm_id: Cow::Borrowed(&msg.swarm_id),
- message: msg.message.clone(), // Clone required by automerge_msg_codec ownership rules
- epoch: epoch_to_shadow(&msg.epoch),
- ready: msg.ready,
- };
- rmp_serde::to_vec(&shadow)
-}
-pub fn bytes_to_sync_envelope(bytes: &[u8]) -> Result<SyncEnvelope, TransportError> {
- let shadow: SyncEnvelopeShadow = rmp_serde::from_slice(bytes)?;
- Ok(SyncEnvelope {
- swarm_id: shadow.swarm_id.into_owned(),
- epoch: epoch_from_shadow(shadow.epoch)?,
- message: shadow.message,
- ready: shadow.ready,
- })
-}
-
-#[derive(Error, Debug)]
-pub enum TransportError {
- #[error("Mycelium delivered a message without a sender public key")]
- NoSender,
-
- #[error("Base64 decoding failed: {0}")]
- Base64Decode(#[from] base64::DecodeError),
-
- #[error("UTF-8 decoding failed: {0}")]
- Utf8Decode(#[from] std::string::FromUtf8Error),
-
- #[error("MessagePack serialization failed: {0}")]
- MessagePackEncode(#[from] rmp_serde::encode::Error),
-
- #[error("MessagePack deserialization failed: {0}")]
- MessagePackDecode(#[from] rmp_serde::decode::Error),
-
- #[error("Unknown topic: {0}")]
- UnknownTopic(String),
-
- #[error("Invalid payload: {0}")]
- InvalidPayload(String),
-
- #[error("Network HTTP request failed: {0}")]
- Http(#[from] reqwest::Error),
-
- #[error("Mycelium server rejected outbound message to {peer} (HTTP {status})")]
- MyceliumRejected {
- peer: String,
- status: reqwest::StatusCode,
- },
-}
diff --git a/src/transport/mycelium/daemon.rs b/src/transport/mycelium/daemon.rs
new file mode 100644
index 0000000..58ebe0c
--- /dev/null
+++ b/src/transport/mycelium/daemon.rs
@@ -0,0 +1,333 @@
+//! Running Mycelium for this node: either a daemon the operator runs (its REST URL), or
+//! one we start and supervise (`SupervisedMycelium`): its keys handed over in memory,
+//! never on disk, and it dies with us.
+
+use anyhow::{Context, Result, anyhow, bail};
+use std::fs::File;
+use std::io::Write;
+use std::net::TcpListener;
+use std::os::unix::io::FromRawFd;
+use std::path::PathBuf;
+use std::process::Stdio;
+use tokio::process::{Child, Command};
+use url::Url;
+
+/// Represents where a 32-byte cryptographic key originates.
+#[derive(Debug, Clone)]
+pub(crate) enum KeySource {
+ Raw([u8; 32]),
+ Path(PathBuf),
+}
+
+/// Atomic structure ensuring private network credentials are always handled together.
+#[derive(Debug, Clone)]
+pub(crate) struct PrivateNetworkConfig {
+ pub(crate) name: String,
+ pub(crate) key: KeySource,
+}
+
+/// Comprehensive parameters required to spawn or target an instance.
+#[derive(Debug, Clone)]
+pub(crate) struct MyceliumSpawnConfig {
+ pub(crate) rest_url_opt: Option<Url>,
+ pub(crate) key_file_opt: Option<PathBuf>,
+ pub(crate) network_key_file: Option<PathBuf>,
+ /// Set together with `network_key_file` (`Settings::check`).
+ pub(crate) network_name: Option<String>,
+ pub(crate) tcp_port: Option<u16>,
+ pub(crate) peers: Vec<String>,
+}
+
+impl Default for MyceliumSpawnConfig {
+ fn default() -> Self {
+ Self {
+ rest_url_opt: None,
+ key_file_opt: None,
+ network_key_file: None,
+ network_name: None,
+ tcp_port: None,
+ peers: vec!["tls://127.0.0.1:9651".to_string()],
+ }
+ }
+}
+
+pub(crate) enum MyceliumRuntime {
+ Supervised(Box<SupervisedMycelium>),
+ External,
+}
+
+pub(crate) struct MyceliumSetup {
+ pub(crate) runtime: MyceliumRuntime,
+ pub(crate) rest_url: Url,
+}
+
+impl MyceliumSetup {
+ /// Evaluates the configuration payload and initializes the chosen execution strategy.
+ pub(crate) async fn bootstrap(config: MyceliumSpawnConfig) -> Result<Self> {
+ // TODO warn for all ignored options if rest_url_opt is set
+ if let Some(url) = config.rest_url_opt {
+ tracing::info!(url = %url, "Using a daemon running elsewhere");
+ return Ok(Self {
+ runtime: MyceliumRuntime::External,
+ rest_url: url,
+ });
+ }
+
+ tracing::info!("Starting the supervised daemon");
+
+ let key = match config.key_file_opt {
+ Some(key_file) => KeySource::Path(key_file),
+ None => {
+ let mut bytes = [0u8; 32];
+ getrandom::fill(&mut bytes)?;
+ KeySource::Raw(bytes)
+ }
+ };
+
+ let private_net_config =
+ config
+ .network_key_file
+ .zip(config.network_name)
+ .map(|(net_key_path, name)| PrivateNetworkConfig {
+ name,
+ key: KeySource::Path(net_key_path),
+ });
+
+ let supervised_node =
+ SupervisedMycelium::spawn(key, private_net_config, config.tcp_port, &config.peers)
+ .await
+ .context("Starting the supervised Mycelium daemon")?;
+
+ let resolved_url = Url::parse(&format!(
+ "http://127.0.0.1:{}",
+ supervised_node.config.api_port
+ ))?;
+
+ Ok(Self {
+ runtime: MyceliumRuntime::Supervised(Box::new(supervised_node)),
+ rest_url: resolved_url,
+ })
+ }
+
+ /// Blocks until the background daemon drops out. If running externally, hangs open indefinitely.
+ pub(crate) async fn wait_for_exit(&mut self) -> std::io::Result<std::process::ExitStatus> {
+ match &mut self.runtime {
+ MyceliumRuntime::Supervised(node) => node.wait_for_exit().await,
+ MyceliumRuntime::External => {
+ std::future::pending::<std::io::Result<std::process::ExitStatus>>().await
+ }
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct MyceliumNodeConfig {
+ pub(crate) tcp_port: u16,
+ pub(crate) api_port: u16,
+ pub(crate) rpc_port: u16,
+ pub(crate) private_network: Option<String>,
+}
+
+pub(crate) struct SupervisedMycelium {
+ child: Child,
+ pub(crate) config: MyceliumNodeConfig,
+ _key_file_handle: Option<File>,
+ _net_key_file_handle: Option<File>,
+}
+
+impl SupervisedMycelium {
+ async fn spawn(
+ node_key: KeySource,
+ private_network: Option<PrivateNetworkConfig>,
+ explicit_tcp: Option<u16>,
+ peers: &[String],
+ ) -> Result<Self> {
+ // Linux only: keys are handed over in memory (memfd, never on disk), and the
+ // child dies with us (PR_SET_PDEATHSIG). Elsewhere, run myceliumd yourself.
+ if cfg!(not(target_os = "linux")) {
+ bail!(
+ "Supervising Mycelium is only supported on Linux; run myceliumd yourself \
+ and pass --mycelium-rest-url"
+ );
+ }
+
+ let tcp_port = match explicit_tcp {
+ Some(p) => p,
+ None => Self::find_available_port().context("Failed to allocate TCP wire port")?,
+ };
+ // Local only, so any free port will do.
+ let api_port = Self::find_available_port().context("Failed to allocate API port")?;
+ let rpc_port = Self::find_available_port().context("Failed to allocate JSON-RPC port")?;
+
+ let config = MyceliumNodeConfig {
+ tcp_port,
+ api_port,
+ rpc_port,
+ private_network: private_network.as_ref().map(|pn| pn.name.clone()),
+ };
+
+ let parent_pid = std::process::id();
+
+ let (key_file_handle, key_file_path, node_fd) =
+ Self::resolve_key_strategy("mycelium_node_key", node_key, parent_pid)?;
+
+ let (net_key_file_handle, net_key_file_path, net_fd) = match private_network {
+ Some(net_cfg) => {
+ let (h, p, fd) =
+ Self::resolve_key_strategy("mycelium_net_key", net_cfg.key, parent_pid)?;
+ (h, Some(p), fd)
+ }
+ None => (None, None, None),
+ };
+
+ let api_addr = format!("127.0.0.1:{}", config.api_port);
+ let rpc_addr = format!("127.0.0.1:{}", config.rpc_port);
+
+ let mut cmd = Command::new("mycelium");
+
+ let fds_to_inherit: Vec<i32> = [node_fd, net_fd].iter().filter_map(|&x| x).collect();
+ // The child must never outlive us. An orphaned Mycelium keeps running with our
+ // key; the next one we start then shares its address, and replies (including
+ // Mycelium's own message ACKs) go to whichever instance the routers picked,
+ // leaving our messages pending forever. Drop only runs on a clean shutdown, so
+ // let the kernel signal the child when we die, however we die.
+ let parent_pid = std::process::id() as libc::pid_t;
+ unsafe {
+ cmd.pre_exec(move || {
+ for &fd in &fds_to_inherit {
+ libc::fcntl(fd, libc::F_SETFD, 0);
+ }
+ // Own process group: a Ctrl-C in the terminal goes to continuum only,
+ // which then stops Mycelium itself, instead of both exiting at once (which
+ // shows up as "A transport stopped").
+ if libc::setpgid(0, 0) != 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ #[cfg(target_os = "linux")]
+ {
+ if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) != 0 {
+ return Err(std::io::Error::last_os_error());
+ }
+ // We may have died between fork and prctl.
+ if libc::getppid() != parent_pid {
+ libc::_exit(1);
+ }
+ }
+ Ok(())
+ });
+ }
+
+ cmd.arg("--key-file")
+ .arg(&key_file_path)
+ .arg("--tcp-listen-port")
+ .arg(config.tcp_port.to_string())
+ .arg("--disable-quic")
+ .arg("--api-addr")
+ .arg(&api_addr)
+ .arg("--jsonrpc-addr")
+ .arg(&rpc_addr)
+ .arg("--no-tun");
+
+ // Peers come from `--mycelium-peers`; no local multicast discovery.
+ cmd.arg("--disable-peer-discovery");
+
+ if let Some(ref net_name) = config.private_network {
+ cmd.arg("--network-name").arg(net_name);
+ }
+ if let Some(ref net_key_path) = net_key_file_path {
+ cmd.arg("--network-key-file").arg(net_key_path);
+ }
+
+ if !peers.is_empty() {
+ cmd.arg("--peers").arg(peers.join(" "));
+ }
+
+ cmd.env("RUST_LOG", "info");
+
+ cmd.stdout(Stdio::piped());
+ cmd.stderr(Stdio::piped());
+
+ let mut child = cmd.spawn().context("Running `mycelium`")?;
+
+ // Its output goes where ours does.
+ if let Some(mut stdout) = child.stdout.take() {
+ tokio::spawn(async move {
+ let mut host_stdout = tokio::io::stdout();
+ let _ = tokio::io::copy(&mut stdout, &mut host_stdout).await;
+ });
+ }
+
+ if let Some(mut stderr) = child.stderr.take() {
+ tokio::spawn(async move {
+ let mut host_stderr = tokio::io::stderr();
+ let _ = tokio::io::copy(&mut stderr, &mut host_stderr).await;
+ });
+ }
+
+ let node = Self {
+ child,
+ config,
+ _key_file_handle: key_file_handle,
+ _net_key_file_handle: net_key_file_handle,
+ };
+
+ Ok(node)
+ }
+
+ fn resolve_key_strategy(
+ desc_tag: &str,
+ source: KeySource,
+ parent_pid: u32,
+ ) -> Result<(Option<File>, PathBuf, Option<i32>)> {
+ match source {
+ KeySource::Raw(raw_bytes) => {
+ let (handle, fd) = Self::create_ephemeral_file(desc_tag, raw_bytes)?;
+ let pseudo_path = PathBuf::from(format!("/proc/{}/fd/{}", parent_pid, fd));
+ Ok((Some(handle), pseudo_path, Some(fd)))
+ }
+ KeySource::Path(existing_path) => Ok((None, existing_path, None)),
+ }
+ }
+
+ fn find_available_port() -> Result<u16> {
+ let listener = TcpListener::bind("127.0.0.1:0")?;
+ let addr = listener.local_addr()?;
+ Ok(addr.port())
+ }
+
+ fn create_ephemeral_file(desc_tag: &str, raw_key: [u8; 32]) -> Result<(File, i32)> {
+ #[cfg(target_os = "linux")]
+ {
+ let memfd_name = std::ffi::CString::new(desc_tag)?;
+ let fd = unsafe { libc::memfd_create(memfd_name.as_ptr(), 0) };
+ if fd < 0 {
+ return Err(anyhow!(
+ "memfd_create failed natively inside Linux kernel for tag: {}",
+ desc_tag
+ ));
+ }
+ let mut file = unsafe { File::from_raw_fd(fd) };
+ file.write_all(&raw_key)?;
+ file.flush()?;
+ Ok((file, fd))
+ }
+
+ // Unreachable: `spawn` refuses to run elsewhere.
+ #[cfg(not(target_os = "linux"))]
+ {
+ let _ = (desc_tag, raw_key);
+ bail!("In-memory key files need Linux")
+ }
+ }
+
+ pub(crate) async fn wait_for_exit(&mut self) -> std::io::Result<std::process::ExitStatus> {
+ self.child.wait().await
+ }
+}
+
+impl Drop for SupervisedMycelium {
+ fn drop(&mut self) {
+ // The memfd key files go away with their handles.
+ let _ = self.child.start_kill();
+ }
+}
diff --git a/src/transport/mycelium/mod.rs b/src/transport/mycelium/mod.rs
new file mode 100644
index 0000000..2bb399c
--- /dev/null
+++ b/src/transport/mycelium/mod.rs
@@ -0,0 +1,683 @@
+use anyhow::{Context, Result, anyhow};
+use async_trait::async_trait;
+use base64::engine::{Engine, general_purpose::STANDARD as BASE64};
+use reqwest::StatusCode;
+use serde::{Deserialize, Serialize};
+use std::time::Duration;
+use thiserror::Error;
+use url::Url;
+
+use crate::transport::wire;
+use crate::transport::{
+ EncodedMessage, HandlerFn, MessageHandler, ParsedMessage, Transport, hand_on_probe, is_probe,
+};
+
+pub(crate) mod daemon;
+
+pub(crate) use daemon::{MyceliumSetup, MyceliumSpawnConfig};
+
+const MYCELIUM_POLL_TIMEOUT_SECS: u64 = 300;
+/// How long Mycelium tries to deliver a message before dropping it.
+const DELIVERY_LIMIT: Duration = Duration::from_secs(300);
+/// The largest inbound message read (its JSON, base64 payload included: about 190 MiB of
+/// payload). Anyone can send us anything; this bounds what one message costs before it
+/// is even parsed. Generous, because a sync message can carry a whole history until
+/// large deltas travel as blobs.
+const MAX_INBOUND_BYTES: usize = 256 << 20;
+/// Polls of the inbox in progress at once.
+const INBOX_POLLS: usize = 2;
+/// How long a request to our Mycelium may take, before what its body adds
+/// (`post_timeout`).
+const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
+/// The slowest our Mycelium is expected to take a message body, in bytes per second:
+/// a large sync message gets more time than a STATUS, instead of failing (and being
+/// generated and sent whole again) at a fixed limit.
+const POST_MIN_RATE: u64 = 1 << 20;
+
+/// The one Mycelium topic all of continuum's messages travel on; what a message is, its
+/// payload says (`wire`).
+const TOPIC: &str = "continuum";
+/// How many senders on another wire version are remembered as already logged.
+const MAX_OTHER_VERSIONS: usize = 1024;
+
+const EMPTY_QUEUE_DELAY: Duration = Duration::from_millis(25);
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct InboundMessage {
+ id: String,
+ src_pk: String,
+ topic: String,
+ payload: String,
+}
+
+// Deserialize only for debugging
+#[derive(Debug, Serialize, Deserialize)]
+struct OutboundMessage {
+ dst: Destination,
+ topic: String,
+ payload: String,
+}
+
+// Deserialize only for debugging
+#[derive(Debug, Serialize, Deserialize)]
+struct Destination {
+ pk: String,
+}
+
+/// The part of Mycelium's `/api/v1/admin` answer we use.
+#[derive(Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct MyceliumAdminInfo {
+ pub(crate) node_pubkey: String,
+}
+
+fn is_pubkey(s: &str) -> bool {
+ s.len() == 64 && s.chars().all(|c| c.is_ascii_hexdigit())
+}
+
+pub(crate) struct MyceliumTransport {
+ /// The daemon, kept to wait for its exit.
+ setup: tokio::sync::Mutex<MyceliumSetup>,
+ client: reqwest::Client,
+ /// For the long-polling inbox (its timeout is longer).
+ poll_client: reqwest::Client,
+ messages_endpoint: Url,
+ pubkey_basepath: Url,
+ local_id: String,
+}
+
+impl MyceliumTransport {
+ pub(crate) async fn connect(setup: MyceliumSetup) -> Result<Self> {
+ let rest_url = setup.rest_url.clone();
+
+ let client = reqwest::Client::builder()
+ .timeout(REQUEST_TIMEOUT)
+ .build()?;
+ let poll_client = reqwest::Client::builder()
+ .timeout(Duration::from_secs(MYCELIUM_POLL_TIMEOUT_SECS + 30))
+ .build()?;
+
+ let admin: MyceliumAdminInfo = client
+ .get(rest_url.join("/api/v1/admin")?)
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await
+ .context("Querying local Mycelium identity")?;
+
+ let transport = Self {
+ setup: tokio::sync::Mutex::new(setup),
+ client,
+ poll_client,
+ messages_endpoint: rest_url.join("/api/v1/messages")?,
+ pubkey_basepath: rest_url.join("/api/v1/pubkey/")?,
+ local_id: admin.node_pubkey,
+ };
+ transport.discard_backlog().await;
+ Ok(transport)
+ }
+
+ /// Drops what Mycelium queued for us before we started. A Mycelium that outlives us
+ /// (an external one, or across our restart) keeps accepting messages meant for us:
+ /// HELLO retries, keepalives, sync messages of sessions that ended with our previous
+ /// run. Processing them would only replay stale handshakes. Everything that still
+ /// matters is resent: peers retry their HELLOs, and sync starts over after the
+ /// handshake. Runs before we send anything, so no answer to us is dropped.
+ ///
+ /// Bounded in time, not count: the backlog is whatever piled up while we were away,
+ /// which a local loop drains in well under a second even for large swarms. Whatever
+ /// is still arriving after that is current traffic (peers that don't know we're
+ /// back), and it is processed normally, exactly as without this step.
+ async fn discard_backlog(&self) {
+ const DRAIN_FOR: Duration = Duration::from_secs(2);
+ let deadline = std::time::Instant::now() + DRAIN_FOR;
+ let mut discarded = 0u64;
+ while std::time::Instant::now() < deadline {
+ let response = self
+ .client
+ .get(self.messages_endpoint.clone())
+ .query(&[("timeout", 0)])
+ .query(&[("topic", encode_topic(TOPIC))])
+ .send()
+ .await;
+ match response {
+ Ok(response) if response.status() == StatusCode::NO_CONTENT => break,
+ Ok(response) if response.status().is_success() => discarded += 1,
+ Ok(response) => {
+ tracing::warn!(status = %response.status(), "Couldn't drain the inbox");
+ break;
+ }
+ Err(error) => {
+ tracing::warn!(error = %error, "Couldn't drain the inbox");
+ break;
+ }
+ }
+ }
+ if discarded > 0 {
+ tracing::info!(
+ discarded,
+ "Discarded messages Mycelium queued for us before we started"
+ );
+ }
+ }
+
+ async fn lookup_pubkey(&self, ip: &std::net::Ipv6Addr) -> Result<String> {
+ #[derive(Deserialize)]
+ struct PublicKeyResponse {
+ #[serde(rename = "publicKey")]
+ public_key: String,
+ }
+
+ let response: PublicKeyResponse = self
+ .client
+ .get(self.pubkey_basepath.join(&ip.to_string())?)
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?;
+ Ok(response.public_key)
+ }
+}
+
+#[async_trait]
+impl Transport for MyceliumTransport {
+ fn local_id(&self) -> &str {
+ &self.local_id
+ }
+
+ fn is_peer_id(&self, id: &str) -> bool {
+ is_pubkey(id)
+ }
+
+ fn delivery_limit(&self) -> Duration {
+ DELIVERY_LIMIT
+ }
+
+ async fn resolve_seed(&self, hint: &str) -> Result<String> {
+ if is_pubkey(hint) {
+ return Ok(hint.to_owned());
+ }
+
+ if let Ok(ip) = hint.parse::<std::net::Ipv6Addr>() {
+ return self.lookup_pubkey(&ip).await;
+ }
+
+ let ip = tokio::time::timeout(Duration::from_secs(10), tokio::net::lookup_host((hint, 0)))
+ .await
+ .context("DNS lookup timed out")??
+ .find_map(|addr| match addr.ip() {
+ std::net::IpAddr::V6(ip) => Some(ip),
+ _ => None,
+ })
+ .ok_or_else(|| anyhow!("{hint}: no AAAA record found"))?;
+
+ self.lookup_pubkey(&ip).await
+ }
+
+ fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
+ encode_message(peer, message)
+ }
+
+ async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> {
+ tracing::trace!(payload_bytes = message.body.len(), "Sending message");
+ let response = self
+ .client
+ .post(self.messages_endpoint.clone())
+ .header(reqwest::header::CONTENT_TYPE, "application/json")
+ .timeout(post_timeout(message.body.len()))
+ .body(message.body.clone())
+ .send()
+ .await?;
+
+ let status = response.status();
+ if !status.is_success() {
+ tracing::error!(status = %status, "Mycelium rejected outbound message");
+ return Err(TransportError::MyceliumRejected {
+ peer: message.peer.clone(),
+ status,
+ }
+ .into());
+ }
+
+ // Mycelium's message ID, for GET /api/v1/messages/status/{id} when chasing
+ // a message that never arrived.
+ if tracing::enabled!(tracing::Level::DEBUG) {
+ let Ok(body) = response
+ .bytes()
+ .await
+ .map_err(anyhow::Error::from)
+ .and_then(|bytes| Ok(serde_json::from_slice::<serde_json::Value>(&bytes)?))
+ else {
+ tracing::debug!(to = %message.peer, "Sent message (couldn't decode response from Mycelium REST API)");
+ return Ok(());
+ };
+ let Ok(outbound) = serde_json::from_slice::<OutboundMessage>(&message.body) else {
+ tracing::debug!(to = %message.peer, body_len = %message.body.len(), "Sent message (couldn't parse outbound message)");
+ return Ok(());
+ };
+ let Ok(parsed) = decode_payload(&outbound.topic, &outbound.payload) else {
+ tracing::debug!(to = %message.peer, message = ?outbound, "Sent message (couldn't decode outbound message)");
+ return Ok(());
+ };
+ let Some(id) = body.get("id").and_then(|id| id.as_str()) else {
+ tracing::debug!(to = %message.peer, message = ?parsed, "Sent message (couldn't read id from Mycelium REST API response)");
+ return Ok(());
+ };
+ tracing::debug!(to = %message.peer, message = ?parsed, id = %id, "Sent message");
+ }
+ Ok(())
+ }
+
+ async fn start_consumer(&self, handler: MessageHandler) -> Result<()> {
+ // Senders on another wire version, logged once each.
+ let mut other_versions = std::collections::HashSet::new();
+ tracing::info!(
+ endpoint = self.messages_endpoint.as_str(),
+ "Starting the consumer"
+ );
+
+ // Several polls at once, so one message's HTTP round trip overlaps the next one's
+ // and handling them: the messages still go to the handler one at a time.
+ let (tx, mut rx) = tokio::sync::mpsc::channel(INBOX_POLLS * 4);
+ for _ in 0..INBOX_POLLS {
+ tokio::spawn(poll_inbox(
+ self.poll_client.clone(),
+ self.messages_endpoint.clone(),
+ tx.clone(),
+ ));
+ }
+ drop(tx);
+
+ // PINGs and PONGs aren't handled after the message before them (`is_probe`).
+ let handler: std::sync::Arc<HandlerFn> = std::sync::Arc::from(handler);
+ while let Some(inbound_message) = rx.recv().await {
+ match parse_message(&inbound_message) {
+ Ok((sender, parsed)) => {
+ tracing::debug!(
+ from = %inbound_message.src_pk,
+ message = ?parsed,
+ id = %inbound_message.id,
+ //payload_bytes = raw_bytes.len(),
+ "Received message"
+ );
+ let payload_len = base64_decoded_len(&inbound_message.payload);
+ if is_probe(&parsed) {
+ hand_on_probe(&handler, sender, parsed, payload_len);
+ } else if let Err(error) = handler(sender, parsed, payload_len).await {
+ tracing::warn!(error = %format!("{error:#}"), "Failed to process inbound message");
+ }
+ }
+ Err(TransportError::Wire(wire::WireError::Version(version))) => {
+ // Bounded: anyone can send us anything.
+ if other_versions.len() >= MAX_OTHER_VERSIONS {
+ other_versions.clear();
+ }
+ if other_versions.insert(inbound_message.src_pk.clone()) {
+ tracing::warn!(
+ from = %inbound_message.src_pk,
+ version,
+ ours = wire::VERSION,
+ "Peer speaks another wire version (another build); ignoring its messages"
+ );
+ }
+ }
+ Err(transport_error) => {
+ // Who sent it and what it claims to be.
+ let kind = BASE64
+ .decode(&inbound_message.payload)
+ .ok()
+ .and_then(|payload| payload.get(1).copied());
+ tracing::error!(
+ error = %transport_error,
+ from = %inbound_message.src_pk,
+ kind = ?kind,
+ payload_bytes = base64_decoded_len(&inbound_message.payload),
+ topic = %inbound_message.topic,
+ "Dropping poisoned packet: parsing failed gracefully"
+ );
+ }
+ }
+ }
+ Ok(())
+ }
+
+ async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> {
+ let mut guard = self.setup.lock().await;
+ guard.wait_for_exit().await
+ }
+}
+
+/// Long-polls the inbox and passes what it pops to `tx` (`start_consumer`).
+async fn poll_inbox(
+ client: reqwest::Client,
+ endpoint: Url,
+ tx: tokio::sync::mpsc::Sender<InboundMessage>,
+) {
+ loop {
+ tracing::trace!("Polling the inbox");
+
+ let response = match client
+ .get(endpoint.clone())
+ .query(&[("timeout", MYCELIUM_POLL_TIMEOUT_SECS)])
+ .query(&[("topic", encode_topic(TOPIC))])
+ .send()
+ .await
+ {
+ Ok(resp) => resp,
+ Err(error) => {
+ tracing::warn!(error = %error, "Polling the inbox failed");
+ tokio::time::sleep(Duration::from_millis(250)).await;
+ continue;
+ }
+ };
+
+ let status = response.status();
+
+ if status == StatusCode::NO_CONTENT {
+ tracing::trace!("The inbox is empty");
+ tokio::time::sleep(EMPTY_QUEUE_DELAY).await;
+ continue;
+ }
+
+ if !status.is_success() {
+ tracing::warn!(status = %status, "Polling the inbox failed");
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ continue;
+ }
+
+ let raw_bytes = match read_capped(response, MAX_INBOUND_BYTES).await {
+ Ok(Some(bytes)) => bytes,
+ Ok(None) => {
+ tracing::warn!(
+ limit_bytes = MAX_INBOUND_BYTES,
+ "Dropping an inbound message larger than the limit"
+ );
+ continue;
+ }
+ Err(error) => {
+ tracing::warn!(error = %error, "Couldn't read a message from the inbox");
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ continue;
+ }
+ };
+
+ let inbound_message = match serde_json::from_slice::<InboundMessage>(&raw_bytes) {
+ Ok(msg) => msg,
+ Err(error) => {
+ tracing::error!(
+ error = %error,
+ raw_payload = %String::from_utf8_lossy(&raw_bytes),
+ "Dropping a message that doesn't decode"
+ );
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ continue;
+ }
+ };
+
+ if tx.send(inbound_message).await.is_err() {
+ return; // the consumer stopped
+ }
+ }
+}
+
+/// The body of `response`, unless it is larger than `limit` bytes (None): read no
+/// further than that, whatever the sender claims.
+async fn read_capped(
+ mut response: reqwest::Response,
+ limit: usize,
+) -> reqwest::Result<Option<bytes::Bytes>> {
+ if response
+ .content_length()
+ .is_some_and(|length| length > limit as u64)
+ {
+ return Ok(None);
+ }
+ let mut body = bytes::BytesMut::new();
+ while let Some(chunk) = response.chunk().await? {
+ if body.len() + chunk.len() > limit {
+ return Ok(None);
+ }
+ body.extend_from_slice(&chunk);
+ }
+ Ok(Some(body.freeze()))
+}
+
+/// A message as the Mycelium REST API takes it (the JSON body of an outbound message).
+/// Also the simulator's wire format (`transport::sim`), so tests go through the same
+/// encoding.
+pub(crate) fn encode_message(peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
+ let (outbound, payload_len) = format_message(peer, message)?;
+ Ok(EncodedMessage {
+ peer: peer.to_owned(),
+ address: peer.to_owned(),
+ body: serde_json::to_vec(&outbound)?.into(),
+ kind: message.kind(),
+ payload_len,
+ message: None,
+ })
+}
+
+/// The message in a body made by [`encode_message`].
+#[cfg(test)]
+pub(crate) fn decode_message(body: &[u8]) -> Result<ParsedMessage> {
+ let outbound: OutboundMessage = serde_json::from_slice(body)?;
+ Ok(decode_payload(&outbound.topic, &outbound.payload)?)
+}
+
+fn encode_topic(topic: &str) -> String {
+ BASE64.encode(topic.as_bytes())
+}
+
+fn decode_topic(topic: &str) -> Result<String, TransportError> {
+ let bytes = BASE64.decode(topic)?;
+ Ok(String::from_utf8(bytes)?)
+}
+
+fn encode_payload(payload: &Vec<u8>) -> String {
+ BASE64.encode(payload)
+}
+
+/// How long posting a body of `bytes` may take: `REQUEST_TIMEOUT`, plus the body at
+/// `POST_MIN_RATE`.
+fn post_timeout(bytes: usize) -> Duration {
+ REQUEST_TIMEOUT + Duration::from_millis(bytes as u64 * 1000 / POST_MIN_RATE)
+}
+
+/// Length of what padded base64 `encoded` decodes to.
+fn base64_decoded_len(encoded: &str) -> usize {
+ let padding = encoded.bytes().rev().take_while(|&b| b == b'=').count();
+ (encoded.len() / 4 * 3).saturating_sub(padding)
+}
+
+fn decode_payload(topic_encoded: &str, payload: &str) -> Result<ParsedMessage, TransportError> {
+ let topic = decode_topic(topic_encoded)?;
+ if topic != TOPIC {
+ return Err(TransportError::UnknownTopic(topic));
+ }
+ Ok(wire::decode(&BASE64.decode(payload)?)?)
+}
+
+fn parse_message(message: &InboundMessage) -> Result<(String, ParsedMessage), TransportError> {
+ let sender = message.src_pk.clone();
+
+ if sender.is_empty() {
+ return Err(TransportError::NoSender);
+ }
+
+ let parsed = decode_payload(&message.topic, &message.payload)?;
+
+ Ok((sender, parsed))
+}
+
+/// The message for Mycelium's API, and the length of its payload.
+fn format_message(
+ recipient: &str,
+ message: &ParsedMessage,
+) -> Result<(OutboundMessage, usize), TransportError> {
+ let payload_bytes = wire::encode(message)?;
+ let outbound = OutboundMessage {
+ dst: Destination {
+ pk: recipient.to_owned(),
+ },
+ topic: encode_topic(TOPIC),
+ payload: encode_payload(&payload_bytes),
+ };
+ Ok((outbound, payload_bytes.len()))
+}
+
+#[derive(Error, Debug)]
+pub(crate) enum TransportError {
+ #[error("Mycelium delivered a message without a sender public key")]
+ NoSender,
+
+ #[error("Base64 decoding failed: {0}")]
+ Base64Decode(#[from] base64::DecodeError),
+
+ #[error("UTF-8 decoding failed: {0}")]
+ Utf8Decode(#[from] std::string::FromUtf8Error),
+
+ #[error(transparent)]
+ Wire(#[from] wire::WireError),
+
+ #[error("Unknown topic: {0}")]
+ UnknownTopic(String),
+
+ #[error("Network HTTP request failed: {0}")]
+ Http(#[from] reqwest::Error),
+
+ #[error("Mycelium server rejected outbound message to {peer} (HTTP {status})")]
+ MyceliumRejected {
+ peer: String,
+ status: reqwest::StatusCode,
+ },
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::transport::{
+ BlobBody, BlobId, BlobMessage, Epoch, HeadsSummary, MAX_LISTED_HEADS, Status,
+ };
+ use automerge::ChangeHash;
+
+ /// Mycelium splits messages into chunks of this size.
+ const MYCELIUM_CHUNK: usize = 1300;
+
+ #[test]
+ fn post_timeout_grows_with_the_body() {
+ assert_eq!(post_timeout(0), REQUEST_TIMEOUT);
+ assert_eq!(
+ post_timeout(64 << 20),
+ REQUEST_TIMEOUT + Duration::from_secs(64)
+ );
+ }
+
+ #[test]
+ fn largest_keepalive_fits_in_one_chunk() {
+ // A digest with a full sample: the most a keepalive carries.
+ let status = Status {
+ heads: HeadsSummary::of(
+ (0..=MAX_LISTED_HEADS as u8)
+ .map(|n| ChangeHash([n; 32]))
+ .collect(),
+ ),
+ epoch: Epoch {
+ generation: u64::MAX,
+ id: [0xff; 32],
+ },
+ epoch_seq: u32::MAX,
+ ready: true,
+ syncs_with_you: true,
+ };
+ let payload = wire::encode(&ParsedMessage::StatusReply(status.clone())).unwrap();
+ // Mycelium's message data: a length byte, the topic, then our payload.
+ let size = 1 + TOPIC.len() + payload.len();
+ assert!(
+ size + 50 <= MYCELIUM_CHUNK,
+ "STATUS with {MAX_LISTED_HEADS} heads takes {size} bytes"
+ );
+ let ParsedMessage::StatusReply(decoded) = wire::decode(&payload).unwrap() else {
+ panic!("not a STATUS_REPLY");
+ };
+ assert_eq!(decoded.heads, status.heads);
+ }
+
+ #[test]
+ fn blob_messages_round_trip() {
+ let epoch = Epoch {
+ generation: 3,
+ id: [7; 32],
+ };
+ let id = BlobId::Change(ChangeHash([9; 32]));
+ let decode_blob = |payload: &[u8]| match wire::decode(payload).unwrap() {
+ ParsedMessage::Blob(blob) => blob,
+ other => panic!("not a blob: {other:?}"),
+ };
+ let chunk = ParsedMessage::Blob(BlobMessage {
+ epoch,
+ epoch_seq: 4,
+ id,
+ body: BlobBody::Chunk {
+ index: 5,
+ data: vec![1u8; 64 * 1024].into(),
+ },
+ });
+ let bytes = wire::encode(&chunk).unwrap();
+ // A 64 KiB chunk costs only a few bytes of framing.
+ assert!(bytes.len() < 64 * 1024 + 128, "{} bytes", bytes.len());
+ let decoded = decode_blob(&bytes);
+ assert_eq!(decoded.id, id);
+ assert_eq!(decoded.epoch, epoch);
+ assert_eq!(decoded.epoch_seq, 4);
+ assert!(
+ matches!(decoded.body, BlobBody::Chunk { index: 5, ref data } if data.len() == 64 * 1024)
+ );
+
+ let manifest = ParsedMessage::Blob(BlobMessage {
+ epoch,
+ epoch_seq: 4,
+ id,
+ body: BlobBody::Manifest {
+ size: 100,
+ chunk_size: 64,
+ hashes: vec![[1; 32], [2; 32]],
+ },
+ });
+ let decoded = decode_blob(&wire::encode(&manifest).unwrap());
+ assert!(matches!(
+ decoded.body,
+ BlobBody::Manifest { size: 100, chunk_size: 64, ref hashes } if hashes.len() == 2
+ ));
+ }
+
+ #[test]
+ fn messages_round_trip_through_the_api_body() {
+ let status = ParsedMessage::Status(Status {
+ heads: HeadsSummary::of(vec![ChangeHash([1; 32])]),
+ epoch: Epoch::GENESIS,
+ epoch_seq: 0,
+ ready: true,
+ syncs_with_you: true,
+ });
+ let encoded = encode_message("peer", &status).unwrap();
+ assert!(matches!(
+ decode_message(&encoded.body).unwrap(),
+ ParsedMessage::Status(_)
+ ));
+ }
+
+ #[test]
+ fn listed_heads_equal_their_digest() {
+ let heads: Vec<_> = (0..3u8).map(|n| ChangeHash([n; 32])).collect();
+ let listed = HeadsSummary::of(heads.clone());
+ let digest = HeadsSummary::Digest {
+ digest: listed.digest(),
+ sample: Vec::new(),
+ };
+ assert_eq!(listed, digest);
+ assert_ne!(listed, HeadsSummary::of(heads[..2].to_vec()));
+ }
+}
diff --git a/src/transport/paths.rs b/src/transport/paths.rs
new file mode 100644
index 0000000..ba16794
--- /dev/null
+++ b/src/transport/paths.rs
@@ -0,0 +1,1247 @@
+//! Paths: one address space over any number of transports, below the identity layer
+//! (`transport::identity`). An address is `<scheme>:<target>`: the transport by its
+//! scheme (`mycelium`, `libp2p`, …), and on it a transport identity (Mycelium's public
+//! key, libp2p's peer ID) or a location (`Transport::locate`), e.g. a multiaddr.
+//! Addresses as paths keeps them are `<scheme>:<transport identity>` (`canonical`).
+//!
+//! Every transport carries the same messages (`wire`); paths routes each send to the
+//! transport of its address, hands every transport's messages to one handler under the
+//! sender's address, and puts together what the transports say about themselves
+//! (addresses, seeds they found, figures). The transports are in order of preference,
+//! as configured (`--transports`).
+//!
+//! A message to a node with several addresses goes to the first of them that takes it
+//! (`send_first`), in the order `rank` gives. A send that fails is tried at once at the
+//! next address, the message encoded for its transport: when one transport fails, the
+//! next carries on within the same send.
+//!
+//! Paths measures the addresses of such nodes itself (and every path the node's
+//! keepalive PINGs go over, below): every `PROBE_EVERY`, each address it had a choice of
+//! within `ACTIVE_FOR` gets a PING with a random nonce, which the
+//! other side's paths answers at once with a PONG, over the same path. Answers give each
+//! path's round-trip time (smoothed); `UNANSWERED_DOWN` PINGs in a row unanswered, or a
+//! send that failed within `DOWN_FOR`, make a path down. `rank` puts paths that are up
+//! first, the one in use first among them, and the rest by round-trip time; it changes
+//! the one in use when another is faster by `SWITCH_GAIN` (a quarter) and at least
+//! `SWITCH_MIN` (both measured), so it doesn't flap. Until both are measured, the
+//! configured order decides.
+//!
+//! PINGs and PONGs then go up as other messages do: they tell the node the peer is
+//! alive. The node's keepalive is a PING too (`swarm::peer::keepalive`), which paths
+//! sends like any message, and notes as the probe of the path it went over.
+//!
+//! The debug firewall (`transport::firewall`) works here, below everything else, through
+//! a `Gate`: messages from an address it blocks are dropped as they arrive, and sends to
+//! one are dropped silently, as by a lossy network.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex, Weak};
+use std::time::Duration;
+
+use anyhow::{Context, Result, anyhow};
+use tokio::time::Instant;
+use tracing::{debug, info};
+
+use crate::locks::MutexExt;
+use crate::transport::stats::{PathCounts, PathInfo, PathsStats, TransportStats};
+use crate::transport::traffic::Traffic;
+use crate::transport::{Connection, EncodedMessage, ParsedMessage, Transport};
+
+/// Whether to drop what comes from or goes to an address (the debug firewall,
+/// `transport::firewall`; the identity layer knows which node is where).
+pub(crate) trait Gate: Send + Sync {
+ fn blocks_inbound(&self, address: &str) -> bool;
+ fn blocks_outbound(&self, address: &str) -> bool;
+}
+
+/// Called with the sender's address, the message and its `payload_len`.
+pub(crate) type PathHandler = dyn Fn(String, ParsedMessage, usize) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + Send>>
+ + Send
+ + Sync
+ + 'static;
+
+/// One of the node's transports.
+struct Link {
+ scheme: &'static str,
+ inner: Arc<dyn Transport>,
+}
+
+impl Link {
+ fn address(&self, target: &str) -> String {
+ format!("{}:{target}", self.scheme)
+ }
+
+ fn own_address(&self) -> String {
+ self.address(self.inner.local_id())
+ }
+}
+
+/// The scheme of `address`.
+pub(crate) fn scheme_of(address: &str) -> &str {
+ address.split_once(':').map_or("", |(scheme, _)| scheme)
+}
+
+/// A path whose send failed is down for this long (unless it answers a PING, or a
+/// send there works again).
+const DOWN_FOR: Duration = Duration::from_secs(60);
+/// How often a path in use is pinged, how long an answer may take, and how many PINGs
+/// in a row unanswered make it down.
+const PROBE_EVERY: Duration = Duration::from_secs(30);
+const PROBE_TIMEOUT: Duration = Duration::from_secs(15);
+const UNANSWERED_DOWN: u32 = 3;
+/// How often the probe task looks for paths due.
+const PROBE_TICK: Duration = Duration::from_secs(5);
+/// Paths this node had a choice of within this long are probed; ones it didn't use for
+/// twice as long are forgotten.
+const ACTIVE_FOR: Duration = Duration::from_secs(300);
+/// The path in use is changed for one this much faster (a fraction of its round-trip
+/// time), and at least `SWITCH_MIN`.
+const SWITCH_GAIN: f64 = 0.25;
+const SWITCH_MIN: Duration = Duration::from_millis(5);
+/// Paths kept at most: they are of peers this node chose among, but bounded anyway.
+const MAX_PATHS: usize = 4096;
+
+/// What paths knows about one path (an address as paths keep it).
+#[derive(Debug, Default)]
+struct PathState {
+ /// When it was last among several to choose from (`rank`).
+ used: Option<Instant>,
+ /// Smoothed round-trip time of its PINGs.
+ rtt: Option<Duration>,
+ /// The PING out, by nonce, and when it went.
+ probe: Option<(u64, Instant)>,
+ last_probe: Option<Instant>,
+ /// PINGs in a row unanswered.
+ unanswered: u32,
+ /// When a send there last failed.
+ failed_at: Option<Instant>,
+}
+
+impl PathState {
+ fn is_down(&self, now: Instant) -> bool {
+ self.unanswered >= UNANSWERED_DOWN
+ || self
+ .failed_at
+ .is_some_and(|at| now.saturating_duration_since(at) < DOWN_FOR)
+ }
+}
+
+/// Every path's state, and which one is in use for each set to choose from.
+#[derive(Debug, Default)]
+struct Table {
+ paths: HashMap<String, PathState>,
+ /// By the set's addresses (sorted, joined): the one in use.
+ chosen: HashMap<String, String>,
+ counts: PathCounts,
+}
+
+impl Table {
+ /// `usable`, with each one's place in the configured order, best first (as
+ /// `Paths::rank` says), noting the choice.
+ fn rank(&mut self, mut usable: Vec<(String, usize)>, now: Instant) -> Vec<String> {
+ usable.sort_by_key(|(_, preference)| *preference);
+ usable.dedup_by(|a, b| a.0 == b.0);
+ if usable.len() < 2 {
+ return usable.into_iter().map(|(address, _)| address).collect();
+ }
+ if self.paths.len() >= MAX_PATHS {
+ self.forget_unused(now);
+ }
+ for (address, _) in &usable {
+ if self.paths.len() < MAX_PATHS || self.paths.contains_key(address) {
+ self.paths.entry(address.clone()).or_default().used = Some(now);
+ }
+ }
+ let rtt = |table: &Self, address: &str| table.paths.get(address).and_then(|path| path.rtt);
+ let (up, down): (Vec<_>, Vec<_>) = usable.into_iter().partition(|(address, _)| {
+ !self
+ .paths
+ .get(address)
+ .is_some_and(|path| path.is_down(now))
+ });
+ let mut set: Vec<&str> = up
+ .iter()
+ .chain(down.iter())
+ .map(|(address, _)| address.as_str())
+ .collect();
+ set.sort_unstable();
+ let key = set.join(" ");
+ let mut ranked: Vec<String> = Vec::new();
+ if !up.is_empty() {
+ let mut current = self
+ .chosen
+ .get(&key)
+ .filter(|chosen| up.iter().any(|(address, _)| address == *chosen))
+ .cloned()
+ .unwrap_or_else(|| up[0].0.clone());
+ let fastest = up
+ .iter()
+ .filter_map(|(address, _)| Some((address, rtt(self, address)?)))
+ .min_by_key(|(_, rtt)| *rtt);
+ if let (Some((fastest, fastest_rtt)), Some(current_rtt)) =
+ (fastest, rtt(self, &current))
+ && *fastest != current
+ && current_rtt.saturating_sub(fastest_rtt)
+ >= SWITCH_MIN.max(current_rtt.mul_f64(SWITCH_GAIN))
+ {
+ info!(
+ from = %current,
+ to = %fastest,
+ from_rtt = ?current_rtt,
+ to_rtt = ?fastest_rtt,
+ "Switching to a faster path"
+ );
+ self.counts.switches += 1;
+ current = fastest.clone();
+ }
+ if self.chosen.len() < MAX_PATHS || self.chosen.contains_key(&key) {
+ self.chosen.insert(key, current.clone());
+ }
+ // The rest: measured ones by round-trip time, then unmeasured ones in the
+ // configured order.
+ let mut rest: Vec<&(String, usize)> = up
+ .iter()
+ .filter(|(address, _)| *address != current)
+ .collect();
+ rest.sort_by_key(|(address, preference)| {
+ (
+ rtt(self, address).is_none(),
+ rtt(self, address),
+ *preference,
+ )
+ });
+ ranked.push(current);
+ ranked.extend(rest.into_iter().map(|(address, _)| address.clone()));
+ }
+ ranked.extend(down.into_iter().map(|(address, _)| address));
+ ranked
+ }
+
+ /// A send to `address` failed.
+ fn failed(&mut self, address: &str, now: Instant) {
+ self.counts.send_failures += 1;
+ if let Some(path) = self.paths.get_mut(address) {
+ path.failed_at = Some(now);
+ // A PING that didn't go out gets no answer; it is no unanswered one.
+ path.probe = None;
+ }
+ }
+
+ /// A send to `address` worked.
+ fn sent(&mut self, address: &str) {
+ if let Some(path) = self.paths.get_mut(address) {
+ path.failed_at = None;
+ }
+ }
+
+ /// The PINGs due now, with their nonces (from `nonce`): to every path used within
+ /// `ACTIVE_FOR` that wasn't pinged within `PROBE_EVERY` and has no PING out. A PING
+ /// out for `PROBE_TIMEOUT` counts as unanswered.
+ fn due(&mut self, now: Instant, mut nonce: impl FnMut() -> u64) -> Vec<(String, u64)> {
+ self.forget_unused(now);
+ let mut due = Vec::new();
+ for (address, path) in &mut self.paths {
+ if let Some((_, sent)) = path.probe
+ && now.saturating_duration_since(sent) >= PROBE_TIMEOUT
+ {
+ path.probe = None;
+ path.unanswered += 1;
+ self.counts.probes_unanswered += 1;
+ if path.unanswered == UNANSWERED_DOWN {
+ info!(address = %address, "A path stopped answering PINGs; using others");
+ }
+ }
+ let active = path
+ .used
+ .is_some_and(|used| now.saturating_duration_since(used) < ACTIVE_FOR);
+ let rested = path
+ .last_probe
+ .is_none_or(|last| now.saturating_duration_since(last) >= PROBE_EVERY);
+ if active && rested && path.probe.is_none() {
+ let nonce = nonce();
+ path.probe = Some((nonce, now));
+ path.last_probe = Some(now);
+ self.counts.probes_sent += 1;
+ due.push((address.clone(), nonce));
+ }
+ }
+ due
+ }
+
+ /// The node sent a PING with `nonce` to `address` (its keepalive): the probe of that
+ /// path, which its PONG measures, even if it is a peer's only path (then it isn't
+ /// pinged otherwise: `due` pings only paths there was a choice of).
+ fn pinged(&mut self, address: &str, nonce: u64, now: Instant) {
+ if self.paths.len() >= MAX_PATHS && !self.paths.contains_key(address) {
+ self.forget_unused(now);
+ if self.paths.len() >= MAX_PATHS {
+ return;
+ }
+ }
+ let path = self.paths.entry(address.to_owned()).or_default();
+ path.probe = Some((nonce, now));
+ path.last_probe = Some(now);
+ self.counts.probes_sent += 1;
+ }
+
+ /// The PING with `nonce` to `address` (from `due`) goes now: its round trip counts
+ /// from here, not from when it was found due.
+ fn sending(&mut self, address: &str, nonce: u64, now: Instant) {
+ if let Some(path) = self.paths.get_mut(address)
+ && let Some((expected, sent)) = &mut path.probe
+ && *expected == nonce
+ {
+ *sent = now;
+ }
+ }
+
+ /// A PONG with `nonce` from `address`: the round trip, if it answers our PING out.
+ fn pong(&mut self, address: &str, nonce: u64, now: Instant) -> Option<Duration> {
+ let path = self.paths.get_mut(address)?;
+ let (expected, sent) = path.probe?;
+ if expected != nonce {
+ return None;
+ }
+ let sample = now.saturating_duration_since(sent);
+ path.probe = None;
+ if path.unanswered >= UNANSWERED_DOWN {
+ info!(address = %address, "A path answers PINGs again");
+ }
+ path.unanswered = 0;
+ path.failed_at = None;
+ path.rtt = Some(match path.rtt {
+ Some(rtt) => (rtt * 3 + sample) / 4,
+ None => sample,
+ });
+ self.counts.probes_answered += 1;
+ path.rtt
+ }
+
+ /// Forgets the paths neither used nor pinged for twice `ACTIVE_FOR`, and choices
+ /// among them.
+ fn forget_unused(&mut self, now: Instant) {
+ let recent = |at: Option<Instant>| {
+ at.is_some_and(|at| now.saturating_duration_since(at) < ACTIVE_FOR * 2)
+ };
+ self.paths
+ .retain(|_, path| recent(path.used) || recent(path.last_probe));
+ let paths = &self.paths;
+ self.chosen
+ .retain(|key, _| key.split(' ').all(|address| paths.contains_key(address)));
+ }
+
+ fn info(&self, address: &str, now: Instant) -> PathInfo {
+ let path = self.paths.get(address);
+ PathInfo {
+ address: address.to_owned(),
+ rtt_ms: path
+ .and_then(|path| path.rtt)
+ .map(|rtt| rtt.as_secs_f64() * 1000.0),
+ down: path.is_some_and(|path| path.is_down(now)),
+ }
+ }
+}
+
+pub(crate) struct Paths {
+ /// In order of preference.
+ links: Vec<Link>,
+ /// Its own PINGs and PONGs are counted here, under the address (`traffic`).
+ traffic: Option<Arc<Traffic>>,
+ /// The debug firewall, if any.
+ gate: Option<Weak<dyn Gate>>,
+ /// A std mutex: never held across an await.
+ table: Mutex<Table>,
+}
+
+impl Paths {
+ /// Over `transports`, by scheme, in order of preference; its own PINGs and PONGs
+ /// counted in `traffic`, the firewall through `gate`.
+ pub(crate) fn new(
+ transports: Vec<(&'static str, Arc<dyn Transport>)>,
+ traffic: Option<Arc<Traffic>>,
+ gate: Option<Weak<dyn Gate>>,
+ ) -> Self {
+ Self {
+ links: transports
+ .into_iter()
+ .map(|(scheme, inner)| Link { scheme, inner })
+ .collect(),
+ traffic,
+ gate,
+ table: Mutex::default(),
+ }
+ }
+
+ fn gate(&self) -> Option<Arc<dyn Gate>> {
+ self.gate.as_ref()?.upgrade()
+ }
+
+ /// The transport `address` is on, and the transport identity at it
+ /// (`Transport::locate`, which notes where that is if the address says); None if no
+ /// transport of ours has its scheme, or it isn't one of that transport's.
+ fn split(&self, address: &str) -> Option<(&Link, String)> {
+ let (scheme, target) = address.split_once(':')?;
+ let link = self.links.iter().find(|link| link.scheme == scheme)?;
+ let target = link.inner.locate(target)?;
+ Some((link, target))
+ }
+
+ /// Whether one of our transports can use `address`.
+ pub(crate) fn usable(&self, address: &str) -> bool {
+ self.split(address).is_some()
+ }
+
+ /// `address` as paths keep it: `<scheme>:<transport identity>`.
+ pub(crate) fn canonical(&self, address: &str) -> Option<String> {
+ self.split(address)
+ .map(|(link, target)| link.address(&target))
+ }
+
+ /// Where the node at `address` (as paths keep it) is, as others can use it
+ /// (`Transport::locations`).
+ pub(crate) fn locations(&self, address: &str) -> Vec<String> {
+ let Some((link, target)) = self.split(address) else {
+ return Vec::new();
+ };
+ link.inner
+ .locations(&target)
+ .iter()
+ .map(|location| link.address(location))
+ .collect()
+ }
+
+ /// Where `address`'s transport is in the order of preference (none of ours: last).
+ pub(crate) fn preference(&self, address: &str) -> usize {
+ let scheme = scheme_of(address);
+ self.links
+ .iter()
+ .position(|link| link.scheme == scheme)
+ .unwrap_or(usize::MAX)
+ }
+
+ /// The usable ones of `addresses` (as paths keep them), best first: paths that are
+ /// up before those down, the one in use first, then by round-trip time, then in
+ /// the configured order (module docs). Several addresses are paths to measure.
+ pub(crate) fn rank(&self, addresses: &[String]) -> Vec<String> {
+ let usable: Vec<(String, usize)> = addresses
+ .iter()
+ .filter(|address| self.usable(address))
+ .map(|address| (address.clone(), self.preference(address)))
+ .collect();
+ self.table.lock_or_recover().rank(usable, Instant::now())
+ }
+
+ /// The smoothed round-trip time of the path to `address`, if measured.
+ pub(crate) fn rtt(&self, address: &str) -> Option<Duration> {
+ self.table.lock_or_recover().paths.get(address)?.rtt
+ }
+
+ /// What paths knows about each of `addresses`.
+ pub(crate) fn info(&self, addresses: &[String]) -> Vec<PathInfo> {
+ let now = Instant::now();
+ let infos = {
+ let table = self.table.lock_or_recover();
+ addresses
+ .iter()
+ .map(|address| table.info(address, now))
+ .collect()
+ };
+ self.located(infos)
+ }
+
+ /// `infos` with where each path goes, where the transport knows
+ /// (`Transport::location`); asked outside the table's lock.
+ fn located(&self, mut infos: Vec<PathInfo>) -> Vec<PathInfo> {
+ for info in &mut infos {
+ if let Some((link, location)) = self
+ .split(&info.address)
+ .and_then(|(link, target)| Some((link, link.inner.location(&target)?)))
+ {
+ info.address = link.address(&location);
+ }
+ }
+ infos
+ }
+
+ /// Whether `address` is one of ours.
+ pub(crate) fn is_own(&self, address: &str) -> bool {
+ self.links.iter().any(|link| link.own_address() == address)
+ }
+
+ /// Our address on the transport of `address`, as `address` names it there.
+ pub(crate) fn own_address(&self, address: &str) -> Option<String> {
+ let scheme = scheme_of(address);
+ self.links
+ .iter()
+ .find(|link| link.scheme == scheme)
+ .map(Link::own_address)
+ }
+
+ /// The longest of the transports': until then, something may still be on its way.
+ pub(crate) fn delivery_limit(&self) -> Duration {
+ self.links
+ .iter()
+ .map(|link| link.inner.delivery_limit())
+ .max()
+ .unwrap_or_default()
+ }
+
+ /// Each transport's (`Transport::local_addresses`), with its scheme.
+ pub(crate) fn local_addresses(&self) -> Vec<String> {
+ self.links
+ .iter()
+ .flat_map(|link| {
+ link.inner
+ .local_addresses()
+ .into_iter()
+ .map(|target| link.address(&target))
+ })
+ .collect()
+ }
+
+ /// The schemes of the transports, in order of preference.
+ pub(crate) fn transports(&self) -> Vec<String> {
+ self.links
+ .iter()
+ .map(|link| link.scheme.to_owned())
+ .collect()
+ }
+
+ /// Each transport's, with its scheme.
+ pub(crate) fn discovered_seeds(&self) -> Vec<String> {
+ self.links
+ .iter()
+ .flat_map(|link| {
+ link.inner
+ .discovered_seeds()
+ .into_iter()
+ .map(|seed| link.address(&seed))
+ })
+ .collect()
+ }
+
+ /// The address of the node a seed leads to, as paths keep it: an address of ours
+ /// (`<scheme>:...`) whose transport reads it at once; with a scheme of ours, what
+ /// that transport resolves the rest to (an address the transport can't read as an
+ /// identity, e.g. a libp2p multiaddr without its peer ID, still resolves); without
+ /// one, what the first transport that can resolves it to.
+ pub(crate) async fn resolve(&self, seed: &str) -> Result<String> {
+ if let Some(address) = self.canonical(seed) {
+ return Ok(address);
+ }
+ let scheme_and_rest = seed.split_once(':').and_then(|(scheme, rest)| {
+ let link = self.links.iter().find(|link| link.scheme == scheme)?;
+ Some((link, rest))
+ });
+ if let Some((link, rest)) = scheme_and_rest {
+ let target = link
+ .inner
+ .resolve_seed(rest)
+ .await
+ .with_context(|| format!("{}: {seed}", link.scheme))?;
+ return Ok(link.address(&target));
+ }
+ let mut failures = Vec::new();
+ for link in &self.links {
+ match link.inner.resolve_seed(seed).await {
+ Ok(target) => return Ok(link.address(&target)),
+ Err(error) => failures.push(format!("{}: {error:#}", link.scheme)),
+ }
+ }
+ Err(anyhow!("{}", failures.join("; ")))
+ }
+
+ /// `message` encoded for `address`: `peer` and `address` of the result are
+ /// `address`, and it keeps the message (to encode it for another address).
+ pub(crate) fn encode(&self, address: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
+ let (link, target) = self
+ .split(address)
+ .with_context(|| format!("no transport for {address}"))?;
+ let mut encoded = link.inner.encode(&target, message)?;
+ encoded.peer = address.to_owned();
+ encoded.address = address.to_owned();
+ encoded.message = Some(Arc::new(message.clone()));
+ Ok(encoded)
+ }
+
+ /// Sends `message` to the first of `addresses` (`rank`ed) that takes it: to its own
+ /// `address` as encoded, to others encoded again for their transport (if it keeps
+ /// the message). Returns the address it went to; an error with every address's if
+ /// none took it.
+ pub(crate) async fn send_first(
+ &self,
+ message: &EncodedMessage,
+ addresses: &[String],
+ ) -> Result<String> {
+ let mut failures: Vec<(String, anyhow::Error)> = Vec::new();
+ for address in addresses {
+ let encoded = if *address == message.address {
+ message.clone()
+ } else {
+ let Some(parsed) = &message.message else {
+ continue;
+ };
+ match self.encode(address, parsed) {
+ Ok(mut encoded) => {
+ encoded.peer = message.peer.clone();
+ encoded
+ }
+ Err(error) => {
+ failures.push((address.clone(), error));
+ continue;
+ }
+ }
+ };
+ // A PING is noted before it goes: on a fast link, its PONG may be back
+ // before the send returns.
+ if let Some(ParsedMessage::Ping(nonce)) = message.message.as_deref() {
+ self.table
+ .lock_or_recover()
+ .pinged(address, *nonce, Instant::now());
+ }
+ match self.send(&encoded).await {
+ Ok(()) => {
+ let mut table = self.table.lock_or_recover();
+ table.sent(address);
+ if !failures.is_empty() {
+ table.counts.failovers += 1;
+ drop(table);
+ debug!(
+ peer = %message.peer,
+ address = %address,
+ failed = %describe(&failures),
+ "Sent at another address after a send failed"
+ );
+ }
+ return Ok(address.clone());
+ }
+ Err(error) => {
+ self.table.lock_or_recover().failed(address, Instant::now());
+ failures.push((address.clone(), error));
+ }
+ }
+ }
+ match failures.len() {
+ 0 => Err(anyhow!("no address to send to")),
+ // The transport's error as it is.
+ 1 => Err(failures.pop().expect("one").1),
+ _ => Err(anyhow!("{}", describe(&failures))),
+ }
+ }
+
+ /// Sends `message` to its `address` (dropped silently if the firewall blocks it).
+ pub(crate) async fn send(&self, message: &EncodedMessage) -> Result<()> {
+ if self
+ .gate()
+ .is_some_and(|gate| gate.blocks_outbound(&message.address))
+ {
+ tracing::trace!(to = %message.address, "Firewall dropped outbound message");
+ return Ok(());
+ }
+ let (link, target) = self
+ .split(&message.address)
+ .with_context(|| format!("no transport for {}", message.address))?;
+ let mut inner = message.clone();
+ inner.peer = target.clone();
+ inner.address = target;
+ link.inner.send_encoded(&inner).await
+ }
+
+ /// How the transport of `address` is connected there (`Transport::connection`).
+ pub(crate) fn connection(&self, address: &str) -> Option<Connection> {
+ let (link, target) = self.split(address)?;
+ link.inner.connection(&target)
+ }
+
+ /// Paths' own figures, and every transport's.
+ pub(crate) fn stats(&self) -> TransportStats {
+ let mut stats = TransportStats::default();
+ for link in &self.links {
+ stats.merge(link.inner.stats());
+ }
+ let now = Instant::now();
+ let (paths, counts) = {
+ let table = self.table.lock_or_recover();
+ let paths: Vec<PathInfo> = table
+ .paths
+ .keys()
+ .map(|address| table.info(address, now))
+ .collect();
+ (paths, table.counts)
+ };
+ let mut paths = self.located(paths);
+ paths.sort_by(|a, b| a.address.cmp(&b.address));
+ stats.paths = Some(PathsStats { paths, counts });
+ stats
+ }
+
+ /// Sends a PING (or with `pong`, a PONG) with `nonce` to `address`, counted
+ /// (`traffic`).
+ async fn send_probe(&self, address: &str, pong: bool, nonce: u64) -> Result<()> {
+ let message = if pong {
+ ParsedMessage::Pong(nonce)
+ } else {
+ ParsedMessage::Ping(nonce)
+ };
+ let encoded = self.encode(address, &message)?;
+ let sent = self.send(&encoded).await;
+ if let Some(traffic) = &self.traffic {
+ match &sent {
+ Ok(()) => traffic.sent(address, encoded.kind, encoded.payload_len),
+ Err(_) => traffic.refused(address, encoded.kind, encoded.payload_len),
+ }
+ }
+ sent
+ }
+
+ /// Pings the paths due (`Table::due`), every `PROBE_TICK`, until dropped. Each PING
+ /// goes on its own, timed from when it goes: a send that takes long (a dial) neither
+ /// holds up the others nor counts in their round trips.
+ async fn probe(self: Arc<Self>) {
+ let mut tick = tokio::time::interval(PROBE_TICK);
+ tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ loop {
+ tick.tick().await;
+ let due = self
+ .table
+ .lock_or_recover()
+ .due(Instant::now(), random_nonce);
+ for (address, nonce) in due {
+ let paths = Arc::clone(&self);
+ tokio::spawn(async move {
+ paths
+ .table
+ .lock_or_recover()
+ .sending(&address, nonce, Instant::now());
+ if let Err(error) = paths.send_probe(&address, false, nonce).await {
+ debug!(to = %address, error = %format!("{error:#}"), "Couldn't send a PING");
+ paths
+ .table
+ .lock_or_recover()
+ .failed(&address, Instant::now());
+ }
+ });
+ }
+ }
+ }
+
+ /// A message from `address`, handed on to `handler`: a PING answered and a PONG
+ /// noted first; dropped if the firewall blocks it.
+ async fn receive(
+ self: Arc<Self>,
+ address: String,
+ message: ParsedMessage,
+ payload_len: usize,
+ handler: &PathHandler,
+ ) -> Result<()> {
+ if self
+ .gate()
+ .is_some_and(|gate| gate.blocks_inbound(&address))
+ {
+ tracing::trace!(from = %address, "Firewall dropped inbound message");
+ return Ok(());
+ }
+ match message {
+ ParsedMessage::Ping(nonce) => {
+ let paths = Arc::clone(&self);
+ let to = address.clone();
+ tokio::spawn(async move {
+ if let Err(error) = paths.send_probe(&to, true, nonce).await {
+ debug!(to = %to, error = %format!("{error:#}"), "Couldn't answer a PING");
+ }
+ });
+ }
+ ParsedMessage::Pong(nonce) => {
+ self.table
+ .lock_or_recover()
+ .pong(&address, nonce, Instant::now());
+ }
+ _ => {}
+ }
+ handler(address, message, payload_len).await
+ }
+
+ /// Every transport's consumer, each message handed to `handler` under the sender's
+ /// address (PINGs and PONGs end here), and the probe task; returns when the first
+ /// consumer does.
+ pub(crate) async fn start_consumer(self: &Arc<Self>, handler: Arc<PathHandler>) -> Result<()> {
+ let probing = tokio::spawn(Arc::clone(self).probe());
+ let mut consumers: Vec<std::pin::Pin<Box<dyn Future<Output = Result<()>> + Send>>> =
+ Vec::new();
+ for link in &self.links {
+ let inner = Arc::clone(&link.inner);
+ let scheme = link.scheme;
+ let handler = Arc::clone(&handler);
+ let paths = Arc::clone(self);
+ consumers.push(Box::pin(async move {
+ inner
+ .start_consumer(Box::new(move |sender, message, payload_len| {
+ let paths = Arc::clone(&paths);
+ let handler = Arc::clone(&handler);
+ Box::pin(async move {
+ paths
+ .receive(
+ format!("{scheme}:{sender}"),
+ message,
+ payload_len,
+ &*handler,
+ )
+ .await
+ })
+ }))
+ .await
+ }));
+ }
+ let result = first(consumers)
+ .await
+ .unwrap_or_else(|| Err(anyhow!("no transports")));
+ probing.abort();
+ result
+ }
+
+ /// When the first transport exits.
+ pub(crate) async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> {
+ let waits = self
+ .links
+ .iter()
+ .map(|link| {
+ let inner = Arc::clone(&link.inner);
+ Box::pin(async move { inner.wait_for_exit().await })
+ as std::pin::Pin<Box<dyn Future<Output = _> + Send>>
+ })
+ .collect();
+ first(waits)
+ .await
+ .unwrap_or_else(|| Err(std::io::Error::other("no transports")))
+ }
+}
+
+/// A PING's nonce: random, so a PONG can't be made up for a PING not seen.
+pub(crate) fn random_nonce() -> u64 {
+ let mut bytes = [0u8; 8];
+ if getrandom::fill(&mut bytes).is_err() {
+ // No randomness: a nonce still tells answers apart, if not unguessably.
+ return std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map_or(0, |since| since.as_nanos() as u64);
+ }
+ u64::from_le_bytes(bytes)
+}
+
+/// `address: error; …` for each failed send.
+fn describe(failures: &[(String, anyhow::Error)]) -> String {
+ failures
+ .iter()
+ .map(|(address, error)| format!("{address}: {error:#}"))
+ .collect::<Vec<_>>()
+ .join("; ")
+}
+
+/// The first of the `results` to arrive, from tasks that run until they end.
+async fn first<T: Send + 'static>(
+ tasks: Vec<std::pin::Pin<Box<dyn Future<Output = T> + Send>>>,
+) -> Option<T> {
+ let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
+ for task in tasks {
+ let sender = sender.clone();
+ tokio::spawn(async move {
+ let _ = sender.send(task.await).await;
+ });
+ }
+ drop(sender);
+ receiver.recv().await
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::document::replica::Epoch;
+ use crate::transport::sim::SimNet;
+ use crate::transport::{HeadsSummary, Status};
+
+ /// "a" on two sim nets, as `sim` (preferred) and `other`; "b" on both too.
+ fn two_nets() -> (Arc<Paths>, Arc<Paths>) {
+ let (first, second) = (SimNet::new(1), SimNet::new(2));
+ let on_both = |id: &str| {
+ Arc::new(Paths::new(
+ vec![
+ ("sim", first.transport(id) as Arc<dyn Transport>),
+ ("other", second.transport(id) as Arc<dyn Transport>),
+ ],
+ None,
+ None,
+ ))
+ };
+ (on_both("a"), on_both("b"))
+ }
+
+ #[test]
+ fn addresses_by_scheme() {
+ let (a, _b) = two_nets();
+ assert_eq!(a.local_addresses(), ["sim:a", "other:a"]);
+ assert_eq!(a.transports(), ["sim", "other"]);
+ assert_eq!(a.canonical("other:b").as_deref(), Some("other:b"));
+ assert_eq!(a.canonical("pigeon:b"), None);
+ assert_eq!(a.canonical("b"), None); // No scheme.
+ assert!(a.preference("sim:b") < a.preference("other:b"));
+ assert_eq!(a.preference("pigeon:b"), usize::MAX);
+ assert!(a.is_own("other:a") && !a.is_own("other:b"));
+ assert_eq!(a.own_address("other:b").as_deref(), Some("other:a"));
+ assert_eq!(a.own_address("pigeon:b"), None);
+ }
+
+ #[tokio::test]
+ async fn seeds_resolve_to_addresses() {
+ let (a, _b) = two_nets();
+ assert_eq!(a.resolve("other:b").await.unwrap(), "other:b");
+ // Without a scheme: the first transport that resolves it.
+ assert_eq!(a.resolve("b").await.unwrap(), "sim:b");
+ assert!(a.resolve("nobody").await.is_err());
+ }
+
+ fn paths(addresses: &[&str]) -> Vec<(String, usize)> {
+ addresses
+ .iter()
+ .enumerate()
+ .map(|(preference, address)| (address.to_string(), preference))
+ .collect()
+ }
+
+ /// Answers every PING due `rtt` later, as from the path it went to (`rtts`).
+ fn answer(table: &mut Table, now: Instant, rtts: &[(&str, u64)]) -> Instant {
+ let mut nonces = 0;
+ let due = table.due(now, || {
+ nonces += 1;
+ nonces
+ });
+ for (address, nonce) in due {
+ let rtt = rtts
+ .iter()
+ .find(|(to, _)| *to == address)
+ .map(|(_, ms)| *ms);
+ if let Some(ms) = rtt {
+ table.pong(&address, nonce, now + Duration::from_millis(ms));
+ }
+ }
+ now + PROBE_EVERY
+ }
+
+ #[test]
+ fn the_configured_order_until_measured_then_the_faster_path() {
+ let mut table = Table::default();
+ let mut now = Instant::now();
+ let set = paths(&["libp2p:x", "mycelium:x"]);
+ assert_eq!(table.rank(set.clone(), now), ["libp2p:x", "mycelium:x"]);
+ // libp2p a bit slower: not enough to switch.
+ for _ in 0..8 {
+ now = answer(&mut table, now, &[("libp2p:x", 110), ("mycelium:x", 100)]);
+ }
+ assert_eq!(table.rank(set.clone(), now), ["libp2p:x", "mycelium:x"]);
+ assert_eq!(table.counts.switches, 0);
+ // Much slower: switched, and it stays switched while that holds.
+ for _ in 0..8 {
+ now = answer(&mut table, now, &[("libp2p:x", 300), ("mycelium:x", 100)]);
+ }
+ assert_eq!(table.rank(set.clone(), now), ["mycelium:x", "libp2p:x"]);
+ assert_eq!(table.rank(set.clone(), now), ["mycelium:x", "libp2p:x"]);
+ assert_eq!(table.counts.switches, 1);
+ // Small differences under SWITCH_MIN never switch.
+ let lan = paths(&["libp2p:y", "mycelium:y"]);
+ table.rank(lan.clone(), now);
+ for _ in 0..8 {
+ now = answer(&mut table, now, &[("libp2p:y", 4), ("mycelium:y", 1)]);
+ }
+ assert_eq!(table.rank(lan, now), ["libp2p:y", "mycelium:y"]);
+ }
+
+ #[test]
+ fn unanswered_or_failing_paths_are_down() {
+ let mut table = Table::default();
+ let mut now = Instant::now();
+ let set = paths(&["libp2p:x", "mycelium:x"]);
+ table.rank(set.clone(), now);
+ // libp2p stops answering: down after UNANSWERED_DOWN PINGs, then last.
+ for _ in 0..=UNANSWERED_DOWN {
+ now = answer(&mut table, now, &[("mycelium:x", 100)]);
+ table.rank(set.clone(), now);
+ }
+ assert!(table.info("libp2p:x", now).down);
+ assert_eq!(table.rank(set.clone(), now), ["mycelium:x", "libp2p:x"]);
+ // It answers again: up, and measured.
+ now = answer(&mut table, now, &[("libp2p:x", 50), ("mycelium:x", 100)]);
+ assert!(!table.info("libp2p:x", now).down);
+
+ // A failed send: down for DOWN_FOR, unless a send there works.
+ table.failed("mycelium:x", now);
+ assert!(table.info("mycelium:x", now).down);
+ assert!(!table.info("mycelium:x", now + DOWN_FOR).down);
+ table.sent("mycelium:x");
+ assert!(!table.info("mycelium:x", now).down);
+ }
+
+ #[test]
+ fn only_paths_in_use_are_pinged() {
+ let mut table = Table::default();
+ let now = Instant::now();
+ let mut nonce = 0;
+ let mut next = || {
+ nonce += 1;
+ nonce
+ };
+ // One address: nothing to choose, nothing to measure.
+ table.rank(paths(&["libp2p:z"]), now);
+ assert!(table.due(now, &mut next).is_empty());
+ table.rank(paths(&["libp2p:x", "mycelium:x"]), now);
+ assert_eq!(table.due(now, &mut next).len(), 2);
+ // Not again within PROBE_EVERY; a stale answer is ignored.
+ assert!(table.due(now + PROBE_TIMEOUT, &mut next).is_empty());
+ assert_eq!(table.pong("libp2p:x", 99, now), None);
+ // Unused for ACTIVE_FOR: no more PINGs; for twice as long: forgotten.
+ assert!(table.due(now + ACTIVE_FOR, &mut next).is_empty());
+ table.due(now + ACTIVE_FOR * 2, &mut next);
+ assert!(table.paths.is_empty() && table.chosen.is_empty());
+ }
+
+ /// A PING's round trip counts from when it goes, not from when it was found due.
+ #[test]
+ fn round_trips_count_from_the_send() {
+ let mut table = Table::default();
+ let now = Instant::now();
+ table.rank(paths(&["libp2p:x", "mycelium:x"]), now);
+ let due = table.due(now, || 7);
+ assert_eq!(due.len(), 2);
+ // Sent a second late (after another path's slow send).
+ let sent = now + Duration::from_secs(1);
+ table.sending("libp2p:x", 7, sent);
+ table.sending("libp2p:x", 8, sent + Duration::from_secs(1)); // not its PING
+ table.pong("libp2p:x", 7, sent + Duration::from_millis(100));
+ let rtt = table.info("libp2p:x", now).rtt_ms.unwrap();
+ assert!((rtt - 100.0).abs() < 1e-6, "{rtt}");
+ }
+
+ /// The node's keepalive PING on a path is its probe: paths doesn't ping it too, and
+ /// the PONG measures it.
+ #[test]
+ fn keepalive_pings_are_probes() {
+ let mut table = Table::default();
+ let now = Instant::now();
+ let mut nonce = 100;
+ let mut next = || {
+ nonce += 1;
+ nonce
+ };
+ table.rank(paths(&["libp2p:x", "mycelium:x"]), now);
+ table.pinged("libp2p:x", 7, now);
+ let due = table.due(now, &mut next);
+ assert_eq!(due.len(), 1);
+ assert_eq!(due[0].0, "mycelium:x");
+ assert!(
+ table
+ .pong("libp2p:x", 7, now + Duration::from_millis(20))
+ .is_some()
+ );
+ assert_eq!(table.counts.probes_sent, 2);
+ // A peer's only path is measured by its keepalives, and not pinged otherwise.
+ table.pinged("libp2p:z", 8, now);
+ assert!(
+ table
+ .pong("libp2p:z", 8, now + Duration::from_millis(30))
+ .is_some()
+ );
+ let rtt = table.info("libp2p:z", now).rtt_ms.unwrap();
+ assert!((rtt - 30.0).abs() < 1e-6, "{rtt}");
+ let later = now + PROBE_EVERY;
+ assert!(
+ table
+ .due(later, &mut next)
+ .iter()
+ .all(|(address, _)| address != "libp2p:z")
+ );
+ // Forgotten once the keepalives stop.
+ table.due(now + ACTIVE_FOR * 2, &mut next);
+ assert!(!table.paths.contains_key("libp2p:z"));
+ }
+
+ /// Blocks one address, both ways.
+ struct Blocking(&'static str);
+
+ impl Gate for Blocking {
+ fn blocks_inbound(&self, address: &str) -> bool {
+ address == self.0
+ }
+ fn blocks_outbound(&self, address: &str) -> bool {
+ address == self.0
+ }
+ }
+
+ /// The firewall drops what comes from a blocked address before anything sees it
+ /// (a PING isn't answered), and sends to one silently.
+ #[tokio::test]
+ async fn the_firewall_drops_both_ways() {
+ let net = SimNet::new(4);
+ let gate: Arc<dyn Gate> = Arc::new(Blocking("sim:a"));
+ let b = Arc::new(Paths::new(
+ vec![("sim", net.transport("b") as Arc<dyn Transport>)],
+ None,
+ Some(Arc::downgrade(&gate)),
+ ));
+ let a = Arc::new(Paths::new(
+ vec![("sim", net.transport("a") as Arc<dyn Transport>)],
+ None,
+ None,
+ ));
+ let received = |paths: &Arc<Paths>| {
+ let (sender, received) = tokio::sync::mpsc::unbounded_channel();
+ let handler: Arc<PathHandler> = Arc::new(move |from, message, _| {
+ let sender = sender.clone();
+ Box::pin(async move {
+ let _ = sender.send((from, message.kind()));
+ Ok(())
+ })
+ });
+ let paths = Arc::clone(paths);
+ tokio::spawn(async move { paths.start_consumer(handler).await });
+ received
+ };
+ let (mut at_a, mut at_b) = (received(&a), received(&b));
+
+ // a's PING never reaches b, so nothing answers it.
+ let ping = a.encode("sim:b", &ParsedMessage::Ping(5)).unwrap();
+ a.send(&ping).await.unwrap();
+ // b's send to a "works", and goes nowhere.
+ let status = b.encode("sim:a", &some_status()).unwrap();
+ b.send(&status).await.unwrap();
+ tokio::time::sleep(Duration::from_millis(200)).await;
+ assert!(at_b.try_recv().is_err());
+ assert!(at_a.try_recv().is_err());
+
+ // Without the gate: a PING is answered by b's paths, and goes on up there.
+ drop(gate);
+ a.send(&ping).await.unwrap();
+ assert_eq!(
+ at_b.recv().await.unwrap(),
+ ("sim:a".to_owned(), crate::transport::MessageKind::Ping)
+ );
+ assert_eq!(
+ at_a.recv().await.unwrap(),
+ ("sim:b".to_owned(), crate::transport::MessageKind::Pong)
+ );
+ }
+
+ /// A transport whose sends all fail.
+ struct Broken;
+
+ #[async_trait::async_trait]
+ impl Transport for Broken {
+ fn local_id(&self) -> &str {
+ "us"
+ }
+ fn is_peer_id(&self, id: &str) -> bool {
+ !id.is_empty()
+ }
+ fn delivery_limit(&self) -> Duration {
+ Duration::from_secs(1)
+ }
+ async fn resolve_seed(&self, seed: &str) -> Result<String> {
+ Ok(seed.to_owned())
+ }
+ fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
+ crate::transport::mycelium::encode_message(peer, message)
+ }
+ async fn send_encoded(&self, _message: &EncodedMessage) -> Result<()> {
+ Err(anyhow!("broken"))
+ }
+ async fn start_consumer(&self, _handler: crate::transport::MessageHandler) -> Result<()> {
+ std::future::pending().await
+ }
+ async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> {
+ std::future::pending().await
+ }
+ }
+
+ fn some_status() -> ParsedMessage {
+ ParsedMessage::Status(Status {
+ heads: HeadsSummary::of(Vec::new()),
+ epoch: Epoch::GENESIS,
+ epoch_seq: 0,
+ ready: false,
+ syncs_with_you: false,
+ })
+ }
+
+ /// When the preferred transport fails, the same send goes over the next one, and
+ /// the failed address is ranked last for a while.
+ #[tokio::test]
+ async fn a_failed_send_goes_to_the_next_address() {
+ let net = SimNet::new(3);
+ let a = Paths::new(
+ vec![
+ ("broken", Arc::new(Broken) as Arc<dyn Transport>),
+ ("sim", net.transport("a") as Arc<dyn Transport>),
+ ],
+ None,
+ None,
+ );
+ let b = Arc::new(Paths::new(
+ vec![("sim", net.transport("b") as Arc<dyn Transport>)],
+ None,
+ None,
+ ));
+ let (sender, mut received) = tokio::sync::mpsc::unbounded_channel();
+ let handler: Arc<PathHandler> = Arc::new(move |from, _, _| {
+ let sender = sender.clone();
+ Box::pin(async move {
+ let _ = sender.send(from);
+ Ok(())
+ })
+ });
+ tokio::spawn(async move { b.start_consumer(handler).await });
+
+ let addresses = vec!["sim:b".to_owned(), "broken:b".to_owned()];
+ let ranked = a.rank(&addresses);
+ assert_eq!(ranked, ["broken:b", "sim:b"]); // The configured order.
+ let mut encoded = a.encode(&ranked[0], &some_status()).unwrap();
+ encoded.peer = "node-b".to_owned();
+ assert_eq!(a.send_first(&encoded, &ranked).await.unwrap(), "sim:b");
+ assert_eq!(received.recv().await.unwrap(), "sim:a");
+ assert_eq!(a.rank(&addresses), ["sim:b", "broken:b"]);
+
+ // Without the message kept, nothing else can be tried: the transport's error.
+ encoded.message = None;
+ let error = a
+ .send_first(&encoded, &["broken:b".to_owned()])
+ .await
+ .unwrap_err();
+ assert_eq!(error.to_string(), "broken");
+ assert!(a.send_first(&encoded, &[]).await.is_err());
+ }
+
+ /// A message sent to an address arrives under the sender's address on that
+ /// transport.
+ #[tokio::test]
+ async fn messages_arrive_under_the_senders_address() {
+ let (a, b) = two_nets();
+ let (sender, mut received) = tokio::sync::mpsc::unbounded_channel();
+ let handler: Arc<PathHandler> = Arc::new(move |from, message, _| {
+ let sender = sender.clone();
+ Box::pin(async move {
+ let _ = sender.send((from, message.kind()));
+ Ok(())
+ })
+ });
+ tokio::spawn(async move { b.start_consumer(handler).await });
+ let status = some_status();
+ for to in ["other:b", "sim:b"] {
+ let encoded = a.encode(to, &status).unwrap();
+ assert_eq!(encoded.address, to);
+ a.send(&encoded).await.unwrap();
+ }
+ let mut from: Vec<String> = Vec::new();
+ for _ in 0..2 {
+ from.push(received.recv().await.unwrap().0);
+ }
+ from.sort();
+ assert_eq!(from, ["other:a", "sim:a"]);
+ assert!(a.encode("pigeon:b", &status).is_err());
+ }
+}
diff --git a/src/transport/sim.rs b/src/transport/sim.rs
index ce5977f..500050b 100644
--- a/src/transport/sim.rs
+++ b/src/transport/sim.rs
@@ -26,6 +26,7 @@ use tracing::{debug, warn};
use super::mycelium::{decode_message, encode_message};
use super::{EncodedMessage, MessageHandler, MessageKind, ParsedMessage, Transport};
+use crate::locks::MutexExt;
use crate::testrand::Rng;
/// How one direction of a link behaves.
@@ -102,9 +103,7 @@ impl SimNet {
}
fn lock(&self) -> MutexGuard<'_, NetInner> {
- self.inner
- .lock()
- .unwrap_or_else(|poisoned| poisoned.into_inner())
+ self.inner.lock_or_recover()
}
/// A transport for a new node `id` on this net.
@@ -256,6 +255,15 @@ impl Transport for SimTransport {
&self.id
}
+ /// Any name will do.
+ fn is_peer_id(&self, id: &str) -> bool {
+ !id.is_empty()
+ }
+
+ fn delivery_limit(&self) -> std::time::Duration {
+ crate::simtest::DELIVERY_LIMIT
+ }
+
/// A seed is a node ID; one no node has doesn't resolve (yet).
async fn resolve_seed(&self, seed: &str) -> Result<String> {
if self.net.is_known(seed) {
@@ -311,10 +319,10 @@ impl Transport for SimTransport {
mod tests {
use super::*;
use crate::document::replica::Epoch;
- use crate::transport::{HeadsSummary, Keepalive};
+ use crate::transport::{HeadsSummary, Status};
- fn ping() -> ParsedMessage {
- ParsedMessage::Ping(Keepalive {
+ fn status() -> ParsedMessage {
+ ParsedMessage::Status(Status {
heads: HeadsSummary::of(Vec::new()),
epoch: Epoch::GENESIS,
epoch_seq: 0,
@@ -345,13 +353,13 @@ mod tests {
let received = record(net.transport("b"));
let settle = || tokio::time::sleep(Duration::from_millis(30));
- a.send_message("b", &ping()).await.unwrap();
+ a.send_message("b", &status()).await.unwrap();
settle().await;
net.take_down("b");
- a.send_message("b", &ping()).await.unwrap();
+ a.send_message("b", &status()).await.unwrap();
settle().await;
net.bring_up("b");
- a.send_message("b", &ping()).await.unwrap();
+ a.send_message("b", &status()).await.unwrap();
settle().await;
assert_eq!(*received.lock().unwrap(), vec!["a", "a"]);
@@ -373,7 +381,7 @@ mod tests {
let a = net.transport("a");
let _b = net.transport("b");
for _ in 0..200 {
- a.send_message("b", &ping()).await.unwrap();
+ a.send_message("b", &status()).await.unwrap();
}
tokio::time::sleep(Duration::from_millis(50)).await;
net.counts()
@@ -400,8 +408,8 @@ mod tests {
let at_a = record(Arc::clone(&a));
let at_b = record(Arc::clone(&b));
- a.send_message("b", &ping()).await.unwrap();
- b.send_message("a", &ping()).await.unwrap();
+ a.send_message("b", &status()).await.unwrap();
+ b.send_message("a", &status()).await.unwrap();
tokio::time::sleep(Duration::from_millis(30)).await;
assert!(at_b.lock().unwrap().is_empty());
@@ -412,8 +420,8 @@ mod tests {
async fn inject_returns_once_handled() {
let net = SimNet::new(1);
let received = record(net.transport("b"));
- net.inject("ghost", "b", &ping()).await.unwrap();
+ net.inject("ghost", "b", &status()).await.unwrap();
assert_eq!(*received.lock().unwrap(), vec!["ghost"]);
- assert!(net.inject("ghost", "nobody", &ping()).await.is_err());
+ assert!(net.inject("ghost", "nobody", &status()).await.is_err());
}
}
diff --git a/src/transport/stats.rs b/src/transport/stats.rs
new file mode 100644
index 0000000..398ffdf
--- /dev/null
+++ b/src/transport/stats.rs
@@ -0,0 +1,189 @@
+//! Figures about the transports themselves (`Transport::stats`), for /admin/stats
+//! (`transports`, `docs/measurements.md`): how they find and reach peers, besides the
+//! messages counted per peer (`traffic`).
+
+use std::collections::BTreeMap;
+
+use serde::Serialize;
+
+#[derive(Debug, Clone, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct TransportStats {
+ /// The identity layer's (`transport::identity`); absent without one.
+ pub(crate) identity: Option<IdentityStats>,
+ /// The paths layer's (`transport::paths`); absent without one.
+ pub(crate) paths: Option<PathsStats>,
+ /// The libp2p transport's; absent if it isn't one of this node's.
+ pub(crate) libp2p: Option<Libp2pStats>,
+}
+
+impl TransportStats {
+ /// Takes what `other` has.
+ pub(crate) fn merge(&mut self, other: TransportStats) {
+ if other.identity.is_some() {
+ self.identity = other.identity;
+ }
+ if other.paths.is_some() {
+ self.paths = other.paths;
+ }
+ if other.libp2p.is_some() {
+ self.libp2p = other.libp2p;
+ }
+ }
+}
+
+/// How the paths to peers with several addresses do (`transport::paths`).
+#[derive(Debug, Clone, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PathsStats {
+ /// Every path measured: those of peers this node chose among lately, and those its
+ /// keepalives went over.
+ pub(crate) paths: Vec<PathInfo>,
+ #[serde(flatten)]
+ pub(crate) counts: PathCounts,
+}
+
+/// One path, as `transport::paths` measures it.
+#[derive(Debug, Clone, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PathInfo {
+ /// `<scheme>:<where it goes>`: where the transport says the path goes now
+ /// (`Transport::location`; libp2p: the remote address of its connection, ending in
+ /// the peer ID), else whom it goes to, `<scheme>:<transport identity>` (a libp2p
+ /// peer ID without a connection, a Mycelium public key, an onion service).
+ pub(crate) address: String,
+ /// Smoothed round-trip time of its PINGs (ours, and our keepalives); absent before
+ /// an answer.
+ pub(crate) rtt_ms: Option<f64>,
+ /// Its PINGs went unanswered three times in a row, or a send there failed within a
+ /// minute: used only when no other path works.
+ pub(crate) down: bool,
+}
+
+/// Counts since start.
+#[derive(Debug, Clone, Copy, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PathCounts {
+ pub(crate) probes_sent: u64,
+ pub(crate) probes_answered: u64,
+ pub(crate) probes_unanswered: u64,
+ /// Sends that failed on a path (each tried at the next).
+ pub(crate) send_failures: u64,
+ /// Sends that went over another path after one failed.
+ pub(crate) failovers: u64,
+ /// Changes of the path in use for one faster.
+ pub(crate) switches: u64,
+}
+
+/// Who is where (`transport::identity`): the books now, and counts since start.
+#[derive(Debug, Clone, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct IdentityStats {
+ /// Nodes proven at an address, by transport (scheme); a node on several counts on
+ /// each.
+ pub(crate) proven: BTreeMap<String, usize>,
+ /// Nodes known only from hints (the member list, HELLO_REPLYs), proven nowhere.
+ pub(crate) hinted: usize,
+ #[serde(flatten)]
+ pub(crate) counts: IdentityCounts,
+}
+
+#[derive(Debug, Clone, Copy, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct IdentityCounts {
+ /// Proofs that didn't hold (a bad signature, or made for other addresses): normally
+ /// none.
+ pub(crate) proofs_rejected: u64,
+ /// Nodes that proved themselves at another address on a transport than before: a
+ /// node restarted with a new transport identity (a new libp2p key or Mycelium key).
+ pub(crate) address_changes: u64,
+ /// Messages dropped because no node proved itself at their address: some after a
+ /// restart are normal, a steady stream is not.
+ pub(crate) dropped_unproven: u64,
+ /// Sends refused because the peer hadn't proven where it is yet (`NotProven`).
+ pub(crate) refused_unproven: u64,
+ /// Seeds no node answered at in time.
+ pub(crate) seeds_unanswered: u64,
+}
+
+/// Whether others can dial this node, as AutoNAT found out.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) enum Reachability {
+ #[default]
+ Unknown,
+ /// Others can't dial it: it needs a relay.
+ Private,
+ Public,
+}
+
+impl Reachability {
+ pub(crate) const ALL: [Reachability; 3] = [Self::Unknown, Self::Private, Self::Public];
+
+ pub(crate) fn as_str(self) -> &'static str {
+ match self {
+ Self::Unknown => "unknown",
+ Self::Private => "private",
+ Self::Public => "public",
+ }
+ }
+}
+
+#[derive(Debug, Clone, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct Libp2pStats {
+ pub(crate) reachability: Reachability,
+ /// The address AutoNAT confirmed others can dial, while public.
+ pub(crate) public_address: Option<String>,
+ /// Open connections, now.
+ pub(crate) connections: Connections,
+ pub(crate) relay_slots: RelaySlots,
+ /// Direct connections tried over a relayed one (DCUtR), since start.
+ pub(crate) hole_punches: HolePunches,
+ /// Connections that couldn't be made, by cause, since start.
+ pub(crate) dial_failures: DialFailures,
+}
+
+#[derive(Debug, Clone, Copy, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct Connections {
+ pub(crate) direct: usize,
+ /// Through a relay (a circuit).
+ pub(crate) relayed: usize,
+}
+
+/// Slots this node holds on relays, to be reachable through them.
+#[derive(Debug, Clone, Copy, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct RelaySlots {
+ /// Held now.
+ pub(crate) held: usize,
+ /// Relays picked that never granted a slot, not picked again for a while now.
+ pub(crate) paused: usize,
+ /// Since start: slots granted (not counting renewals), lost after being granted, and
+ /// asked for but never granted (a relay unreachable, or another peer at its
+ /// address: a relay restarted without its libp2p key).
+ pub(crate) granted: u64,
+ pub(crate) lost: u64,
+ pub(crate) not_granted: u64,
+}
+
+#[derive(Debug, Clone, Copy, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct HolePunches {
+ pub(crate) succeeded: u64,
+ /// The relay carries on.
+ pub(crate) failed: u64,
+}
+
+#[derive(Debug, Clone, Copy, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct DialFailures {
+ /// No address of the peer could be reached (refused, timed out, no route).
+ pub(crate) unreachable: u64,
+ /// Another peer answered at the address: one restarted without its libp2p key.
+ pub(crate) wrong_peer_id: u64,
+ /// No address known for the peer.
+ pub(crate) no_addresses: u64,
+ pub(crate) other: u64,
+}
diff --git a/src/transport/tor.rs b/src/transport/tor.rs
new file mode 100644
index 0000000..b49cc67
--- /dev/null
+++ b/src/transport/tor.rs
@@ -0,0 +1,673 @@
+//! Tor as a transport (`--transports tor`, in builds with the `tor` feature), for
+//! reachability: every node runs an onion service (arti), reachable from anywhere
+//! without a forwarded port or a relay. Not for anonymity: a node's other addresses are
+//! in the member list next to its onion address.
+//!
+//! A node's transport identity is its onion service, `<onion>.onion:<port>`. The onion
+//! service's own key lives in arti's state (`--tor-state-dir`), so its address stays the
+//! same across restarts.
+//!
+//! Peers talk over one long-lived connection per pair (opening a stream over Tor costs
+//! a round trip through the rendezvous circuit): a Tor stream to the other's onion
+//! service, either way. Tor authenticates the onion service to the one who dials it,
+//! but nothing about the dialer: whoever dials arrives anonymously. So at the start of
+//! every connection, the challenge handshake (`transport::challenge`) proves both node
+//! keys, bound to the onion address dialed, and then the dialer says which onion
+//! service is its own (`send_claim`): the connection's peer from then on. That claim is
+//! checked before the connection is used (`Shared::check`): the onion service must be
+//! where the dialer's node key is. A dial proves that (Tor proves the onion service, the
+//! handshake the key there), so a claim holds if dialing the claimed onion service found
+//! the same key, now or before; if it wasn't dialed yet, the listener dials it to check,
+//! and closes that connection again. A claim whose check failed is refused without a
+//! dial for a while (`FAILED_CHECK_FOR`). Then messages go as on libp2p: their length (4
+//! bytes, big endian), then the message as `wire` encodes it. A connection that breaks
+//! is dialed again on the next send. Both ends may dial at once; each then sends on the
+//! newest connection and reads from both until the older one closes.
+//!
+//! Onion circuits are slow (round trips of a second or more, seconds to set one up),
+//! so the timeouts are generous and messages smaller than on libp2p; the paths layer
+//! (`transport::paths`) prefers faster transports while they work.
+
+use std::collections::HashMap;
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex, MutexGuard};
+use std::time::{Duration, Instant};
+
+use anyhow::{Context, Result, anyhow, bail};
+use arti_client::config::TorClientConfigBuilder;
+use arti_client::{DataStream, TorClient};
+use async_trait::async_trait;
+use futures::StreamExt;
+use safelog::DisplayRedacted as _;
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadHalf, WriteHalf};
+use tokio::sync::{mpsc, watch};
+use tor_cell::relaycell::msg::{Connected, End, EndReason};
+use tor_hsservice::config::OnionServiceConfigBuilder;
+use tor_hsservice::{RunningOnionService, StreamRequest, handle_rend_requests};
+use tor_proto::stream::IncomingStreamRequest;
+use tor_rtcompat::PreferredRuntime;
+use tracing::{debug, info, warn};
+
+use crate::identity::NodeKey;
+use crate::locks::MutexExt;
+use crate::transport::challenge::{self, Role};
+use crate::transport::{
+ EncodedMessage, HandlerFn, Hex, MessageHandler, ParsedMessage, Transport, hand_on_probe,
+ is_probe, wire,
+};
+
+/// A message read, with the sender's onion service and its payload length.
+type Inbound = (String, ParsedMessage, usize);
+
+/// The onion service's nickname in arti's state (one per node).
+const NICKNAME: &str = "continuum";
+/// The largest message read. Sync messages above 16 chunks travel as blobs, and a chunk
+/// is at most 4 MiB.
+const MAX_MESSAGE: usize = 8 << 20;
+/// How long reaching an onion service may take (building circuits, the rendezvous).
+const CONNECT_TIMEOUT: Duration = Duration::from_secs(60);
+/// How long the challenge handshake may take, a few round trips.
+const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30);
+/// How long a message may take, before what its size adds (`stream_timeout`).
+const STREAM_TIMEOUT: Duration = Duration::from_secs(30);
+/// The slowest a message is expected to go over Tor, in bytes per second.
+const MIN_RATE: u64 = 64 << 10;
+/// Messages read and not handled yet, at most.
+const READS: usize = 32;
+/// A claim whose check failed (`Shared::check`) is refused without dialing again for
+/// this long: claims of onion services that aren't the claimer's don't make us dial
+/// over and over. A dial of our own there that finds the claimer's key ends it early.
+const FAILED_CHECK_FOR: Duration = Duration::from_secs(600);
+/// Onion services remembered (with the node key found there, `Shared::proven`), at most.
+const MAX_PEERS: usize = 4096;
+
+/// How long a message of `bytes` may take.
+fn stream_timeout(bytes: usize) -> Duration {
+ STREAM_TIMEOUT + Duration::from_millis(bytes as u64 * 1000 / MIN_RATE)
+}
+
+/// What a Tor transport is made with.
+pub(crate) struct TorConfig {
+ /// The node key: it proves who we are in the challenge handshake.
+ pub(crate) key: Arc<NodeKey>,
+ /// Arti's state (the onion service's key, among others) and cache, in `state` and
+ /// `cache` below.
+ pub(crate) state_dir: PathBuf,
+ /// The onion service's port.
+ pub(crate) port: u16,
+}
+
+/// `<onion>.onion:<port>`, from `target`, which may end in `/<node key>` (how nodes
+/// published their onion service before it was their transport identity).
+fn parse_location(target: &str) -> Option<String> {
+ let location = match target.split_once('/') {
+ Some((location, key)) if is_key(key) => location,
+ Some(_) => return None,
+ None => target,
+ };
+ let (host, port) = location.rsplit_once(':')?;
+ let name = host.strip_suffix(".onion")?;
+ if name.is_empty() || !name.bytes().all(|byte| byte.is_ascii_alphanumeric()) {
+ return None;
+ }
+ port.parse::<u16>().ok()?;
+ Some(location.to_ascii_lowercase())
+}
+
+/// Whether `id` is a node key, 64 lowercase hex digits.
+fn is_key(id: &str) -> bool {
+ id.len() == 64
+ && id
+ .bytes()
+ .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f'))
+}
+
+/// After the handshake, the dialer says what the connection is for (`read_claim`):
+/// carrying messages from its onion service, `claim`, or (empty) only checking which
+/// node key is at the listener's (`Shared::check`).
+async fn send_claim<S: AsyncWrite + Unpin>(stream: &mut S, claim: &str) -> Result<()> {
+ let length = u8::try_from(claim.len()).map_err(|_| anyhow!("{claim} is too long"))?;
+ stream.write_all(&[length]).await?;
+ stream.write_all(claim.as_bytes()).await?;
+ stream.flush().await?;
+ Ok(())
+}
+
+/// The dialer's claim (`send_claim`): its onion service, or None for a check.
+async fn read_claim<S: AsyncRead + Unpin>(stream: &mut S) -> Result<Option<String>> {
+ let mut length = [0u8; 1];
+ stream.read_exact(&mut length).await?;
+ if length[0] == 0 {
+ return Ok(None);
+ }
+ let mut claim = vec![0u8; length[0] as usize];
+ stream.read_exact(&mut claim).await?;
+ let claim = String::from_utf8(claim).map_err(|_| anyhow!("a claim that isn't text"))?;
+ match parse_location(&claim) {
+ Some(location) if location == claim => Ok(Some(location)),
+ _ => bail!("{claim} is no onion address with a port"),
+ }
+}
+
+/// One connection to a peer: where to write. A reader task reads the other half.
+struct Connection {
+ id: u64,
+ writer: tokio::sync::Mutex<WriteHalf<DataStream>>,
+}
+
+struct Shared {
+ key: Arc<NodeKey>,
+ client: Arc<TorClient<PreferredRuntime>>,
+ /// Our onion service, `<onion>.onion:<port>`: our transport identity.
+ local_id: String,
+ /// The node key found at each onion service dialed: proven there (`check`).
+ proven: Mutex<HashMap<String, [u8; 32]>>,
+ /// Claims whose check failed lately (`FAILED_CHECK_FOR`): an onion service and the
+ /// node key claiming it, and when.
+ failed_checks: Mutex<HashMap<(String, [u8; 32]), Instant>>,
+ /// The connection to write to each peer on, by onion service.
+ connections: Mutex<HashMap<String, Arc<Connection>>>,
+ /// One dial at a time per onion service.
+ dialing: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
+ next_connection: AtomicU64,
+ /// Messages read, for the consumer: in order, and PINGs and PONGs apart, to be
+ /// handed on at once (`is_probe`).
+ inbound: mpsc::Sender<Inbound>,
+ probes: mpsc::Sender<Inbound>,
+}
+
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+ mutex.lock_or_recover()
+}
+
+pub(crate) struct TorTransport {
+ shared: Arc<Shared>,
+ /// Kept: dropping it stops the onion service.
+ _service: Arc<RunningOnionService>,
+ /// Messages, and PINGs and PONGs (`Shared::inbound`, `Shared::probes`).
+ incoming: tokio::sync::Mutex<Option<(mpsc::Receiver<Inbound>, mpsc::Receiver<Inbound>)>>,
+ stopped: watch::Receiver<bool>,
+}
+
+impl TorTransport {
+ /// Launches our onion service, and bootstraps arti in the background (that takes a
+ /// while: it fetches what it needs to know of the Tor network). Until it is done,
+ /// dials wait for it (arti bootstraps on demand), and the onion service is published
+ /// once it is: the node starts without waiting for Tor.
+ pub(crate) async fn start(config: TorConfig) -> Result<Self> {
+ let tor_config = TorClientConfigBuilder::from_directories(
+ config.state_dir.join("state"),
+ config.state_dir.join("cache"),
+ )
+ .build()
+ .context("Configuring Tor")?;
+ let client = TorClient::builder()
+ .config(tor_config)
+ .create_unbootstrapped_async()
+ .await
+ .context("Starting Tor")?;
+ {
+ let client = client.clone();
+ let state_dir = config.state_dir.display().to_string();
+ info!(state_dir = %state_dir, "Bootstrapping Tor");
+ // Launching the onion service starts it too; this waits for it to be done.
+ tokio::spawn(async move {
+ match client.bootstrap().await {
+ Ok(()) => info!("Tor bootstrapped"),
+ // Dials try again (on demand).
+ Err(error) => warn!(error = %error, "Bootstrapping Tor failed"),
+ }
+ });
+ }
+ let service_config = OnionServiceConfigBuilder::default()
+ .nickname(
+ NICKNAME
+ .parse()
+ .map_err(|error| anyhow!("Onion service nickname: {error}"))?,
+ )
+ .build()
+ .context("Configuring the onion service")?;
+ let (service, requests) = client
+ .launch_onion_service(service_config)
+ .context("Launching the onion service")?
+ .ok_or_else(|| anyhow!("The onion service is disabled in arti's configuration"))?;
+ let name = service
+ .onion_address()
+ .ok_or_else(|| anyhow!("The onion service has no address"))?;
+ // Arti shows onion addresses redacted unless asked: we publish ours.
+ let onion = format!("{}:{}", name.display_unredacted(), config.port);
+
+ let (inbound, incoming) = mpsc::channel(READS);
+ let (probes, incoming_probes) = mpsc::channel(READS);
+ let shared = Arc::new(Shared {
+ key: config.key,
+ client,
+ local_id: onion.clone(),
+ proven: Mutex::default(),
+ failed_checks: Mutex::default(),
+ connections: Mutex::default(),
+ dialing: Mutex::default(),
+ next_connection: AtomicU64::new(0),
+ inbound,
+ probes,
+ });
+ let (stopping, stopped) = watch::channel(false);
+ {
+ let shared = Arc::clone(&shared);
+ let port = config.port;
+ tokio::spawn(async move {
+ let requests = handle_rend_requests(requests);
+ futures::pin_mut!(requests);
+ while let Some(request) = requests.next().await {
+ let shared = Arc::clone(&shared);
+ tokio::spawn(async move { shared.accept(request, port).await });
+ }
+ warn!("The onion service stopped");
+ let _ = stopping.send(true);
+ });
+ }
+ info!(address = %onion, "Transport started (the onion service may take a minute to be reachable)");
+ Ok(Self {
+ shared,
+ _service: service,
+ incoming: tokio::sync::Mutex::new(Some((incoming, incoming_probes))),
+ stopped,
+ })
+ }
+}
+
+impl Shared {
+ /// A connection to the onion service at `location`, made if there is none.
+ async fn connection(self: &Arc<Self>, location: &str) -> Result<Arc<Connection>> {
+ if let Some(connection) = lock(&self.connections).get(location) {
+ return Ok(Arc::clone(connection));
+ }
+ let dialing = self.dialing(location);
+ let _one_at_a_time = dialing.lock().await;
+ // Someone else may have connected meanwhile.
+ if let Some(connection) = lock(&self.connections).get(location) {
+ return Ok(Arc::clone(connection));
+ }
+ let (_, stream) = self.dial(location, &self.local_id).await?;
+ Ok(self.register(location.to_owned(), stream))
+ }
+
+ /// The lock that keeps dials to `location` one at a time.
+ fn dialing(&self, location: &str) -> Arc<tokio::sync::Mutex<()>> {
+ let mut dialing = lock(&self.dialing);
+ if dialing.len() >= MAX_PEERS {
+ // Only those in use.
+ dialing.retain(|_, lock| Arc::strong_count(lock) > 1);
+ }
+ Arc::clone(dialing.entry(location.to_owned()).or_default())
+ }
+
+ /// Dials the onion service at `location`, runs the handshake and says what for
+ /// (`send_claim`): returns the node key there, now proven there, and the stream.
+ async fn dial(&self, location: &str, claim: &str) -> Result<([u8; 32], DataStream)> {
+ let (host, port) = location
+ .rsplit_once(':')
+ .and_then(|(host, port)| Some((host, port.parse::<u16>().ok()?)))
+ .ok_or_else(|| anyhow!("{location} is no onion address with a port"))?;
+ let mut stream = tokio::time::timeout(CONNECT_TIMEOUT, self.client.connect((host, port)))
+ .await
+ .map_err(|_| anyhow!("reaching {location} took longer than {CONNECT_TIMEOUT:?}"))?
+ .with_context(|| format!("Reaching {location}"))?;
+ let key = challenge::run(
+ &mut stream,
+ &self.key,
+ Role::Dialer,
+ location,
+ HANDSHAKE_TIMEOUT,
+ )
+ .await
+ .with_context(|| format!("Handshake with {location}"))?;
+ tokio::time::timeout(HANDSHAKE_TIMEOUT, send_claim(&mut stream, claim))
+ .await
+ .map_err(|_| anyhow!("no handshake with {location} within {HANDSHAKE_TIMEOUT:?}"))?
+ .with_context(|| format!("Handshake with {location}"))?;
+ let mut proven = lock(&self.proven);
+ if proven.len() < MAX_PEERS || proven.contains_key(location) {
+ proven.insert(location.to_owned(), key);
+ }
+ Ok((key, stream))
+ }
+
+ /// Whether the node with `key` is at the onion service at `location` (a dialer
+ /// claims it is: module docs). Dialed to find out, unless that was done already, or
+ /// failed within `FAILED_CHECK_FOR`.
+ async fn check(&self, location: &str, key: [u8; 32]) -> Result<bool> {
+ if lock(&self.proven).get(location) == Some(&key) {
+ return Ok(true);
+ }
+ let claim = (location.to_owned(), key);
+ if let Some(failed) = self.failed_lately(&claim) {
+ bail!("its check failed {}s ago", failed.as_secs());
+ }
+ let dialing = self.dialing(location);
+ let _one_at_a_time = dialing.lock().await;
+ // A dial or check meanwhile may have found out.
+ if lock(&self.proven).get(location) == Some(&key) {
+ return Ok(true);
+ }
+ if let Some(failed) = self.failed_lately(&claim) {
+ bail!("its check failed {}s ago", failed.as_secs());
+ }
+ let checked = self
+ .dial(location, "")
+ .await
+ .map(|(found, _check_only)| found == key);
+ if !matches!(checked, Ok(true)) {
+ let mut failed = lock(&self.failed_checks);
+ if failed.len() >= MAX_PEERS {
+ failed.retain(|_, at| at.elapsed() < FAILED_CHECK_FOR);
+ }
+ if failed.len() < MAX_PEERS {
+ failed.insert(claim, Instant::now());
+ }
+ }
+ checked
+ }
+
+ /// How long ago the check of `claim` failed, if within `FAILED_CHECK_FOR`.
+ fn failed_lately(&self, claim: &(String, [u8; 32])) -> Option<Duration> {
+ let mut failed = lock(&self.failed_checks);
+ let since = failed.get(claim)?.elapsed();
+ if since < FAILED_CHECK_FOR {
+ return Some(since);
+ }
+ failed.remove(claim);
+ None
+ }
+
+ /// A stream to our onion service: accepted if for our port, then the handshake, and
+ /// the dialer's claim checked (module docs).
+ async fn accept(self: Arc<Self>, request: StreamRequest, port: u16) {
+ let ours = matches!(request.request(), IncomingStreamRequest::Begin(begin) if begin.port() == port);
+ if !ours {
+ let _ = request
+ .reject(End::new_with_reason(EndReason::CONNECTREFUSED))
+ .await;
+ return;
+ }
+ let mut stream = match request.accept(Connected::new_empty()).await {
+ Ok(stream) => stream,
+ Err(error) => {
+ debug!(error = %error, "Couldn't accept a stream to our onion service");
+ return;
+ }
+ };
+ let handshake = async {
+ let key = challenge::run(
+ &mut stream,
+ &self.key,
+ Role::Listener,
+ &self.local_id,
+ HANDSHAKE_TIMEOUT,
+ )
+ .await?;
+ let claim = tokio::time::timeout(HANDSHAKE_TIMEOUT, read_claim(&mut stream))
+ .await
+ .map_err(|_| anyhow!("no claim within {HANDSHAKE_TIMEOUT:?}"))??;
+ Ok::<_, anyhow::Error>((key, claim))
+ };
+ let (key, claim) = match handshake.await {
+ Ok(done) => done,
+ Err(error) => {
+ debug!(error = %format!("{error:#}"), "Dropping a connection to our onion service: the handshake failed");
+ return;
+ }
+ };
+ // A check (`check`): the handshake was all.
+ let Some(location) = claim else {
+ return;
+ };
+ if location == self.local_id {
+ debug!("Dropping a connection to our onion service: it claims to be ours");
+ return;
+ }
+ match self.check(&location, key).await {
+ Ok(true) => {
+ self.register(location, stream);
+ }
+ Ok(false) => {
+ debug!(claimed = %location, key = %Hex(&key), "Dropping a connection to our onion service: another node is at the onion service it claims");
+ }
+ Err(error) => {
+ debug!(claimed = %location, error = %format!("{error:#}"), "Dropping a connection to our onion service: couldn't check the onion service it claims");
+ }
+ }
+ }
+
+ /// A connection to the onion service at `location`, authenticated: the one to write
+ /// to it on from now, and read from until it closes.
+ fn register(self: &Arc<Self>, location: String, stream: DataStream) -> Arc<Connection> {
+ let (reader, writer) = tokio::io::split(stream);
+ let id = self.next_connection.fetch_add(1, Ordering::Relaxed);
+ let connection = Arc::new(Connection {
+ id,
+ writer: tokio::sync::Mutex::new(writer),
+ });
+ lock(&self.connections).insert(location.clone(), Arc::clone(&connection));
+ debug!(peer = %location, connection = id, "Connection opened");
+ let shared = Arc::clone(self);
+ tokio::spawn(async move {
+ let error = shared.read(&location, reader).await;
+ debug!(peer = %location, connection = id, error = %format!("{error:#}"), "Connection closed");
+ shared.forget(&location, id);
+ });
+ connection
+ }
+
+ /// Reads messages from `location` until the connection fails; returns why.
+ async fn read(&self, location: &str, mut reader: ReadHalf<DataStream>) -> anyhow::Error {
+ loop {
+ let mut length = [0u8; 4];
+ if let Err(error) = reader.read_exact(&mut length).await {
+ return error.into();
+ }
+ let length = u32::from_be_bytes(length) as usize;
+ if length > MAX_MESSAGE {
+ return anyhow!("a message of {length} bytes; at most {MAX_MESSAGE} are read");
+ }
+ let mut body = vec![0u8; length];
+ match tokio::time::timeout(stream_timeout(length), reader.read_exact(&mut body)).await {
+ Ok(Ok(_)) => {}
+ Ok(Err(error)) => return error.into(),
+ Err(_) => {
+ return anyhow!(
+ "{length} bytes took longer than {:?}",
+ stream_timeout(length)
+ );
+ }
+ }
+ let message = match wire::decode(&body) {
+ Ok(message) => message,
+ Err(error) => {
+ debug!(peer = %location, error = %error, "Dropping a message that doesn't decode");
+ continue;
+ }
+ };
+ let channel = if is_probe(&message) {
+ &self.probes
+ } else {
+ &self.inbound
+ };
+ if channel
+ .send((location.to_owned(), message, body.len()))
+ .await
+ .is_err()
+ {
+ return anyhow!("the transport stopped");
+ }
+ }
+ }
+
+ /// The connection `id` to `location` is gone: not to be written to any more.
+ fn forget(&self, location: &str, id: u64) {
+ let mut connections = lock(&self.connections);
+ if connections
+ .get(location)
+ .is_some_and(|connection| connection.id == id)
+ {
+ connections.remove(location);
+ }
+ }
+}
+
+#[async_trait]
+impl Transport for TorTransport {
+ fn local_id(&self) -> &str {
+ &self.shared.local_id
+ }
+
+ fn is_peer_id(&self, id: &str) -> bool {
+ parse_location(id).is_some_and(|location| location == id)
+ }
+
+ /// The longest a message may take.
+ fn delivery_limit(&self) -> Duration {
+ stream_timeout(MAX_MESSAGE)
+ }
+
+ fn local_addresses(&self) -> Vec<String> {
+ vec![self.shared.local_id.clone()]
+ }
+
+ /// An onion service (`<onion>.onion:<port>`, with a node key after it as nodes
+ /// published it before, which is left out).
+ fn locate(&self, target: &str) -> Option<String> {
+ parse_location(target)
+ }
+
+ /// An onion service is its own transport identity: nothing to find out.
+ async fn resolve_seed(&self, seed: &str) -> Result<String> {
+ let location = parse_location(seed)
+ .ok_or_else(|| anyhow!("{seed} is no onion address with a port"))?;
+ if location == self.shared.local_id {
+ bail!("{seed} is this node");
+ }
+ Ok(location)
+ }
+
+ fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
+ let body = wire::encode(message)?;
+ Ok(EncodedMessage {
+ peer: peer.to_owned(),
+ address: peer.to_owned(),
+ payload_len: body.len(),
+ body: body.into(),
+ kind: message.kind(),
+ message: None,
+ })
+ }
+
+ async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> {
+ let location = &message.peer;
+ if !self.is_peer_id(location) {
+ bail!("{location} is no Tor transport identity");
+ }
+ if message.body.len() > MAX_MESSAGE {
+ bail!(
+ "a message of {} bytes; at most {MAX_MESSAGE} go over Tor",
+ message.body.len()
+ );
+ }
+ let connection = self.shared.connection(location).await?;
+ let timeout = stream_timeout(message.body.len());
+ let written = tokio::time::timeout(timeout, async {
+ let mut writer = connection.writer.lock().await;
+ writer
+ .write_all(&(message.body.len() as u32).to_be_bytes())
+ .await?;
+ writer.write_all(&message.body).await?;
+ writer.flush().await
+ })
+ .await;
+ let result = match written {
+ Ok(Ok(())) => Ok(()),
+ Ok(Err(error)) => Err(anyhow::Error::from(error)),
+ Err(_) => Err(anyhow!(
+ "sending to {location} took longer than {timeout:?}"
+ )),
+ };
+ if result.is_err() {
+ // Dialed again on the next send.
+ self.shared.forget(location, connection.id);
+ }
+ result.with_context(|| format!("Sending to {location}"))
+ }
+
+ /// Handles the messages read from every connection, one at a time, in the order
+ /// they were read.
+ async fn start_consumer(&self, handler: MessageHandler) -> Result<()> {
+ let Some((mut incoming, mut probes)) = self.incoming.lock().await.take() else {
+ bail!("Tor consumer already started");
+ };
+ let handler: Arc<HandlerFn> = Arc::from(handler);
+ {
+ let handler = Arc::clone(&handler);
+ tokio::spawn(async move {
+ while let Some((peer, message, payload_len)) = probes.recv().await {
+ hand_on_probe(&handler, peer, message, payload_len);
+ }
+ });
+ }
+ while let Some((peer, message, payload_len)) = incoming.recv().await {
+ if let Err(error) = handler(peer.clone(), message, payload_len).await {
+ warn!(peer = %peer, error = %format!("{error:#}"), "Failed to process inbound message");
+ }
+ }
+ Ok(())
+ }
+
+ async fn wait_for_exit(&self) -> std::io::Result<std::process::ExitStatus> {
+ let mut stopped = self.stopped.clone();
+ let _ = stopped.wait_for(|stopped| *stopped).await;
+ Err(std::io::Error::other("the onion service stopped"))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const ONION: &str = "vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion";
+
+ #[test]
+ fn locations() {
+ let key = "ab".repeat(32);
+ assert!(is_key(&key));
+ assert!(!is_key(&key.to_uppercase()));
+ assert!(!is_key(&key[1..]));
+ let location = format!("{ONION}:9662");
+ assert_eq!(parse_location(&location), Some(location.clone()));
+ // As nodes published them before: the key is left out.
+ assert_eq!(
+ parse_location(&format!("{ONION}:9662/{key}")),
+ Some(location.clone())
+ );
+ assert_eq!(
+ parse_location(&location.to_uppercase().replace(".ONION", ".onion")),
+ Some(location)
+ );
+ // A port is needed, a key must be one, and only onion addresses.
+ assert_eq!(parse_location(ONION), None);
+ assert_eq!(parse_location(&format!("{ONION}:9662/xyz")), None);
+ assert_eq!(parse_location("example.org:9662"), None);
+ assert_eq!(parse_location(&format!("{ONION}:99999")), None);
+ }
+
+ #[tokio::test]
+ async fn claims_round_trip() {
+ let (mut dialer, mut listener) = tokio::io::duplex(1024);
+ let location = format!("{ONION}:9662");
+ send_claim(&mut dialer, &location).await.unwrap();
+ send_claim(&mut dialer, "").await.unwrap();
+ send_claim(&mut dialer, "example.org:1").await.unwrap();
+ assert_eq!(read_claim(&mut listener).await.unwrap(), Some(location));
+ assert_eq!(read_claim(&mut listener).await.unwrap(), None);
+ assert!(read_claim(&mut listener).await.is_err());
+ }
+}
diff --git a/src/transport/traffic.rs b/src/transport/traffic.rs
index 9dcd91a..64595cb 100644
--- a/src/transport/traffic.rs
+++ b/src/transport/traffic.rs
@@ -1,12 +1,12 @@
//! Messages and bytes exchanged with each peer, by kind: counted where every message
//! passes anyway, at the transport. No messages of our own.
//!
-//! Bytes are payload bytes, what the Mycelium message carries, without its framing,
-//! the topic, or the base64 of its HTTP API. Counters only grow (from the start of the
-//! process); rates are for whoever reads them.
+//! Bytes are payload bytes, what a message carries, without the transport's framing
+//! (for Mycelium: the topic, or the base64 of its HTTP API). Counters only grow (from
+//! the start of the process); rates are for whoever reads them.
//!
-//! A message dropped by the debug firewall still counts as sent (the firewall plays a
-//! lossy network) but not as received.
+//! A message dropped by the debug firewall (in the paths layer, below) still counts as
+//! sent (the firewall plays a lossy network) but not as received.
use std::collections::HashMap;
use std::sync::{Arc, Mutex, MutexGuard};
@@ -14,14 +14,14 @@ use std::sync::{Arc, Mutex, MutexGuard};
use anyhow::Result;
use async_trait::async_trait;
use serde::Serialize;
-use utoipa::ToSchema;
+use crate::locks::MutexExt;
use crate::transport::{EncodedMessage, MessageHandler, MessageKind, ParsedMessage, Transport};
-#[derive(Debug, Default, Clone, Copy, Serialize, ToSchema)]
-pub struct Counter {
- pub messages: u64,
- pub bytes: u64,
+#[derive(Debug, Default, Clone, Copy, Serialize)]
+pub(crate) struct Counter {
+ pub(crate) messages: u64,
+ pub(crate) bytes: u64,
}
impl Counter {
@@ -31,15 +31,20 @@ impl Counter {
}
}
-#[derive(Debug, Default, Clone, Copy, Serialize, ToSchema)]
+#[derive(Debug, Default, Clone, Copy, Serialize)]
#[serde(rename_all = "camelCase")]
-pub struct ByKind {
- pub hello: Counter,
- pub hello_reply: Counter,
- pub sync: Counter,
- pub ping: Counter,
- pub pong: Counter,
- pub blob: Counter,
+pub(crate) struct ByKind {
+ pub(crate) hello: Counter,
+ pub(crate) hello_reply: Counter,
+ pub(crate) sync: Counter,
+ pub(crate) status: Counter,
+ pub(crate) status_reply: Counter,
+ pub(crate) blob: Counter,
+ /// WHO and ME (`transport::identity`), counted by the identity layer under the
+ /// address they went to or came from.
+ pub(crate) identity: Counter,
+ /// PING and PONG (`transport::paths`), counted by the paths layer under the address.
+ pub(crate) paths: Counter,
}
impl ByKind {
@@ -48,21 +53,23 @@ impl ByKind {
MessageKind::Hello => &mut self.hello,
MessageKind::HelloReply => &mut self.hello_reply,
MessageKind::Sync => &mut self.sync,
- MessageKind::Ping => &mut self.ping,
- MessageKind::Pong => &mut self.pong,
+ MessageKind::Status => &mut self.status,
+ MessageKind::StatusReply => &mut self.status_reply,
MessageKind::Blob => &mut self.blob,
+ MessageKind::Who | MessageKind::Me => &mut self.identity,
+ MessageKind::Ping | MessageKind::Pong => &mut self.paths,
}
}
}
-#[derive(Debug, Default, Clone, Copy, Serialize, ToSchema)]
+#[derive(Debug, Default, Clone, Copy, Serialize)]
#[serde(rename_all = "camelCase")]
-pub struct PeerTraffic {
- /// Handed to Mycelium (it accepted them).
- pub sent: ByKind,
- pub received: ByKind,
- /// Sends Mycelium refused (e.g. no route to the peer); not counted as sent.
- pub refused: ByKind,
+pub(crate) struct PeerTraffic {
+ /// Handed to the transport (it accepted them).
+ pub(crate) sent: ByKind,
+ pub(crate) received: ByKind,
+ /// Sends the transport refused (e.g. no route to the peer); not counted as sent.
+ pub(crate) refused: ByKind,
}
#[derive(Debug, Default)]
@@ -73,15 +80,13 @@ struct Table {
}
#[derive(Debug, Default)]
-pub struct Traffic {
+pub(crate) struct Traffic {
table: Mutex<Table>,
}
impl Traffic {
fn lock(&self) -> MutexGuard<'_, Table> {
- self.table
- .lock()
- .unwrap_or_else(|poisoned| poisoned.into_inner())
+ self.table.lock_or_recover()
}
fn count(&self, peer: &str, bytes: usize, pick: impl Fn(&mut PeerTraffic) -> &mut Counter) {
@@ -94,41 +99,41 @@ impl Traffic {
}
}
- fn sent(&self, peer: &str, kind: MessageKind, bytes: usize) {
+ pub(crate) fn sent(&self, peer: &str, kind: MessageKind, bytes: usize) {
self.count(peer, bytes, |t| t.sent.get_mut(kind));
}
- fn refused(&self, peer: &str, kind: MessageKind, bytes: usize) {
+ pub(crate) fn refused(&self, peer: &str, kind: MessageKind, bytes: usize) {
self.count(peer, bytes, |t| t.refused.get_mut(kind));
}
- fn received(&self, peer: &str, kind: MessageKind, bytes: usize) {
+ pub(crate) fn received(&self, peer: &str, kind: MessageKind, bytes: usize) {
self.count(peer, bytes, |t| t.received.get_mut(kind));
}
- pub fn peer(&self, peer: &str) -> Option<PeerTraffic> {
+ pub(crate) fn peer(&self, peer: &str) -> Option<PeerTraffic> {
self.lock().peers.get(peer).copied()
}
- pub fn total(&self) -> PeerTraffic {
+ pub(crate) fn total(&self) -> PeerTraffic {
self.lock().total
}
/// Drops the counters of peers not kept (anyone can send us messages; the totals
/// keep their share).
- pub fn retain(&self, mut keep: impl FnMut(&str) -> bool) {
+ pub(crate) fn retain(&self, mut keep: impl FnMut(&str) -> bool) {
self.lock().peers.retain(|peer, _| keep(peer));
}
}
/// Counts what passes through `inner`.
-pub struct CountingTransport {
+pub(crate) struct CountingTransport {
inner: Arc<dyn Transport>,
traffic: Arc<Traffic>,
}
impl CountingTransport {
- pub fn new(inner: Arc<dyn Transport>, traffic: Arc<Traffic>) -> Self {
+ pub(crate) fn new(inner: Arc<dyn Transport>, traffic: Arc<Traffic>) -> Self {
Self { inner, traffic }
}
}
@@ -139,6 +144,58 @@ impl Transport for CountingTransport {
self.inner.local_id()
}
+ fn is_peer_id(&self, id: &str) -> bool {
+ self.inner.is_peer_id(id)
+ }
+
+ fn delivery_limit(&self) -> std::time::Duration {
+ self.inner.delivery_limit()
+ }
+
+ fn local_addresses(&self) -> Vec<String> {
+ self.inner.local_addresses()
+ }
+
+ fn addresses_of(&self, peer: &str) -> Vec<String> {
+ self.inner.addresses_of(peer)
+ }
+
+ fn learn_addresses(&self, peer: &str, addresses: &[String]) {
+ self.inner.learn_addresses(peer, addresses)
+ }
+
+ fn can_reach(&self, peer: &str) -> bool {
+ self.inner.can_reach(peer)
+ }
+
+ fn discovered_seeds(&self) -> Vec<String> {
+ self.inner.discovered_seeds()
+ }
+
+ fn transports(&self) -> Vec<String> {
+ self.inner.transports()
+ }
+
+ fn destination(&self, peer: &str) -> Option<crate::transport::Destination> {
+ self.inner.destination(peer)
+ }
+
+ fn connection(&self, target: &str) -> Option<crate::transport::Connection> {
+ self.inner.connection(target)
+ }
+
+ fn stats(&self) -> crate::transport::stats::TransportStats {
+ self.inner.stats()
+ }
+
+ fn answers_pings(&self) -> bool {
+ self.inner.answers_pings()
+ }
+
+ fn rtt(&self, peer: &str) -> Option<std::time::Duration> {
+ self.inner.rtt(peer)
+ }
+
async fn resolve_seed(&self, seed: &str) -> Result<String> {
self.inner.resolve_seed(seed).await
}
@@ -181,20 +238,20 @@ mod tests {
#[test]
fn totals_survive_retain() {
let traffic = Traffic::default();
- traffic.sent("a", MessageKind::Ping, 100);
+ traffic.sent("a", MessageKind::Status, 100);
traffic.sent("b", MessageKind::Sync, 1000);
- traffic.received("b", MessageKind::Pong, 90);
+ traffic.received("b", MessageKind::StatusReply, 90);
traffic.refused("a", MessageKind::Sync, 500);
let b = traffic.peer("b").unwrap();
assert_eq!(b.sent.sync.messages, 1);
- assert_eq!(b.received.pong.bytes, 90);
+ assert_eq!(b.received.status_reply.bytes, 90);
traffic.retain(|peer| peer == "b");
assert!(traffic.peer("a").is_none());
let total = traffic.total();
- assert_eq!(total.sent.ping.bytes, 100);
+ assert_eq!(total.sent.status.bytes, 100);
assert_eq!(total.sent.sync.bytes, 1000);
assert_eq!(total.refused.sync.bytes, 500);
}
diff --git a/src/transport/wire.rs b/src/transport/wire.rs
new file mode 100644
index 0000000..fcbee85
--- /dev/null
+++ b/src/transport/wire.rs
@@ -0,0 +1,550 @@
+//! The wire format of a message: one byte for the format's version (`VERSION`), one
+//! saying what the message is (`MessageKind`), then the message itself in MessagePack, as derived from its type in `transport` (structs as
+//! arrays, fields in declaration order). The codecs below make hashes, IDs and bytes
+//! compact (MessagePack binaries, not arrays of numbers) and check their lengths.
+//!
+//! The wire format may change freely until the first release: there is no compatibility
+//! with older nodes to keep. But every change bumps `VERSION`, so nodes on different
+//! builds tell each other apart instead of misreading each other's messages. The
+//! transport's topic says which application a message is for, not which version.
+
+use thiserror::Error;
+
+use super::{MessageKind, ParsedMessage};
+
+/// The wire format's version, the first byte of every payload. Bump it with every change
+/// to a message type or a codec (`golden_bytes` fails then).
+pub(crate) const VERSION: u8 = 6;
+
+/// What the second byte of a payload says.
+fn tag(kind: MessageKind) -> u8 {
+ match kind {
+ MessageKind::Hello => 0,
+ MessageKind::HelloReply => 1,
+ MessageKind::Sync => 2,
+ MessageKind::Status => 3,
+ MessageKind::StatusReply => 4,
+ MessageKind::Blob => 5,
+ MessageKind::Who => 6,
+ MessageKind::Me => 7,
+ MessageKind::Ping => 8,
+ MessageKind::Pong => 9,
+ }
+}
+
+#[derive(Error, Debug)]
+pub(crate) enum WireError {
+ #[error("empty payload")]
+ Empty,
+
+ #[error("wire format version {0} (we speak version {VERSION})")]
+ Version(u8),
+
+ #[error("unknown message kind {0}")]
+ UnknownKind(u8),
+
+ #[error("MessagePack serialization failed: {0}")]
+ Encode(#[from] rmp_serde::encode::Error),
+
+ #[error("MessagePack deserialization failed: {0}")]
+ Decode(#[from] rmp_serde::decode::Error),
+}
+
+/// The payload of `message`.
+pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> {
+ let mut payload = vec![VERSION, tag(message.kind())];
+ match message {
+ ParsedMessage::Hello(hello) => rmp_serde::encode::write(&mut payload, hello)?,
+ ParsedMessage::HelloReply(reply) => rmp_serde::encode::write(&mut payload, reply)?,
+ ParsedMessage::Sync(envelope) => rmp_serde::encode::write(&mut payload, envelope)?,
+ ParsedMessage::Status(status) | ParsedMessage::StatusReply(status) => {
+ rmp_serde::encode::write(&mut payload, status)?
+ }
+ ParsedMessage::Blob(blob) => rmp_serde::encode::write(&mut payload, blob)?,
+ ParsedMessage::Who(proof) | ParsedMessage::Me(proof) => {
+ rmp_serde::encode::write(&mut payload, proof)?
+ }
+ ParsedMessage::Ping(nonce) | ParsedMessage::Pong(nonce) => {
+ rmp_serde::encode::write(&mut payload, nonce)?
+ }
+ }
+ Ok(payload)
+}
+
+/// The message in `payload` (as made by [`encode`]).
+pub(crate) fn decode(payload: &[u8]) -> Result<ParsedMessage, WireError> {
+ let (&version, payload) = payload.split_first().ok_or(WireError::Empty)?;
+ if version != VERSION {
+ return Err(WireError::Version(version));
+ }
+ let (&kind, body) = payload.split_first().ok_or(WireError::Empty)?;
+ let is = |wanted: MessageKind| kind == tag(wanted);
+ Ok(if is(MessageKind::Hello) {
+ ParsedMessage::Hello(rmp_serde::from_slice(body)?)
+ } else if is(MessageKind::HelloReply) {
+ ParsedMessage::HelloReply(rmp_serde::from_slice(body)?)
+ } else if is(MessageKind::Sync) {
+ ParsedMessage::Sync(rmp_serde::from_slice(body)?)
+ } else if is(MessageKind::Status) {
+ ParsedMessage::Status(rmp_serde::from_slice(body)?)
+ } else if is(MessageKind::StatusReply) {
+ ParsedMessage::StatusReply(rmp_serde::from_slice(body)?)
+ } else if is(MessageKind::Blob) {
+ ParsedMessage::Blob(rmp_serde::from_slice(body)?)
+ } else if is(MessageKind::Who) {
+ ParsedMessage::Who(rmp_serde::from_slice(body)?)
+ } else if is(MessageKind::Me) {
+ ParsedMessage::Me(rmp_serde::from_slice(body)?)
+ } else if is(MessageKind::Ping) {
+ ParsedMessage::Ping(rmp_serde::from_slice(body)?)
+ } else if is(MessageKind::Pong) {
+ ParsedMessage::Pong(rmp_serde::from_slice(body)?)
+ } else {
+ return Err(WireError::UnknownKind(kind));
+ })
+}
+
+/// Field codecs (`#[serde(with = "...")]`) for the message types.
+pub(crate) mod codec {
+ use serde::de::Error as _;
+ use serde::{Deserialize, Deserializer, Serialize, Serializer};
+ use serde_bytes::{ByteBuf, Bytes};
+
+ /// Exactly `N` bytes, or a decoding error naming `what`.
+ fn fixed<const N: usize, E: serde::de::Error>(bytes: &[u8], what: &str) -> Result<[u8; N], E> {
+ bytes
+ .try_into()
+ .map_err(|_| E::custom(format!("{what} must be {N} bytes")))
+ }
+
+ /// A run ID (`RunId`).
+ pub(crate) mod run_id {
+ use super::*;
+
+ pub(crate) fn serialize<S: Serializer>(value: &[u8; 16], s: S) -> Result<S::Ok, S::Error> {
+ Bytes::new(value).serialize(s)
+ }
+
+ pub(crate) fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<[u8; 16], D::Error> {
+ fixed(&ByteBuf::deserialize(d)?, "a run id")
+ }
+ }
+
+ /// A 32-byte hash.
+ pub(crate) mod bytes32 {
+ use super::*;
+
+ pub(crate) fn serialize<S: Serializer>(value: &[u8; 32], s: S) -> Result<S::Ok, S::Error> {
+ Bytes::new(value).serialize(s)
+ }
+
+ pub(crate) fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<[u8; 32], D::Error> {
+ fixed(&ByteBuf::deserialize(d)?, "a hash")
+ }
+ }
+
+ /// A 64-byte signature.
+ pub(crate) mod bytes64 {
+ use super::*;
+
+ pub(crate) fn serialize<S: Serializer>(value: &[u8; 64], s: S) -> Result<S::Ok, S::Error> {
+ Bytes::new(value).serialize(s)
+ }
+
+ pub(crate) fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<[u8; 64], D::Error> {
+ fixed(&ByteBuf::deserialize(d)?, "a signature")
+ }
+ }
+
+ /// 32-byte hashes.
+ pub(crate) mod hashes32 {
+ use super::*;
+
+ pub(crate) fn serialize<S: Serializer>(
+ value: &[[u8; 32]],
+ s: S,
+ ) -> Result<S::Ok, S::Error> {
+ s.collect_seq(value.iter().map(|hash| Bytes::new(hash)))
+ }
+
+ pub(crate) fn deserialize<'de, D: Deserializer<'de>>(
+ d: D,
+ ) -> Result<Vec<[u8; 32]>, D::Error> {
+ Vec::<ByteBuf>::deserialize(d)?
+ .iter()
+ .map(|hash| fixed(hash, "a hash"))
+ .collect()
+ }
+ }
+
+ /// An Automerge change hash.
+ pub(crate) mod change_hash {
+ use super::*;
+ use automerge::ChangeHash;
+
+ pub(crate) fn serialize<S: Serializer>(
+ value: &ChangeHash,
+ s: S,
+ ) -> Result<S::Ok, S::Error> {
+ Bytes::new(&value.0).serialize(s)
+ }
+
+ pub(crate) fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<ChangeHash, D::Error> {
+ Ok(ChangeHash(fixed(
+ &ByteBuf::deserialize(d)?,
+ "a change hash",
+ )?))
+ }
+ }
+
+ /// An epoch: generation and ID.
+ pub(crate) mod epoch {
+ use super::*;
+ use crate::document::replica::Epoch;
+
+ pub(crate) fn serialize<S: Serializer>(value: &Epoch, s: S) -> Result<S::Ok, S::Error> {
+ (value.generation, Bytes::new(&value.id)).serialize(s)
+ }
+
+ pub(crate) fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Epoch, D::Error> {
+ let (generation, id): (u64, ByteBuf) = Deserialize::deserialize(d)?;
+ Ok(Epoch {
+ generation,
+ id: fixed(&id, "an epoch id")?,
+ })
+ }
+ }
+
+ /// Heads as a keepalive carries them: listed (at most `MAX_LISTED_HEADS`, digest
+ /// empty), or as a digest (none listed).
+ pub(crate) mod heads {
+ use super::*;
+ use crate::transport::{HeadsSummary, MAX_LISTED_HEADS};
+ use automerge::ChangeHash;
+
+ pub(crate) fn serialize<S: Serializer>(
+ value: &HeadsSummary,
+ s: S,
+ ) -> Result<S::Ok, S::Error> {
+ let (listed, digest): (Vec<&Bytes>, &Bytes) = match value {
+ HeadsSummary::Heads(heads) => (
+ heads.iter().map(|head| Bytes::new(&head.0)).collect(),
+ Bytes::new(&[]),
+ ),
+ HeadsSummary::Digest { digest, sample } => (
+ sample.iter().map(|head| Bytes::new(&head.0)).collect(),
+ Bytes::new(digest),
+ ),
+ };
+ (listed, digest).serialize(s)
+ }
+
+ pub(crate) fn deserialize<'de, D: Deserializer<'de>>(
+ d: D,
+ ) -> Result<HeadsSummary, D::Error> {
+ let (listed, digest): (Vec<ByteBuf>, ByteBuf) = Deserialize::deserialize(d)?;
+ if listed.len() > MAX_LISTED_HEADS {
+ return Err(D::Error::custom(format!(
+ "more than {MAX_LISTED_HEADS} heads listed"
+ )));
+ }
+ let listed = listed
+ .iter()
+ .map(|head| -> Result<ChangeHash, D::Error> {
+ Ok(ChangeHash(fixed(head, "a head")?))
+ })
+ .collect::<Result<Vec<_>, _>>()?;
+ if digest.is_empty() {
+ return Ok(HeadsSummary::Heads(listed));
+ }
+ Ok(HeadsSummary::Digest {
+ digest: fixed(&digest, "a heads digest")?,
+ sample: listed,
+ })
+ }
+ }
+
+ /// Change hashes, each a 32-byte binary.
+ pub(crate) mod change_hashes {
+ use super::*;
+ use automerge::ChangeHash;
+
+ pub(crate) fn serialize<S: Serializer>(
+ value: &[ChangeHash],
+ s: S,
+ ) -> Result<S::Ok, S::Error> {
+ s.collect_seq(value.iter().map(|hash| Bytes::new(&hash.0)))
+ }
+
+ pub(crate) fn deserialize<'de, D: Deserializer<'de>>(
+ d: D,
+ ) -> Result<Vec<ChangeHash>, D::Error> {
+ Vec::<ByteBuf>::deserialize(d)?
+ .iter()
+ .map(|hash| Ok(ChangeHash(fixed(hash, "a change hash")?)))
+ .collect()
+ }
+ }
+
+ /// An Automerge sync message, in Automerge's own encoding (left encoded:
+ /// `SyncPayload`).
+ pub(crate) mod sync_message {
+ use super::*;
+ use crate::transport::SyncPayload;
+
+ pub(crate) fn serialize<S: Serializer>(
+ value: &SyncPayload,
+ s: S,
+ ) -> Result<S::Ok, S::Error> {
+ Bytes::new(&value.0).serialize(s)
+ }
+
+ pub(crate) fn deserialize<'de, D: Deserializer<'de>>(
+ d: D,
+ ) -> Result<SyncPayload, D::Error> {
+ Ok(SyncPayload(ByteBuf::deserialize(d)?.into_vec().into()))
+ }
+ }
+
+ /// Shared bytes (`bytes::Bytes`).
+ pub(crate) mod shared_bytes {
+ use super::*;
+
+ pub(crate) fn serialize<S: Serializer>(
+ value: &bytes::Bytes,
+ s: S,
+ ) -> Result<S::Ok, S::Error> {
+ Bytes::new(value).serialize(s)
+ }
+
+ pub(crate) fn deserialize<'de, D: Deserializer<'de>>(
+ d: D,
+ ) -> Result<bytes::Bytes, D::Error> {
+ Ok(ByteBuf::deserialize(d)?.into_vec().into())
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::document::replica::Epoch;
+ use crate::transport::{
+ BlobBody, BlobId, BlobMessage, HeadsSummary, HelloReply, HelloRequest, IdentityProof,
+ ListedPeer, MAX_LISTED_HEADS, Status,
+ };
+ use automerge::ChangeHash;
+
+ #[test]
+ fn handshakes_round_trip() {
+ let epoch = Epoch {
+ generation: 2,
+ id: [3; 32],
+ };
+ let hello = ParsedMessage::Hello(HelloRequest {
+ swarm_id: Some("swarm".into()),
+ epoch,
+ epoch_seq: 1,
+ auth: vec![1, 2, 3],
+ ready: true,
+ run_id: [4; 16],
+ handshake: 7,
+ });
+ let ParsedMessage::Hello(decoded) = decode(&encode(&hello).unwrap()).unwrap() else {
+ panic!("not a HELLO");
+ };
+ assert_eq!(decoded.swarm_id.as_deref(), Some("swarm"));
+ assert_eq!((decoded.epoch, decoded.epoch_seq), (epoch, 1));
+ assert_eq!(decoded.auth, vec![1, 2, 3]);
+ assert_eq!(decoded.run_id, [4; 16]);
+ assert_eq!(decoded.handshake, 7);
+
+ let reply = ParsedMessage::HelloReply(HelloReply {
+ swarm_id: "swarm".into(),
+ peers: vec![ListedPeer {
+ id: "p".into(),
+ addresses: vec!["a".into()],
+ }],
+ epoch,
+ epoch_seq: 1,
+ auth: Vec::new(),
+ ready: false,
+ run_id: [4; 16],
+ away_ms: Some(5),
+ sender_run_id: [6; 16],
+ });
+ let ParsedMessage::HelloReply(decoded) = decode(&encode(&reply).unwrap()).unwrap() else {
+ panic!("not a HELLO_REPLY");
+ };
+ assert_eq!(
+ decoded.peers,
+ vec![ListedPeer {
+ id: "p".into(),
+ addresses: vec!["a".into()],
+ }]
+ );
+ assert_eq!(decoded.away_ms, Some(5));
+ assert_eq!(decoded.sender_run_id, [6; 16]);
+ }
+
+ #[test]
+ fn keepalives_keep_their_kind_and_heads() {
+ let status = Status {
+ heads: HeadsSummary::Digest {
+ digest: [8; 32],
+ sample: vec![ChangeHash([9; 32])],
+ },
+ epoch: Epoch::GENESIS,
+ epoch_seq: 0,
+ ready: true,
+ syncs_with_you: false,
+ };
+ let decoded =
+ decode(&encode(&ParsedMessage::StatusReply(status.clone())).unwrap()).unwrap();
+ let ParsedMessage::StatusReply(decoded) = decoded else {
+ panic!("not a STATUS_REPLY");
+ };
+ assert_eq!(decoded.heads, status.heads);
+ assert_eq!(decoded.heads.listed(), [ChangeHash([9; 32])]);
+ }
+
+ #[test]
+ fn too_many_listed_heads_are_refused() {
+ let status = Status {
+ heads: HeadsSummary::Heads(
+ (0..=MAX_LISTED_HEADS as u8)
+ .map(|n| ChangeHash([n; 32]))
+ .collect(),
+ ),
+ epoch: Epoch::GENESIS,
+ epoch_seq: 0,
+ ready: true,
+ syncs_with_you: true,
+ };
+ assert!(decode(&encode(&ParsedMessage::Status(status)).unwrap()).is_err());
+ }
+
+ #[test]
+ fn unknown_kinds_and_bad_payloads_are_refused() {
+ assert!(matches!(decode(&[]), Err(WireError::Empty)));
+ assert!(matches!(decode(&[VERSION]), Err(WireError::Empty)));
+ assert!(matches!(
+ decode(&[VERSION, 99]),
+ Err(WireError::UnknownKind(99))
+ ));
+ assert!(matches!(
+ decode(&[VERSION + 1, 0]),
+ Err(WireError::Version(version)) if version == VERSION + 1
+ ));
+
+ let blob = ParsedMessage::Blob(BlobMessage {
+ epoch: Epoch::GENESIS,
+ epoch_seq: 0,
+ id: BlobId::Content([1; 32]),
+ body: BlobBody::Missing,
+ });
+ let payload = encode(&blob).unwrap();
+ assert!(matches!(decode(&payload).unwrap(), ParsedMessage::Blob(_)));
+ // A payload cut short doesn't decode.
+ assert!(decode(&payload[..payload.len() - 1]).is_err());
+ // Nor one of another kind.
+ let mut relabelled = payload.clone();
+ relabelled[1] = tag(MessageKind::Hello);
+ assert!(decode(&relabelled).is_err());
+ }
+
+ fn hex(bytes: &[u8]) -> String {
+ bytes.iter().map(|byte| format!("{byte:02x}")).collect()
+ }
+
+ /// The exact bytes of a few messages. If this fails after a change to a message type
+ /// or a codec, the change is a new wire format: bump `VERSION` and take the new bytes.
+ #[test]
+ fn golden_bytes() {
+ let epoch = Epoch {
+ generation: 2,
+ id: [3; 32],
+ };
+ let epoch_hex = format!("9202c420{}", "03".repeat(32));
+ let run = |byte: &str| format!("c410{}", byte.repeat(16));
+
+ let hello = ParsedMessage::Hello(HelloRequest {
+ swarm_id: Some("swarm".into()),
+ epoch,
+ epoch_seq: 1,
+ auth: vec![1, 2, 3],
+ ready: true,
+ run_id: [4; 16],
+ handshake: 7,
+ });
+ assert_eq!(
+ hex(&encode(&hello).unwrap()),
+ format!("060097a5737761726d{epoch_hex}01c403010203c3{}07", run("04"))
+ );
+
+ let reply = ParsedMessage::HelloReply(HelloReply {
+ swarm_id: "swarm".into(),
+ peers: vec![ListedPeer {
+ id: "p".into(),
+ addresses: vec!["a".into()],
+ }],
+ epoch,
+ epoch_seq: 1,
+ auth: Vec::new(),
+ ready: false,
+ run_id: [4; 16],
+ away_ms: Some(5),
+ sender_run_id: [6; 16],
+ });
+ assert_eq!(
+ hex(&encode(&reply).unwrap()),
+ format!(
+ "060199a5737761726d9192a17091a161{epoch_hex}01c400c2{}05{}",
+ run("04"),
+ run("06")
+ )
+ );
+
+ let status = ParsedMessage::Status(Status {
+ heads: HeadsSummary::Heads(Vec::new()),
+ epoch,
+ epoch_seq: 1,
+ ready: true,
+ syncs_with_you: false,
+ });
+ assert_eq!(
+ hex(&encode(&status).unwrap()),
+ format!("0603959290c400{epoch_hex}01c3c2")
+ );
+
+ let who = ParsedMessage::Who(IdentityProof {
+ key: [8; 32],
+ signature: [9; 64],
+ });
+ assert_eq!(
+ hex(&encode(&who).unwrap()),
+ format!("060692c420{}c440{}", "08".repeat(32), "09".repeat(64))
+ );
+ let ParsedMessage::Me(proof) = decode(
+ &encode(&ParsedMessage::Me(IdentityProof {
+ key: [8; 32],
+ signature: [9; 64],
+ }))
+ .unwrap(),
+ )
+ .unwrap() else {
+ panic!("not a ME");
+ };
+ assert_eq!((proof.key, proof.signature), ([8; 32], [9; 64]));
+
+ assert_eq!(hex(&encode(&ParsedMessage::Ping(7)).unwrap()), "060807");
+ assert_eq!(
+ hex(&encode(&ParsedMessage::Pong(u64::MAX)).unwrap()),
+ "0609cfffffffffffffffff"
+ );
+ let ParsedMessage::Pong(nonce) = decode(&[VERSION, 9, 0x2a]).unwrap() else {
+ panic!("not a PONG");
+ };
+ assert_eq!(nonce, 42);
+ }
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-28 06:31:51 +0000