1 files changed, 87 insertions, 65 deletions
diff --git a/src/swarm/bootstrap.rs b/src/swarm/bootstrap.rs
index 27ffdd8..767a85c 100644
--- a/src/swarm/bootstrap.rs
+++ b/src/swarm/bootstrap.rs
@@ -1,55 +1,48 @@
 //! Joining a swarm (or creating one as the genesis node).
+//!
+//! A node starts a new swarm only with `--genesis`. Otherwise it joins through its
+//! seeds and keeps trying until it is ready: seeds that don't resolve are resolved
+//! again with backoff, and HELLOs to resolved ones are retried by the peer worker. A
+//! node without seeds waits (nothing to join yet) and says so now and then.
 
 use std::sync::atomic::Ordering;
 use std::time::Duration;
 
-use anyhow::{Result, bail};
-use tracing::{debug, error, info, warn};
+use anyhow::Result;
+use tokio::time::Instant;
+use tracing::{debug, info, warn};
 use uuid::Uuid;
 
 use crate::state::AppState;
 use crate::swarm::peer::{remember_peer, send_hello_once};
 
-const BOOTSTRAP_TIMEOUT: Duration = Duration::from_secs(300);
-const BOOTSTRAP_RETRY: Duration = Duration::from_millis(500);
+const BOOTSTRAP_POLL: Duration = Duration::from_millis(500);
+/// How often a node that is still joining says why.
+const BOOTSTRAP_REPORT: Duration = Duration::from_secs(300);
+/// Retrying seeds that didn't resolve: first after this, doubling up to the maximum.
+const RESOLVE_RETRY: Duration = Duration::from_secs(5);
+const RESOLVE_RETRY_MAX: Duration = Duration::from_secs(300);
 
-pub(crate) async fn bootstrap(state: &AppState, seeds: &[String]) -> Result<()> {
-    if seeds.is_empty() {
-        info!("Initializing genesis node (no seeds configured)");
+pub(crate) async fn bootstrap(state: &AppState, seeds: &[String], genesis: bool) -> Result<()> {
+    if genesis {
+        info!("Starting a new swarm as its genesis node (--genesis)");
         state.initial_sync_done.store(true, Ordering::Release);
         state.mark_ready(Uuid::new_v4().to_string()).await?;
         return Ok(());
     }
 
-    info!(
-        timeout_seconds = BOOTSTRAP_TIMEOUT.as_secs(),
-        "Joining existing swarm"
-    );
-
-    let mut peers = Vec::new();
-
-    for seed in seeds {
-        match state.transport.resolve_seed(seed).await {
-            Ok(peer) => {
-                debug!(seed = %seed, peer = %peer, "Resolved seed");
-                peers.push(peer)
-            }
-            Err(e) => warn!(seed = %seed, error = %e, "Ignoring seed argument"),
-        }
-    }
-
-    if peers.is_empty() {
-        bail!("None of the configured seeds could be resolved");
-    }
-
-    for peer in peers {
-        remember_peer(state, &peer).await;
-        if let Err(error) = send_hello_once(state, &peer).await {
-            warn!(peer = %peer, error = %format!("{error:#}"), "Initial HELLO failed; will retry");
-        }
+    if seeds.is_empty() {
+        warn!(
+            "No seeds: nothing to join. Add --seeds to join a swarm, or --genesis to start a new one"
+        );
+    } else {
+        info!(seeds = seeds.len(), "Joining existing swarm");
     }
 
-    let deadline = tokio::time::Instant::now() + BOOTSTRAP_TIMEOUT;
+    let mut unresolved: Vec<String> = seeds.to_vec();
+    let mut resolve_retry = RESOLVE_RETRY;
+    let mut next_resolve = Instant::now();
+    let mut next_report = Instant::now() + BOOTSTRAP_REPORT;
     loop {
         if let Some(swarm_id) = state.swarm_id().await {
             // A node joining a compacted swarm rejoins the newer epoch first; it is
@@ -61,39 +54,68 @@ pub(crate) async fn bootstrap(state: &AppState, seeds: &[String]) -> Result<()>
             }
         }
 
-        if tokio::time::Instant::now() >= deadline {
-            let mut peer_diagnostics: Vec<String> = Vec::new();
-
-            let peer_states = state.peer_states.read().await.clone();
+        let now = Instant::now();
+        if !unresolved.is_empty() && now >= next_resolve {
+            resolve_seeds(state, &mut unresolved).await;
+            if !unresolved.is_empty() {
+                next_resolve = now + resolve_retry;
+                resolve_retry = (resolve_retry * 2).min(RESOLVE_RETRY_MAX);
+            }
+        }
+        if now >= next_report {
+            next_report = now + BOOTSTRAP_REPORT;
+            report_joining(state, seeds, &unresolved).await;
+        }
+        tokio::time::sleep(BOOTSTRAP_POLL).await;
+    }
+}
 
-            for (peer, peer_state_mutex) in &peer_states {
-                let peer_state = peer_state_mutex.lock().await;
-                let validated_swarm = peer_state.standing.swarm_id();
-                peer_diagnostics.push(format!(
-                    "{peer}: validated_swarm={:?}, epoch={:?}",
-                    validated_swarm, peer_state.epoch,
-                ))
+/// Resolves the seeds in `unresolved`, says HELLO to each that resolves, and keeps the
+/// rest for later.
+async fn resolve_seeds(state: &AppState, unresolved: &mut Vec<String>) {
+    let mut still = Vec::new();
+    for seed in unresolved.drain(..) {
+        match state.transport.resolve_seed(&seed).await {
+            Ok(peer) => {
+                debug!(seed = %seed, peer = %peer, "Resolved seed");
+                remember_peer(state, &peer).await;
+                if let Err(error) = send_hello_once(state, &peer).await {
+                    warn!(peer = %peer, error = %format!("{error:#}"), "Initial HELLO failed; will retry");
+                }
+            }
+            Err(error) => {
+                warn!(seed = %seed, error = %error, "Seed doesn't resolve; will retry");
+                still.push(seed);
             }
-            let local_swarm = state.swarm_id().await;
-            let local_epoch = state.local_epoch();
-            let initial_sync_done = state.initial_sync_done.load(Ordering::Acquire);
-            error!(
-                local_swarm = ?local_swarm,
-                local_epoch = %local_epoch,
-                initial_sync_done,
-                peers = %peer_diagnostics.join("; "),
-                timeout_seconds = BOOTSTRAP_TIMEOUT.as_secs(),
-                "Bootstrap timed out"
-            );
-            bail!(
-                "Bootstrap timed out after {:?}; local_swarm={:?}; local_epoch={}; initial_sync_done={}; peers=[{}]. Refusing to create a genesis document.",
-                BOOTSTRAP_TIMEOUT,
-                local_swarm,
-                local_epoch,
-                initial_sync_done,
-                peer_diagnostics.join("; ")
-            );
         }
-        tokio::time::sleep(BOOTSTRAP_RETRY).await;
     }
+    *unresolved = still;
+}
+
+/// Why the node is still joining, for the log.
+async fn report_joining(state: &AppState, seeds: &[String], unresolved: &[String]) {
+    if seeds.is_empty() {
+        warn!(
+            "Still no seeds: nothing to join. Add --seeds to join a swarm, or --genesis to start a new one"
+        );
+        return;
+    }
+    let mut peer_diagnostics: Vec<String> = Vec::new();
+    let peer_states = state.peer_states.read().await.clone();
+    for (peer, peer_state_mutex) in &peer_states {
+        let peer_state = peer_state_mutex.lock().await;
+        peer_diagnostics.push(format!(
+            "{peer}: validated_swarm={:?}, epoch={:?}",
+            peer_state.standing.swarm_id(),
+            peer_state.epoch,
+        ));
+    }
+    warn!(
+        local_swarm = ?state.swarm_id().await,
+        local_epoch = %state.local_epoch(),
+        initial_sync_done = state.initial_sync_done.load(Ordering::Acquire),
+        unresolved_seeds = ?unresolved,
+        peers = %peer_diagnostics.join("; "),
+        "Still joining the swarm; trying on"
+    );
 }