//! Joining a swarm (or creating one as the genesis node). //! //! A node starts a new swarm only with `--genesis`. Otherwise it joins through its //! seeds and keeps trying until it is ready: seeds that don't resolve are resolved //! again with backoff, and HELLOs to resolved ones are retried by the peer worker. A //! node without seeds waits (nothing to join yet) and says so now and then. //! //! The genesis node records the swarm ID in the document (`_continuum/swarm`), so //! snapshots carry it forward and every backup names its swarm. //! //! A node recovering from a backup (`--recover-from`) starts on the backup's document, //! swarm and epoch instead, and is ready at once, like a genesis node with content. use std::sync::atomic::Ordering; use std::time::Duration; use std::collections::HashSet; use std::path::Path; use anyhow::{Context, Result, bail}; use serde_json::Value as JsonValue; use tokio::time::Instant; use tracing::{debug, info, warn}; use uuid::Uuid; use crate::document; use crate::document::meta::{self, SWARM}; use crate::document::replica::Epoch; use crate::document::store; use crate::document::{ActorId, Document}; use crate::files::{FileKind, Files}; use crate::force::Force; use crate::remake::snapshot; use crate::state::AppState; use crate::swarm::peer::{remember_peer, send_hello_once}; /// How often a node that is still joining says why. const BOOTSTRAP_REPORT: Duration = Duration::from_secs(300); /// Retrying seeds that didn't resolve: first after this, doubling up to the maximum. const RESOLVE_RETRY: Duration = Duration::from_secs(5); const RESOLVE_RETRY_MAX: Duration = Duration::from_secs(300); /// How this node starts. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Start { /// Join a swarm through seeds (or remembered members). Join, /// Start a swarm, empty (`--genesis`). Genesis, /// Start on a document already loaded as the replica (`StartingDocument`). FromDocument(Origin), } pub(crate) async fn bootstrap(state: &AppState, seeds: &[String], start: Start) -> Result<()> { if let Start::FromDocument(origin) = start { let Some(swarm_id) = state.swarm_id().await else { bail!("starting on a document without a swarm ID"); }; info!( swarm_id = %swarm_id, epoch = %state.local_epoch(), "Starting on {}, ready at once", origin.describe() ); state.initial_sync_done.store(true, Ordering::Release); state.mark_ready(swarm_id).await?; // Members of the swarm still running, if any: they merge with this node, or it // rejoins them if they moved on to a newer epoch. reach_seeds(state, seeds).await; return Ok(()); } if start == Start::Genesis { // A remembered swarm starts again under its ID, from a root of its own: nodes of // it still running hold another, and refuse it (`swarm::roots`); joining them // takes seeds instead. let swarm_id = match state.swarm_id().await { Some(remembered) => { warn!( swarm_id = %remembered, "--genesis with a remembered swarm: starting it again, empty, from a new root; nodes of it still running refuse it (another root): give seeds to join them instead" ); remembered } None => { let swarm_id = Uuid::new_v4().to_string(); info!(swarm_id = %swarm_id, "Starting a new swarm as its genesis node (--genesis)"); swarm_id } }; let recorded = JsonValue::String(swarm_id.clone()); let neighbours = state.neighbours_k; // Plain names (kebab-case): serializing them can't fail. let topology = serde_json::to_value(state.configured_ways.0).unwrap_or_default(); let dissemination = serde_json::to_value(state.configured_ways.1).unwrap_or_default(); // The first users, granted by this node: with the rest, so that no node sees the // swarm without its registry (none named: none). let (by, since) = ( state.local_id().to_owned(), state.clock.now_millis().as_u64(), ); let users: Vec<(Vec, JsonValue)> = state .first_users .iter() .map(|(name, keys)| { let caps = crate::access::registry::all_caps(); let mut entry = crate::access::registry::entry(&by, since, &caps); if !keys.is_empty() { entry["keys"] = serde_json::json!(keys); } (crate::access::registry::user_path(name), entry) }) .collect(); // Its policy too: no change of the swarm is then from before it. let policy = state.first_policy.clone(); state .mutate_doc(move |doc| { for (path, entry) in &users { store::write_path(doc, path, entry)?; } if let Some(policy) = &policy { store::write_path(doc, &crate::access::policy::path(), policy)?; } store::write_path(doc, &swarm_id_path(), &recorded)?; store::write_path( doc, &crate::swarm::topology::neighbours_path(), &JsonValue::from(neighbours), )?; store::write_path(doc, &crate::swarm::topology::topology_path(), &topology)?; store::write_path( doc, &crate::swarm::topology::dissemination_path(), &dissemination, ) }) .await .context( "Recording the swarm ID, its neighbours, its topology and dissemination, its first users and policy", )?; if let Some(root) = crate::swarm::roots::own(state).await { info!(root = %root, "The swarm's root: a node joining may be given it (--genesis-root), to take no other"); } state.initial_sync_done.store(true, Ordering::Release); state.mark_ready(swarm_id).await?; // Remembered members, if any, to find those still running. reach_seeds(state, seeds).await; return Ok(()); } if seeds.is_empty() { warn!( "No seeds: nothing to join. Add --seeds to join a swarm, or --genesis to start a new one" ); } else { info!(seeds = seeds.len(), "Joining existing swarm"); } // An empty document of the first epoch takes a root once its seeds named it, or a // while after the first did (`swarm::roots`). { let replica = state.doc.read().await; if replica.epoch.generation == 0 && replica.doc.get_heads().is_empty() { crate::swarm::roots::await_seeds(state, seeds.len()); } } let mut unresolved: Vec = seeds.to_vec(); let mut failing = HashSet::new(); let mut resolve_retry = RESOLVE_RETRY; let mut next_resolve = Instant::now(); let mut next_report = Instant::now() + BOOTSTRAP_REPORT; state.transport.on_seeds_found(state.join_progress.clone()); loop { if let Some(swarm_id) = state.swarm_id().await { // A node joining a compacted swarm rejoins the newer epoch first; it is // ready only once that rejoin verified the epoch. if state.initial_sync_done.load(Ordering::Acquire) && !state.remake.is_rejoining() { state.mark_ready(swarm_id.clone()).await?; info!(swarm_id = %swarm_id, "Node is ready"); crate::swarm::peer::announce(state); return Ok(()); } } let now = Instant::now(); // Nodes the transport found by itself (libp2p on the LAN): seeds too. let found = state.transport.discovered_seeds(); if !found.is_empty() { for seed in found { if !unresolved.contains(&seed) { info!(seed = %seed, "Found a node nearby; joining through it"); unresolved.push(seed); } } next_resolve = now; } if !unresolved.is_empty() && now >= next_resolve { resolve_seeds(state, &mut unresolved, &mut failing).await; if !unresolved.is_empty() { next_resolve = now + resolve_retry; resolve_retry = (resolve_retry * 2).min(RESOLVE_RETRY_MAX); } } if now >= next_report { next_report = now + BOOTSTRAP_REPORT; report_joining(state, seeds, &unresolved).await; } // The wait for the seeds to name the swarm's root, over by now? crate::swarm::roots::repin(state); // Until something changes (`join_progress`), or it's time to resolve, report or // stop waiting for the seeds. let resolving = async { if unresolved.is_empty() { std::future::pending().await } else { tokio::time::sleep_until(next_resolve).await } }; let root_wait = async { match crate::swarm::roots::waiting_until(state) { Some(until) => tokio::time::sleep_until(until).await, None => std::future::pending().await, } }; tokio::select! { () = state.join_progress.notified() => {} () = resolving => {} () = root_wait => {} () = tokio::time::sleep_until(next_report) => {} } } } fn swarm_id_path() -> Vec { meta::path(&[SWARM]) } /// The swarm ID the document names, if any. fn read_swarm_id(doc: &Document) -> Option { match store::read_path(doc, &swarm_id_path()) { Ok(Some(JsonValue::String(id))) if !id.is_empty() => Some(id), _ => None, } } /// Where a document to start from came from. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Origin { /// A backup to recover from (`--recover-from`). Backup, /// This node's document file (`--document-file`). DocumentFile, /// A store's root, its genesis change built from it (`--store-root`). #[cfg(feature = "federation")] Root, } impl Origin { fn describe(self) -> &'static str { match self { Origin::Backup => "the backup", Origin::DocumentFile => "the document file", #[cfg(feature = "federation")] Origin::Root => "the store's root", } } } /// The swarm holding a store that a node starts from its root (`--store-root`) or joins /// by its ID (`--store-id`). #[cfg(feature = "federation")] pub(crate) struct StoreStart { /// The root's ID, the swarm's. pub(crate) id: crate::federation::store::Id, /// The root, given, and the events it keeps: a fork's. root: Option<(Vec, Vec>)>, } #[cfg(feature = "federation")] impl StoreStart { /// The store `root` or `id` names, if either does: a root the federation takes as one, /// with the events it keeps, which a store's root must be. The file `root` holds the /// root on its first line, and, for a fork, the events it keeps, each on a line of its /// own (`federation::fork::root_file`). pub(crate) fn of(root: Option<&Path>, id: Option<&str>, files: &Files) -> Result> { use crate::federation::fold::Federation; use crate::federation::store::{Id, memory::Store}; let Some(path) = root else { return Ok(id .and_then(Id::from_hex) .map(|id| StoreStart { id, root: None })); }; let file = files .read(FileKind::Store, path) .with_context(|| format!("Reading the store's root {}", path.display()))? .with_context(|| format!("The store's root {} doesn't exist", path.display()))?; let (root, kept) = crate::federation::fork::read_root_file(&file); if let Err(refused) = Store::::new(root.clone(), kept.clone()) { bail!("The store's root {} is none: {}", path.display(), refused.0); } Ok(Some(StoreStart { id: Id::of(&root), root: Some((root, kept)), })) } /// The swarm's ID: the root's, in hex. pub(crate) fn swarm_id(&self) -> String { self.id.to_string() } /// The document to start from, `starting` checked against the store: of its swarm, /// as the state file's `remembered` is. Without one, given the root, the document of /// its genesis change, which every node given the root builds alike; joining, none. pub(crate) fn starting( &self, starting: Option, remembered: Option<&str>, ) -> Result> { let swarm_id = self.swarm_id(); if let Some(remembered) = remembered.filter(|remembered| *remembered != swarm_id) { bail!( "The state file remembers swarm {remembered}, not the store's ({swarm_id}): give the store's state file, or another" ); } if let Some(starting) = starting { if starting.swarm_id != swarm_id { bail!( "{} is of swarm {}, not the store's ({swarm_id})", starting.origin.describe(), starting.swarm_id ); } return Ok(Some(starting)); } let Some((root, kept)) = &self.root else { return Ok(None); }; use crate::federation::store::Fold; let doc = document::events::started(crate::federation::fold::Federation::NAME, root, kept)?; info!(swarm_id = %swarm_id, "Starting the swarm holding the store from its root (--store-root)"); Ok(Some(StartingDocument { doc, epoch: Epoch::GENESIS, swarm_id, origin: Origin::Root, earlier_actors: Vec::new(), signatures: Vec::new(), resumable: None, })) } } /// A document the node starts from, ready at once: on the swarm and epoch it names. pub(crate) struct StartingDocument { /// Under a fresh actor. pub(crate) doc: Document, pub(crate) epoch: Epoch, pub(crate) swarm_id: String, pub(crate) origin: Origin, /// The actors this node wrote with before (from the document file). pub(crate) earlier_actors: Vec, /// The signatures of its changes (from the document file). pub(crate) signatures: Vec<(crate::document::ChangeHash, [u8; 64])>, /// The actor this node wrote with, where it stopped cleanly (the document file's /// `resumable`): it may write on as that one (`--keep-actor`). pub(crate) resumable: Option, } /// What `--recover-from` and `--document-file` say about where to start. pub(crate) struct StartOptions<'a> { pub(crate) recover_from: Option<&'a Path>, pub(crate) document_file: Option<&'a Path>, pub(crate) genesis: bool, /// The swarm the state file remembers. pub(crate) remembered: Option<&'a str>, } /// The document to start from, if any, checked against the other options. Conflicts /// are errors unless forced (`force`): /// * an unreadable document file (`badDocumentFile`: moved aside, start without it); /// * a backup to recover from while the document file holds a document /// (`replaceDocumentFile`: the backup wins, and the file is rewritten from it); /// * a document of another swarm than the state file remembers (`swarmId`). /// /// A document file holding a document with `--genesis` is an error: move it away. pub(crate) fn starting_document( options: &StartOptions<'_>, files: &Files, force: &Force, ) -> Result> { let from_file = match options.document_file { None => None, Some(path) => match document::file::load(path, files) { Ok(loaded) => loaded, Err(error) if force.allows("badDocumentFile") => { warn!(error = %format!("{error:#}"), "Unusable document file; moving it aside and starting without it (--force badDocumentFile)"); let aside = files.move_aside(path, ".bad").with_context(|| { format!("Moving the document file {} aside", path.display()) })?; warn!(moved_to = %aside.display(), "Moved the document file aside"); None } Err(error) => { bail!( "{error:#}; add --force badDocumentFile to move it aside and start without it" ) } }, }; let backup = match options.recover_from { None => None, Some(path) => Some(load_backup(path, files)?), }; let (doc, origin, earlier_actors, signatures, resumable) = match (backup, from_file) { (Some(_), Some(_)) if !force.allows("replaceDocumentFile") => { bail!( "recover-from would replace the document in the document file; add --force replaceDocumentFile to recover the backup anyway" ); } (Some(backup), from_file) => { if from_file.is_some() { warn!( "Recovering the backup instead of the document file's document (--force replaceDocumentFile)" ); } (backup, Origin::Backup, Vec::new(), Vec::new(), None) } (None, Some(loaded)) => ( loaded.doc, Origin::DocumentFile, loaded.actors, loaded.signatures, loaded.resumable, ), (None, None) => return Ok(None), }; if options.genesis && origin == Origin::DocumentFile { bail!( "The document file holds a document of a swarm, and genesis would start a new one; move the file away to start a new swarm" ); } let mut starting = identify(doc, origin, earlier_actors)?; starting.signatures = signatures; starting.resumable = resumable; if let Some(remembered) = options.remembered.filter(|id| *id != starting.swarm_id) { let what = origin.describe(); if !force.allows("swarmId") { bail!( "{what} is of swarm {}, but the state file remembers swarm {remembered}; add --force swarmId to start on {what}'s swarm anyway", starting.swarm_id ); } warn!( document = %starting.swarm_id, remembered = %remembered, "Starting on {what}'s swarm, not the remembered one (--force swarmId)" ); } Ok(Some(starting)) } /// The swarm and epoch `doc` is on. fn identify( doc: Document, origin: Origin, earlier_actors: Vec, ) -> Result { let what = origin.describe(); let Some(swarm_id) = read_swarm_id(&doc) else { bail!("{what} names no swarm (no _continuum/swarm); it predates swarm IDs in documents"); }; let epoch = snapshot::epoch_of(&doc).map_err(|reason| anyhow::anyhow!("{what}: {reason}"))?; Ok(StartingDocument { doc, epoch, swarm_id, origin, earlier_actors, signatures: Vec::new(), resumable: None, }) } /// A backup to recover from, as `GET /admin/document` saves it (`document::load`). fn load_backup(path: &Path, files: &Files) -> Result { let bytes = files .read(FileKind::Backup, path) .with_context(|| format!("Reading the backup {}", path.display()))? .with_context(|| format!("The backup {} doesn't exist", path.display()))?; crate::document::load(&bytes).with_context(|| format!("Loading the backup {}", path.display())) } /// Resolves the seeds in `unresolved`, says HELLO to each that resolves, and keeps the /// rest for later. Warns once per seed that fails (`failing` remembers which), and says /// when such a seed resolves after all. /// Resolves `seeds`, those of a node ready at once, and says HELLO to each; those that /// don't resolve yet (the node, or they, not reachable yet) are tried again in the /// background, backing off as a joining node does, until they resolve or the node stops. async fn reach_seeds(state: &AppState, seeds: &[String]) { let mut unresolved = seeds.to_vec(); let mut failing = HashSet::new(); resolve_seeds(state, &mut unresolved, &mut failing).await; if unresolved.is_empty() { return; } let state = state.clone(); let mut closing = state.closing.subscribe(); tokio::spawn(async move { let mut retry = RESOLVE_RETRY; while !unresolved.is_empty() { tokio::select! { () = tokio::time::sleep(retry) => {} _ = closing.wait_for(|closing| *closing) => return, } resolve_seeds(&state, &mut unresolved, &mut failing).await; retry = (retry * 2).min(RESOLVE_RETRY_MAX); } }); } async fn resolve_seeds( state: &AppState, unresolved: &mut Vec, failing: &mut HashSet, ) { let mut still = Vec::new(); for seed in unresolved.drain(..) { match state.transport.resolve_seed(&seed).await { Ok(peer) => { if failing.remove(&seed) { info!(seed = %seed, peer = %peer, "Seed resolves now"); } else { debug!(seed = %seed, peer = %peer, "Resolved seed"); } remember_peer(state, &peer).await; crate::swarm::roots::seed_found(state, &peer); if let Err(error) = send_hello_once(state, &peer).await { warn!(peer = %peer, error = %format!("{error:#}"), "Initial HELLO failed; will retry"); } } Err(error) => { if failing.insert(seed.clone()) { warn!(seed = %seed, error = %error, "Seed doesn't resolve; trying again until it does"); } else { debug!(seed = %seed, error = %error, "Seed still doesn't resolve"); } still.push(seed); } } } *unresolved = still; } /// Why the node is still joining, for the log. async fn report_joining(state: &AppState, seeds: &[String], unresolved: &[String]) { if seeds.is_empty() { warn!( "Still no seeds: nothing to join. Add --seeds to join a swarm, or --genesis to start a new one" ); return; } let mut peer_diagnostics: Vec = Vec::new(); let peer_states = state.peer_states.read().await.clone(); for (peer, peer_state_mutex) in &peer_states { let peer_state = peer_state_mutex.lock().await; peer_diagnostics.push(format!( "{peer}: validated_swarm={:?}, epoch={:?}", peer_state.standing.swarm_id(), peer_state.epoch, )); } // Awaited outside the macro: its arguments aren't Send, and a spawned bootstrap // (the simulator's) must be. let local_swarm = state.swarm_id().await; warn!( local_swarm = ?local_swarm, local_epoch = %state.local_epoch(), initial_sync_done = state.initial_sync_done.load(Ordering::Acquire), unresolved_seeds = ?unresolved, peers = %peer_diagnostics.join("; "), "Still joining the swarm; trying on" ); } #[cfg(all(test, feature = "federation"))] mod tests { use super::*; use crate::federation::store::Id; /// Given its root, a node starts the swarm holding the store from its genesis change, /// on no other swarm's document; joining by its ID, from nothing. #[test] fn a_store_starts_from_its_root() { let root = b"a root".to_vec(); let store = StoreStart { id: Id::of(&root), root: Some((root, Vec::new())), }; let started = store.starting(None, None).unwrap().unwrap(); assert_eq!(started.swarm_id, store.swarm_id()); assert_eq!(started.origin, Origin::Root); assert_eq!(started.epoch, Epoch::GENESIS); assert_eq!(read_swarm_id(&started.doc), Some(store.swarm_id())); assert!(store.starting(None, Some("another")).is_err()); let other = StoreStart { id: Id::of(b"another"), root: Some((b"another".to_vec(), Vec::new())), }; assert!( store .starting(other.starting(None, None).unwrap(), None) .is_err() ); let joining = StoreStart { id: store.id, root: None, }; assert!(joining.starting(None, None).unwrap().is_none()); } /// A fork's root, and the event it keeps on a line of its own: the swarm starts from /// the changes founding it. Without that event: no store's root. #[test] fn a_fork_starts_from_its_root_and_what_it_keeps() { use crate::federation::fold::Federation; use crate::federation::harness::{self, Party, Spec}; use crate::federation::store::memory::Store; let tv = Party::new("tv", 1); let first = Spec::first(&[&tv], &[]); let mut store = Store::::new(first.root(), Vec::new()).unwrap(); let write = tv.write(&[store.id()], "home/tv/x", harness::str("1")); let head = Id::of(&write); store.offer(write); let mut spec = first.after(first.hash()); spec.parties[0].nodes_since = 1; let (root, kept) = store.fork(&[head], &spec.json().bytes()).unwrap(); let dir = crate::files::scratch_dir("fork-root"); let (path, alone) = (dir.join("fork"), dir.join("alone")); let files = Files::default(); std::fs::write(&path, crate::federation::fork::root_file(&root, &kept)).unwrap(); std::fs::write(&alone, &root).unwrap(); let fork = StoreStart::of(Some(&path), None, &files).unwrap().unwrap(); assert_eq!(fork.id, Id::of(&root)); let started = fork.starting(None, None).unwrap().unwrap(); assert_eq!( document::events::bytes(&started.doc, &head), Some(kept[0].clone()) ); assert!(StoreStart::of(Some(&alone), None, &files).is_err()); } }