//! Shared node state: the document, lifecycle, per-peer state and timing. use std::collections::HashMap; use std::future::Future; use std::sync::atomic::AtomicBool; use std::sync::Arc; use std::time::Duration; use anyhow::{bail, Result}; use automerge::{sync, AutoCommit}; use tokio::sync::{broadcast, mpsc, watch, Mutex, RwLock}; use tokio::time::Instant; use tracing::{debug, error, warn}; use crate::compaction::{Compaction, Host, PeerSummary}; use crate::firewall::Firewall; use crate::replica::{Epoch, Replica}; use crate::sync::{ChangeSet, SyncEvent}; use crate::transport; #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum NodeLifecycle { // TODO maybe rename discovered_cluster_id to cluster_id if that doesn't impair readability. Joining { discovered_cluster_id: Option, }, Ready { cluster_id: String, }, } #[derive(Debug, Clone, Default, PartialEq, Eq)] pub(crate) enum PeerMembership { #[default] Unknown, Member { cluster_id: String, }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Liveness { Active, /// No inbound message for Timing::dormant_after. Kept indefinitely, probed slowly. Dormant { since: Instant, }, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) enum HelloState { #[default] NotSent, /// A send is in flight. `next_delay` is how long to wait after it before retrying. Sending { next_delay: Duration }, /// HELLO was sent (or the send failed); no HELLO_REPLY yet. AwaitingReply { next_attempt: Instant, next_delay: Duration, }, /// HELLO_REPLY received. No more retries. Established, } #[derive(Clone)] pub(crate) struct PendingOutbound { pub(crate) message: transport::EncodedMessage, pub(crate) retry_delay: Duration, pub(crate) next_attempt: Instant, } #[derive(Clone)] pub(crate) struct PeerState { pub(crate) membership: PeerMembership, pub(crate) sync_state: sync::State, pub(crate) hello_state: HelloState, /// The peer sent HELLO; we still owe it a HELLO_REPLY (e.g. we had no cluster ID yet, /// or sending failed). pub(crate) reply_owed: bool, /// Received from this peer, waiting to be processed. pub(crate) pending_inbound_sync: Vec, /// Generated for this peer, waiting for delivery. pub(crate) pending_outbound_sync: Option, /// A sync pass was requested while an envelope was pending. Once that envelope /// is delivered, the peer must be re-enqueued so the deferred changes go out. pub(crate) needs_generate: bool, pub(crate) liveness: Liveness, pub(crate) added_at: Instant, pub(crate) last_seen: Option, pub(crate) last_ping_at: Option, /// Last time a sync message from this peer was applied. pub(crate) last_sync_rx: Option, /// Last self-healing re-handshake (cooldown). pub(crate) last_resync_at: Option, /// Epoch the peer advertised in its most recent message. pub(crate) epoch: Option, /// Consecutive sync exchanges that changed nothing (livelock detection). pub(crate) stalled_exchanges: u32, } impl Default for PeerState { fn default() -> Self { Self { membership: PeerMembership::default(), sync_state: sync::State::new(), hello_state: HelloState::default(), reply_owed: false, pending_inbound_sync: Vec::new(), pending_outbound_sync: None, needs_generate: false, liveness: Liveness::Active, added_at: Instant::now(), last_seen: None, last_ping_at: None, last_sync_rx: None, last_resync_at: None, stalled_exchanges: 0, epoch: None, } } } impl PeerState { /// Start Automerge sync with this peer from scratch. Any envelope generated against /// the old state is discarded; a stale RetryDelivery event is then a no-op. pub(crate) fn reset_sync(&mut self) { self.sync_state = sync::State::new(); self.pending_outbound_sync = None; self.needs_generate = false; } fn quiet_for(&self, now: Instant) -> Duration { now.saturating_duration_since(self.last_seen.unwrap_or(self.added_at)) } /// Records inbound activity. Returns true if the peer was dormant. pub(crate) fn mark_seen(&mut self, now: Instant) -> bool { self.last_seen = Some(now); if matches!(self.liveness, Liveness::Dormant { .. }) { self.liveness = Liveness::Active; // Force a fresh handshake; its reply resets sync on both sides. if !matches!(self.hello_state, HelloState::Sending { .. }) { self.hello_state = HelloState::NotSent; } return true; } false } /// Moves a silent peer to Dormant. Returns true on the transition. pub(crate) fn check_dormancy(&mut self, timing: &Timing, now: Instant) -> bool { if self.liveness != Liveness::Active || self.quiet_for(now) < timing.dormant_after { return false; } self.liveness = Liveness::Dormant { since: now }; self.reset_sync(); if !matches!(self.hello_state, HelloState::Sending { .. }) { self.hello_state = HelloState::AwaitingReply { next_attempt: now + timing.dormant_probe, next_delay: timing.dormant_probe, }; } true } /// True (and records the ping) if an established, quiet peer needs a keepalive. pub(crate) fn take_keepalive_due(&mut self, timing: &Timing, now: Instant) -> bool { let due = self.liveness == Liveness::Active && self.hello_state == HelloState::Established && now.saturating_duration_since(self.added_at) >= timing.keepalive && self.quiet_for(now) >= timing.keepalive && self .last_ping_at .is_none_or(|t| now.saturating_duration_since(t) >= timing.keepalive); if due { self.last_ping_at = Some(now); } due } /// Schedules a fresh handshake, which resets sync on both sides: the peer resets on /// our HELLO, we reset on its reply. The peer worker sends the HELLO on its next tick. /// Usable without awaiting (e.g. under a blocking lock). pub(crate) fn request_resync(&mut self) { if matches!(self.hello_state, HelloState::Sending { .. }) { return; // a HELLO is already on its way } self.hello_state = HelloState::AwaitingReply { next_attempt: Instant::now(), next_delay: Duration::from_millis(500), }; } /// Self-healing check on PING/PONG. Returns true if a re-handshake should start. pub(crate) fn needs_heal( &mut self, local: &transport::HeadsDigest, remote: &transport::HeadsDigest, timing: &Timing, now: Instant, ) -> bool { if local == remote || self.liveness != Liveness::Active || self.hello_state != HelloState::Established { return false; } let sync_idle = now.saturating_duration_since(self.last_sync_rx.unwrap_or(self.added_at)) >= timing.resync_after; let cooled_down = self .last_resync_at .is_none_or(|t| now.saturating_duration_since(t) >= timing.resync_after); if !(sync_idle && cooled_down) { return false; } self.last_resync_at = Some(now); self.hello_state = HelloState::NotSent; // establish_peer resets sync on the reply true } /// Heard from recently. A returning node counts as alive from its first message, /// before the handshake finishes, so a cluster-wide forget can't make it flap. pub(crate) fn is_alive(&self, timing: &Timing, now: Instant) -> bool { self.liveness == Liveness::Active && self .last_seen .is_some_and(|t| now.saturating_duration_since(t) < timing.dormant_after) } } #[derive(Clone, Copy, Debug)] pub(crate) struct Timing { pub(crate) keepalive: Duration, pub(crate) dormant_after: Duration, pub(crate) resync_after: Duration, pub(crate) dormant_probe: Duration, } impl Timing { // Recommended values // Environment | Keepalive | Notes // --------------------------------------------+-----------+------------------------------------------------------------ // Test harness | 1-2 s | Dormancy after about 4–8 s and healing within seconds keep tests short. // LAN or small cluster (up to ~50 nodes) | 15-30 s | Under 3.5 messages/s per node at 50 nodes and 30 s. Current setting. // WAN, mobile or metered links, or 100+ nodes | 60-120 s | At 100 nodes and 30 s it’s about 6–7 messages/s per node. Fine, but rarely worth it on battery or metered links. // Several hundred nodes or more | - | The full mesh becomes the problem in itself, not the interval. Message counts grow O(N²) across the cluster, which calls for partial views (SWIM-style random probing and syncing with a subset of peers). pub(crate) fn from_keepalive(keepalive: Duration) -> Self { Self { keepalive, dormant_after: keepalive * 4, // dormant_after >= 4* keepalive to tolerate three lost PINGs in a row resync_after: keepalive * 2, // resync_after should be >= 2* keepalive to guarantee at least one digest exchange while sync is idle dormant_probe: keepalive * 10, // >= 10*: dormant peers cost almost nothing, but still rejoin within minutes } } } #[derive(Clone)] pub(crate) struct AppState { /// The document and its epoch. Derefs to `AutoCommit`. pub(crate) doc: Arc>, /// Copy of `doc.epoch` for cheap, lock-free reads (handshakes, keepalives, /peers). /// Only ever changed while holding `doc`'s write lock, together with `doc.epoch`. pub(crate) epoch_tx: watch::Sender, pub(crate) lifecycle: Arc>, // Serializes cluster-ID establishment, peer membership, and readiness transitions. //[concurrent]cluster_validation: Arc>, pub(crate) peer_states: Arc>>>>, pub(crate) transport: Arc, pub(crate) firewall: Arc, pub(crate) sync_events: mpsc::UnboundedSender, // TODO see key_watchers.md ("Use a per-key notification registry instead of the global `watch`") pub(crate) change_tx: watch::Sender, pub(crate) initial_sync_done: Arc, pub(crate) change_events: broadcast::Sender>, pub(crate) timing: Timing, /// Per-node compaction state (prebuilt snapshots, switch status, archive). pub(crate) compaction: Arc, /// While rejoining a newer epoch, the previous document, served read-only (and /// marked stale) instead of answering 503. pub(crate) stale: Arc>>, } impl AppState { pub(crate) fn local_id(&self) -> &str { self.transport.local_id() } /// The local epoch. Code that labels or applies sync messages must use the epoch /// of the locked `Replica` instead, so label and document can't disagree. pub(crate) fn local_epoch(&self) -> Epoch { *self.epoch_tx.borrow() } pub(crate) async fn is_ready(&self) -> bool { matches!(*self.lifecycle.read().await, NodeLifecycle::Ready { .. }) } pub(crate) async fn cluster_id(&self) -> Option { match &*self.lifecycle.read().await { NodeLifecycle::Ready { cluster_id } => Some(cluster_id.clone()), NodeLifecycle::Joining { discovered_cluster_id, } => discovered_cluster_id.clone(), } } // Record the expected cluster without making the node Ready. Bootstrap owns readiness. pub(crate) async fn set_expected_cluster_id(&self, cluster_id: String) -> Result<()> { //[concurrent]let _guard = self.cluster_validation.lock().await; let mut lifecycle = self.lifecycle.write().await; match &*lifecycle { NodeLifecycle::Joining { discovered_cluster_id: Some(existing), } if existing != &cluster_id => { bail!( "Conflicting expected cluster IDs: expected {}, received {}", existing, cluster_id ); } NodeLifecycle::Joining { discovered_cluster_id: None, } => { let old_lifecycle = lifecycle.clone(); *lifecycle = NodeLifecycle::Joining { discovered_cluster_id: Some(cluster_id), }; debug!("Node transitioned from {old_lifecycle:?} {lifecycle:?}"); Ok(()) } NodeLifecycle::Joining { discovered_cluster_id: Some(_), } => { warn!("Node already Joining with the same identity"); Ok(()) } NodeLifecycle::Ready { cluster_id: existing, } if existing == &cluster_id => { warn!("Node already Ready with the same identity"); Ok(()) } NodeLifecycle::Ready { cluster_id: existing, } => { bail!( "Cannot change established cluster ID: existing {}, received {}", existing, cluster_id ); } } } // Only bootstrap should call this, after validating document metadata. pub(crate) async fn mark_ready(&self, cluster_id: String) -> Result<()> { //[concurrent]let _guard = self.cluster_validation.lock().await; let mut lifecycle = self.lifecycle.write().await; match &*lifecycle { NodeLifecycle::Joining { discovered_cluster_id: Some(expected), } if expected != &cluster_id => { bail!( "Cannot mark node ready: expected cluster {}, received {}", expected, cluster_id ); } NodeLifecycle::Joining { .. } => { let old_lifecycle = lifecycle.clone(); *lifecycle = NodeLifecycle::Ready { cluster_id }; debug!("Node transitioned from {old_lifecycle:?} {lifecycle:?}"); } NodeLifecycle::Ready { cluster_id: existing, } if existing == &cluster_id => { debug!("Node already ready with the same identity"); } NodeLifecycle::Ready { cluster_id: existing, } => { bail!( "Cannot change established cluster ID: existing {}, received {}", existing, cluster_id ); } } Ok(()) } /// Looks up a peer without creating it. Background work (sync, retries, /// keepalives) must use this so a forgotten peer isn't resurrected. pub(crate) async fn get_peer_state(&self, pk: &str) -> Option>> { self.peer_states.read().await.get(pk).cloned() } pub(crate) async fn get_or_create_peer_state(&self, pk: &str) -> Arc> { let mut peers = self.peer_states.write().await; Arc::clone( peers .entry(pk.to_owned()) .or_insert_with(|| Arc::new(Mutex::new(PeerState::default()))), ) } pub(crate) fn notify_store_changed(&self) { self.change_tx.send_modify(|version| { *version = version.wrapping_add(1); }); } /// Reads the document on the blocking pool; `finish` runs after the lock is released. pub(crate) async fn render( &self, read: impl FnOnce(&AutoCommit) -> T + Send + 'static, finish: impl FnOnce(T) -> R + Send + 'static, ) -> R where R: Send + 'static, { let doc = Arc::clone(&self.doc); tokio::task::spawn_blocking(move || { let data = { let guard = doc.blocking_read(); read(&guard.doc) }; finish(data) }) .await .expect("document read task panicked") } /// Mutates the document on the blocking pool. pub(crate) async fn write_doc( &self, f: impl FnOnce(&mut AutoCommit) -> T + Send + 'static, ) -> T where T: Send + 'static, { let doc = Arc::clone(&self.doc); tokio::task::spawn_blocking(move || f(&mut doc.blocking_write().doc)) .await .expect("document write task panicked") } /// Mutates the document on the blocking pool as one Automerge change, then /// publishes it. Write and notification run in the same blocking task, so a /// cancelled request can't leave a committed write unannounced. pub(crate) async fn mutate_doc( &self, f: impl FnOnce(&mut AutoCommit) -> Result + Send + 'static, ) -> Result where T: Send + 'static, E: Send + 'static, { let state = self.clone(); tokio::task::spawn_blocking(move || { let result = { let mut replica = state.doc.blocking_write(); let result = f(&mut replica.doc); match &result { // One API write = one Automerge change. Ok(_) => { replica.doc.commit(); } // Discard partial ops (e.g. old value deleted, new one not written). Err(_) => { replica.doc.rollback(); } } result }; // lock released before notifying if result.is_ok() { state.publish_change(); } result }) .await .expect("document write task panicked") } pub(crate) fn publish_change(&self) { self.notify_store_changed(); if let Err(error) = self.sync_events.send(SyncEvent::DocumentChanged) { // The write is committed; failing the request would misreport it. error!(error = %error, "Sync worker is gone; change will not be replicated"); } } } impl AppState { /// After the document was replaced: every peer's sync state refers to the old /// document, so start over with all of them. Peers still on the old epoch are /// skipped by the sync worker until they switch too. async fn after_epoch_switch(&self) { let peers: Vec<(String, Arc>)> = self .peer_states .read() .await .iter() .map(|(id, peer)| (id.clone(), Arc::clone(peer))) .collect(); // Sync states were reset under the document lock by the swap itself. self.publish_change(); // Tell peers about the new epoch now instead of with the next keepalive (up to a // keepalive interval later): sync with peers already on it resumes right away, // and peers that never saw the commit start catching up. let keepalive = crate::peer::local_keepalive(self).await; for (id, _) in &peers { if let Err(error) = self .transport .send_message(id, &transport::ParsedMessage::Ping(keepalive.clone())) .await { debug!(peer = %id, error = %format!("{error:#}"), "Epoch announcement failed"); } } for (id, _) in peers { let _ = self.sync_events.send(SyncEvent::PeerNeedsSync(id)); } } } /// A read-only view of the document the node served before it started rejoining. pub(crate) struct StaleView { pub(crate) epoch: Epoch, pub(crate) doc: Arc, } /// Where a read was served from. pub(crate) enum ReadSource { Live, /// The pre-rejoin document of this epoch. Stale(Epoch), } impl AppState { /// Renders a read for the store API: from the live document when the node is ready, /// from the stale view while it rejoins, and otherwise `503`, or the live document /// anyway with `unready_live`. pub(crate) async fn render_read( &self, unready_live: bool, read: impl FnOnce(&AutoCommit) -> T + Send + 'static, finish: impl FnOnce(T) -> R + Send + 'static, ) -> Result<(R, ReadSource), axum::http::StatusCode> where T: Send + 'static, R: Send + 'static, { if self.is_ready().await { return Ok((self.render(read, finish).await, ReadSource::Live)); } let stale = self .stale .read() .unwrap_or_else(|poisoned| poisoned.into_inner()) .as_ref() .map(|view| (view.epoch, Arc::clone(&view.doc))); let Some((epoch, doc)) = stale else { if unready_live { return Ok((self.render(read, finish).await, ReadSource::Live)); } return Err(axum::http::StatusCode::SERVICE_UNAVAILABLE); }; let rendered = tokio::task::spawn_blocking(move || finish(read(&doc))) .await .expect("document read task panicked"); Ok((rendered, ReadSource::Stale(epoch))) } fn set_stale(&self, view: Option) { *self .stale .write() .unwrap_or_else(|poisoned| poisoned.into_inner()) = view; } } pub(crate) fn unix_millis() -> u64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_millis() as u64) .unwrap_or(0) } /// The node as seen by the compaction module. impl Host for AppState { fn compaction(&self) -> &Compaction { &self.compaction } fn node_id(&self) -> String { self.local_id().to_owned() } fn current_epoch(&self) -> Epoch { self.local_epoch() } fn dormant_after(&self) -> Duration { self.timing.dormant_after } fn is_node_ready(&self) -> impl Future + Send { let state = self.clone(); async move { state.is_ready().await } } fn peer_summaries(&self) -> impl Future> + Send { let state = self.clone(); async move { let peers: Vec<(String, Arc>)> = state .peer_states .read() .await .iter() .map(|(id, peer)| (id.clone(), Arc::clone(peer))) .collect(); let mut summaries = Vec::with_capacity(peers.len()); for (id, peer) in peers { let peer = peer.lock().await; summaries.push(PeerSummary { id, active: peer.liveness == Liveness::Active, established: peer.hello_state == HelloState::Established, epoch: peer.epoch, }); } summaries } } fn suspend(&self) -> impl Future + Send { let state = self.clone(); async move { let mut lifecycle = state.lifecycle.write().await; let NodeLifecycle::Ready { cluster_id } = &*lifecycle else { return false; }; let cluster_id = cluster_id.clone(); debug!(cluster_id = %cluster_id, "Node suspended (rejoining)"); *lifecycle = NodeLifecycle::Joining { discovered_cluster_id: Some(cluster_id), }; true } } fn resume(&self) -> impl Future + Send { let state = self.clone(); async move { let mut lifecycle = state.lifecycle.write().await; if let NodeLifecycle::Joining { discovered_cluster_id: Some(cluster_id), } = &*lifecycle { let cluster_id = cluster_id.clone(); debug!(cluster_id = %cluster_id, "Node resumed"); *lifecycle = NodeLifecycle::Ready { cluster_id }; } } } fn set_stale_view(&self, view: Option<(Epoch, AutoCommit)>) { self.set_stale(view.map(|(epoch, doc)| StaleView { epoch, doc: Arc::new(doc), })); } fn initial_sync_done(&self) -> bool { self.initial_sync_done .load(std::sync::atomic::Ordering::Acquire) } fn reset_initial_sync(&self) { self.initial_sync_done .store(false, std::sync::atomic::Ordering::Release); } fn subscribe_changes(&self) -> broadcast::Receiver> { self.change_events.subscribe() } fn read_doc(&self, f: F) -> impl Future + Send where F: FnOnce(&AutoCommit) -> T + Send + 'static, T: Send + 'static, { let state = self.clone(); async move { state.render(f, |value| value).await } } fn update_doc(&self, f: F) -> impl Future + Send where F: FnOnce(&mut AutoCommit) -> T + Send + 'static, T: Send + 'static, { let state = self.clone(); async move { state.write_doc(f).await } } fn commit_doc(&self, f: F) -> impl Future> + Send where F: FnOnce(&mut AutoCommit) -> Result + Send + 'static, T: Send + 'static, E: Send + 'static, { let state = self.clone(); async move { state.mutate_doc(f).await } } fn switch_epoch( &self, epoch: Epoch, f: F, ) -> impl Future> + Send where F: FnOnce(&mut AutoCommit) -> Result + Send + 'static, E: Send + 'static, { let state = self.clone(); async move { let swapped = { let state = state.clone(); tokio::task::spawn_blocking(move || -> Result { let mut replica = state.doc.blocking_write(); let new_doc = f(&mut replica.doc)?; let old_doc = std::mem::replace(&mut replica.doc, new_doc); replica.epoch = epoch; // Updated under the same lock, so the cached epoch never disagrees // with the document for anyone holding the lock. state.epoch_tx.send_replace(epoch); // Reset every peer's sync state before the lock is released, so no // message is generated from or applied to a state that describes the // old document. (Lock order doc -> peer, as in sync generation.) let now = Instant::now(); for peer in state.peer_states.blocking_read().values() { let mut peer = peer.blocking_lock(); peer.reset_sync(); // A fresh start, not an idle sync: no heal re-handshake for it. peer.last_sync_rx = Some(now); } Ok(old_doc) }) .await .expect("epoch switch task panicked") }; if swapped.is_ok() { state.after_epoch_switch().await; } swapped } } }