diff options
| author | tv <tv@krebsco.de> | 2026-10-28 11:57:54 +0100 |
|---|---|---|
| committer | tv <tv@krebsco.de> | 2026-10-28 11:57:54 +0100 |
| commit | e13d027567f781c9729a518913b21ec4ee273039 (patch) | |
| tree | ada25e93f6164e0ce80282616937f19fc8f1aec0 | |
| parent | 8a4e5bab13069a298fa1d8c756e1a6c0bd719dfb (diff) | |
peers from member list start as unknown
| -rw-r--r-- | docs/openapi.yaml | 2 | ||||
| -rw-r--r-- | src/api/peers.rs | 3 | ||||
| -rw-r--r-- | src/api/stats/prometheus.rs | 10 | ||||
| -rw-r--r-- | src/state/host.rs | 6 | ||||
| -rw-r--r-- | src/state/peer.rs | 39 | ||||
| -rw-r--r-- | src/swarm/peer/handshake.rs | 4 | ||||
| -rw-r--r-- | src/swarm/peer/keepalive.rs | 4 |
7 files changed, 54 insertions, 14 deletions
diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 25a4a44..acd2ce0 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1164,7 +1164,7 @@ components: type: [string, 'null'] status: type: string - enum: [active, dormant] + enum: [unknown, active, dormant] handshakeComplete: type: boolean lastSeenSecsAgo: diff --git a/src/api/peers.rs b/src/api/peers.rs index cea3644..eeb3fb2 100644 --- a/src/api/peers.rs +++ b/src/api/peers.rs @@ -23,6 +23,8 @@ use crate::transport::traffic::PeerTraffic; #[derive(Debug, Serialize)] #[serde(rename_all = "lowercase")] pub(crate) enum PeerLivenessStatus { + /// Nothing heard from it since this node started. + Unknown, Active, Dormant, } @@ -134,6 +136,7 @@ pub(crate) async fn list_peers( let ps = peer_state_mutex.lock().await; let (status, dormant_for_secs) = match ps.liveness { + Liveness::Unknown => (PeerLivenessStatus::Unknown, None), Liveness::Active => (PeerLivenessStatus::Active, None), Liveness::Dormant { since } => ( PeerLivenessStatus::Dormant, diff --git a/src/api/stats/prometheus.rs b/src/api/stats/prometheus.rs index 9cbeff4..8b37864 100644 --- a/src/api/stats/prometheus.rs +++ b/src/api/stats/prometheus.rs @@ -117,6 +117,7 @@ struct Peer { key: String, alias: Option<String>, neighbour: bool, + unknown: bool, dormant: bool, last_seen_secs: Option<f64>, last_sync_secs: Option<f64>, @@ -143,6 +144,7 @@ async fn peers(state: &AppState) -> Vec<Peer> { let peer_state = peer_state.lock().await; peers.push(Peer { neighbour, + unknown: peer_state.liveness == Liveness::Unknown, dormant: matches!(peer_state.liveness, Liveness::Dormant { .. }), last_seen_secs: peer_state .last_seen @@ -853,6 +855,14 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String { flag(peer.neighbour), ); } + out.family( + "peer_unknown", + "gauge", + "1 while nothing was heard from the peer since this node started.", + ); + for peer in &peers { + out.sample("peer_unknown", &[("peer", &peer.key)], flag(peer.unknown)); + } out.family("peer_dormant", "gauge", "1 while the peer is dormant."); for peer in &peers { out.sample("peer_dormant", &[("peer", &peer.key)], flag(peer.dormant)); diff --git a/src/state/host.rs b/src/state/host.rs index 6eab803..b55239d 100644 --- a/src/state/host.rs +++ b/src/state/host.rs @@ -11,7 +11,7 @@ use tokio::sync::broadcast; use tokio::time::Instant; use tracing::debug; -use super::{AppState, HelloState, Liveness, NodeLifecycle, StaleView, Standing}; +use super::{AppState, HelloState, NodeLifecycle, StaleView, Standing}; use crate::document::replica::Epoch; use crate::locks::MutexExt; use crate::remake::{Host, PeerSummary, RemakeState}; @@ -42,7 +42,7 @@ impl AppState { let mut peer_state = peer_state_mutex.lock().await; // What it said about syncing with us was about our old epoch. peer_state.syncs_with_us = None; - let gets_our_opening = peer_state.liveness == Liveness::Active + let gets_our_opening = !peer_state.liveness.is_dormant() && matches!(peer_state.standing, Standing::Member { .. }) && peer_state.epoch == Some(local) && peer_state.reports_ready == Some(true); @@ -131,7 +131,7 @@ impl Host for AppState { let peer = peer.lock().await; summaries.push(PeerSummary { id, - active: peer.liveness == Liveness::Active, + active: !peer.liveness.is_dormant(), member: matches!(peer.standing, Standing::Member { .. }) && peer.hello_state == HelloState::Established, ready: peer.reports_ready, diff --git a/src/state/peer.rs b/src/state/peer.rs index b723ccd..9f2da0f 100644 --- a/src/state/peer.rs +++ b/src/state/peer.rs @@ -39,6 +39,11 @@ impl Standing { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Liveness { + /// Nothing heard from it since this node started: known only from the member list + /// or another peer. It counts as active (sync, keepalives and dormancy treat it so) + /// until its first message makes it active, or `Timing::dormant_after` of silence + /// dormant. + Unknown, Active, /// No inbound message for Timing::dormant_after. Kept indefinitely, probed slowly. Dormant { @@ -46,6 +51,12 @@ pub(crate) enum Liveness { }, } +impl Liveness { + pub(crate) fn is_dormant(self) -> bool { + matches!(self, Self::Dormant { .. }) + } +} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) enum HelloState { #[default] @@ -184,7 +195,7 @@ impl PeerState { hello_backoff: Backoff::new(HELLO_RETRY, HELLO_RETRY_MAX), probe_backoff: timing.probe_backoff(), sync: SyncSession::new(timing.sync_reply_backoff()), - liveness: Liveness::Active, + liveness: Liveness::Unknown, added_at: Instant::now(), last_seen: None, last_keepalive_at: None, @@ -219,7 +230,7 @@ impl PeerState { /// An overdue sync answer from an established, active peer (see /// `SyncSession::take_overdue_reply`). pub(crate) fn take_overdue_sync_reply(&mut self, now: Instant) -> Option<usize> { - if self.liveness != Liveness::Active || self.hello_state != HelloState::Established { + if self.liveness.is_dormant() || self.hello_state != HelloState::Established { return None; } let overdue = self.sync.take_overdue_reply(now); @@ -240,6 +251,9 @@ impl PeerState { self.unreachable_since = None; self.probe_backoff.reset(); + if self.liveness == Liveness::Unknown { + self.liveness = Liveness::Active; + } if let Liveness::Dormant { since } = self.liveness { self.incidents.dormant_ms += now.saturating_duration_since(since).as_millis() as u64; self.liveness = Liveness::Active; @@ -287,7 +301,7 @@ impl PeerState { /// Moves a silent peer to Dormant. Returns true on the transition. pub(crate) fn check_dormancy(&mut self, timing: &Timing, now: Instant) -> bool { - if self.liveness != Liveness::Active || self.quiet_for(now) < timing.dormant_after { + if self.liveness.is_dormant() || self.quiet_for(now) < timing.dormant_after { return false; } @@ -340,7 +354,7 @@ impl PeerState { } else { timing.keepalive + timing.keepalive / 2 }; - let due = self.liveness == Liveness::Active + let due = !self.liveness.is_dormant() && self.hello_state == HelloState::Established && now.saturating_duration_since(self.added_at) >= timing.keepalive && self.quiet_for(now) >= quiet_enough @@ -387,7 +401,7 @@ impl PeerState { // While an answer to our last sync message is due, a lost message is the reply // timeout's to resend; healing is for when neither side has anything outstanding. - if self.liveness != Liveness::Active + if self.liveness.is_dormant() || self.hello_state != HelloState::Established || self.sync.reply.is_some() { @@ -413,7 +427,7 @@ impl PeerState { /// Heard from recently. A returning node counts as alive from its first message, /// before the handshake finishes, so a swarm-wide forget can't make it flap. pub(crate) fn is_alive(&self, timing: &Timing, now: Instant) -> bool { - self.liveness == Liveness::Active + !self.liveness.is_dormant() && self .last_seen .is_some_and(|t| now.saturating_duration_since(t) < timing.dormant_after) @@ -424,6 +438,19 @@ impl PeerState { mod tests { use super::*; + /// Unknown until heard from; silent long enough, dormant like an active peer. + #[test] + fn unknown_until_heard_from() { + let timing = Timing::from_keepalive(Duration::from_secs(60), Duration::from_secs(300)); + let mut heard = PeerState::new(true, &timing); + assert_eq!(heard.liveness, Liveness::Unknown); + assert!(!heard.mark_seen(heard.added_at), "no re-handshake: none yet"); + assert_eq!(heard.liveness, Liveness::Active); + let mut silent = PeerState::new(true, &timing); + assert!(silent.check_dormancy(&timing, silent.added_at + timing.dormant_after)); + assert!(silent.liveness.is_dormant()); + } + /// The two sides of a pair take turns: each sends every other interval. #[test] fn keepalives_take_turns() { diff --git a/src/swarm/peer/handshake.rs b/src/swarm/peer/handshake.rs index 1183db1..fbebfc6 100644 --- a/src/swarm/peer/handshake.rs +++ b/src/swarm/peer/handshake.rs @@ -9,7 +9,7 @@ use tokio::sync::Mutex; use tokio::time::Instant; use tracing::{debug, info, instrument, trace, warn}; -use crate::state::{AppState, HelloState, Liveness, PeerState, Standing}; +use crate::state::{AppState, HelloState, PeerState, Standing}; use crate::swarm::sync::SyncEvent; use crate::swarm::topology; use crate::transport::{self, ParsedMessage}; @@ -199,7 +199,7 @@ pub(super) async fn send_hello_reply( let mut peers = Vec::new(); for (known, peer_state_mutex) in candidates { let peer_state = peer_state_mutex.lock().await; - if peer_state.liveness == Liveness::Active + if !peer_state.liveness.is_dormant() && peer_state.hello_state == HelloState::Established && topology::is_neighbour(state, &known) { diff --git a/src/swarm/peer/keepalive.rs b/src/swarm/peer/keepalive.rs index c8a93eb..ea7ade2 100644 --- a/src/swarm/peer/keepalive.rs +++ b/src/swarm/peer/keepalive.rs @@ -16,7 +16,7 @@ use tokio::time::Instant; use tracing::{debug, info, warn}; use super::handshake::ensure_handshake; -use crate::state::{AppState, Liveness, Standing}; +use crate::state::{AppState, Standing}; use crate::swarm::sync::SyncEvent; use crate::swarm::topology; use crate::transport::{self, ParsedMessage}; @@ -139,7 +139,7 @@ pub(crate) async fn local_status(state: &AppState, peer: &str) -> transport::Sta let syncs_with_you = match state.get_peer_state(peer).await { Some(peer_state) => { let peer_state = peer_state.lock().await; - peer_state.liveness == Liveness::Active + !peer_state.liveness.is_dormant() && matches!(peer_state.standing, Standing::Member { .. }) && peer_state.epoch == Some(epoch) } |
