diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/api/mod.rs | 1 | ||||
| -rw-r--r-- | src/api/peers.rs | 4 | ||||
| -rw-r--r-- | src/api/stats.rs | 11 | ||||
| -rw-r--r-- | src/state.rs | 40 |
4 files changed, 52 insertions, 4 deletions
diff --git a/src/api/mod.rs b/src/api/mod.rs index d7f14b3..95d63aa 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -158,6 +158,7 @@ fn response_for_value( crate::transport::traffic::PeerTraffic, crate::transport::traffic::ByKind, crate::transport::traffic::Counter, + crate::state::Incidents, ) ), tags( diff --git a/src/api/peers.rs b/src/api/peers.rs index 9ed11c5..5b7d576 100644 --- a/src/api/peers.rs +++ b/src/api/peers.rs @@ -16,7 +16,7 @@ use utoipa::ToSchema; use super::store_status; use crate::document::replica::Epoch; use crate::document::store; -use crate::state::{AppState, HelloState, Liveness, PeerState, unix_millis}; +use crate::state::{AppState, HelloState, Incidents, Liveness, PeerState, unix_millis}; use crate::swarm::peer::forgotten_path; use crate::transport::traffic::PeerTraffic; @@ -69,6 +69,7 @@ pub(crate) struct PeerStatus { /// Messages and payload bytes exchanged with it, by kind, since we started (or /// since it was last listed). traffic: PeerTraffic, + incidents: Incidents, } #[derive(Debug, Serialize, ToSchema)] @@ -147,6 +148,7 @@ pub(crate) async fn list_peers( neighbour, rtt_ms, traffic, + incidents: ps.incidents, }); } diff --git a/src/api/stats.rs b/src/api/stats.rs index f4a368b..e426c7a 100644 --- a/src/api/stats.rs +++ b/src/api/stats.rs @@ -4,7 +4,7 @@ use axum::{Json, extract::State}; use serde::Serialize; use super::peers::EpochInfo; -use crate::state::AppState; +use crate::state::{AppState, Incidents}; use crate::swarm::{members, stability, topology}; use crate::transport::traffic::PeerTraffic; @@ -22,6 +22,8 @@ pub(crate) struct StatsResponse { stale_hello_replies: u64, /// All peers together, including ones no longer listed in /peers. traffic: PeerTraffic, + /// The peers listed in /peers together (a forgotten peer's are gone). + incidents: Incidents, } #[derive(Debug, Serialize)] @@ -96,6 +98,12 @@ pub(crate) async fn get_stats(State(state): State<AppState>) -> Json<StatsRespon let (connected_fraction, observed) = stability::measured(&state); + let mut incidents = Incidents::default(); + let peers: Vec<_> = state.peer_states.read().await.values().cloned().collect(); + for peer_state in peers { + incidents.add(&peer_state.lock().await.incidents); + } + Json(StatsResponse { uptime_secs: node.uptime.as_secs(), ready: state.is_ready().await, @@ -130,5 +138,6 @@ pub(crate) async fn get_stats(State(state): State<AppState>) -> Json<StatsRespon }, stale_hello_replies: node.stale_hello_replies, traffic: state.traffic.total(), + incidents, }) } diff --git a/src/state.rs b/src/state.rs index aae33f5..dfabf8f 100644 --- a/src/state.rs +++ b/src/state.rs @@ -144,6 +144,34 @@ pub(crate) struct PeerState { /// dormant). Cleared by any message from it; ignored after `dormant_probe`, so the /// peer becomes a neighbour candidate again and gets tried once more. pub(crate) unreachable_since: Option<Instant>, + + /// What went wrong with this peer so far (counted only, for /peers and /admin/stats). + pub(crate) incidents: Incidents, +} + +/// Trouble with a peer, counted since it was first listed. Lost keepalives and refused +/// sends are in the traffic counters (`transport::traffic`): every PONG answers a PING. +#[derive(Debug, Default, Clone, Copy, serde::Serialize, utoipa::ToSchema)] +#[serde(rename_all = "camelCase")] +pub(crate) struct Incidents { + /// Our sync messages left unanswered until the reply timeout (then sent again). + pub(crate) reply_timeouts: u64, + /// Re-handshakes because our documents differed while sync was idle. + pub(crate) heals: u64, + /// Times the peer went dormant. + pub(crate) dormancies: u64, + /// Time spent dormant, in milliseconds, up to its last return (the current + /// dormancy, if any, isn't included). + pub(crate) dormant_ms: u64, +} + +impl Incidents { + pub(crate) fn add(&mut self, other: &Incidents) { + self.reply_timeouts += other.reply_timeouts; + self.heals += other.heals; + self.dormancies += other.dormancies; + self.dormant_ms += other.dormant_ms; + } } /// Unanswered HELLOs after which a peer counts as unreachable (1 + 2 + 4 s apart). @@ -296,6 +324,7 @@ impl PeerState { reported_heads: None, hello_attempts: 0, unreachable_since: None, + incidents: Incidents::default(), } } @@ -311,7 +340,11 @@ impl PeerState { if self.liveness != Liveness::Active || self.hello_state != HelloState::Established { return None; } - self.sync.take_overdue_reply(now) + let overdue = self.sync.take_overdue_reply(now); + if overdue.is_some() { + self.incidents.reply_timeouts += 1; + } + overdue } fn quiet_for(&self, now: Instant) -> Duration { @@ -324,7 +357,8 @@ impl PeerState { self.hello_attempts = 0; self.unreachable_since = None; - if matches!(self.liveness, Liveness::Dormant { .. }) { + if let Liveness::Dormant { since } = self.liveness { + self.incidents.dormant_ms += now.saturating_duration_since(since).as_millis() as u64; self.liveness = Liveness::Active; // Force a fresh handshake; its reply resets sync on both sides. if self.hello_state != HelloState::Sending { @@ -365,6 +399,7 @@ impl PeerState { } self.liveness = Liveness::Dormant { since: now }; + self.incidents.dormancies += 1; self.unreachable_since.get_or_insert(now); self.reset_sync(); @@ -446,6 +481,7 @@ impl PeerState { return false; } self.diverged_since = None; + self.incidents.heals += 1; self.last_resync_at = Some(now); self.hello_state = HelloState::NotSent; // establish_peer resets sync on the reply |