add per peer incident stats

4 files changed, 52 insertions, 4 deletions

diff --git a/src/api/mod.rs b/src/api/mod.rs
index d7f14b3..95d63aa 100644
--- a/src/api/mod.rs
+++ b/src/api/mod.rs
@@ -158,6 +158,7 @@ fn response_for_value(
             crate::transport::traffic::PeerTraffic,
             crate::transport::traffic::ByKind,
             crate::transport::traffic::Counter,
+            crate::state::Incidents,
         )
     ),
     tags(
diff --git a/src/api/peers.rs b/src/api/peers.rs
index 9ed11c5..5b7d576 100644
--- a/src/api/peers.rs
+++ b/src/api/peers.rs
@@ -16,7 +16,7 @@ use utoipa::ToSchema;
 use super::store_status;
 use crate::document::replica::Epoch;
 use crate::document::store;
-use crate::state::{AppState, HelloState, Liveness, PeerState, unix_millis};
+use crate::state::{AppState, HelloState, Incidents, Liveness, PeerState, unix_millis};
 use crate::swarm::peer::forgotten_path;
 use crate::transport::traffic::PeerTraffic;
 
@@ -69,6 +69,7 @@ pub(crate) struct PeerStatus {
     /// Messages and payload bytes exchanged with it, by kind, since we started (or
     /// since it was last listed).
     traffic: PeerTraffic,
+    incidents: Incidents,
 }
 
 #[derive(Debug, Serialize, ToSchema)]
@@ -147,6 +148,7 @@ pub(crate) async fn list_peers(
             neighbour,
             rtt_ms,
             traffic,
+            incidents: ps.incidents,
         });
     }
 
diff --git a/src/api/stats.rs b/src/api/stats.rs
index f4a368b..e426c7a 100644
--- a/src/api/stats.rs
+++ b/src/api/stats.rs
@@ -4,7 +4,7 @@ use axum::{Json, extract::State};
 use serde::Serialize;
 
 use super::peers::EpochInfo;
-use crate::state::AppState;
+use crate::state::{AppState, Incidents};
 use crate::swarm::{members, stability, topology};
 use crate::transport::traffic::PeerTraffic;
 
@@ -22,6 +22,8 @@ pub(crate) struct StatsResponse {
     stale_hello_replies: u64,
     /// All peers together, including ones no longer listed in /peers.
     traffic: PeerTraffic,
+    /// The peers listed in /peers together (a forgotten peer's are gone).
+    incidents: Incidents,
 }
 
 #[derive(Debug, Serialize)]
@@ -96,6 +98,12 @@ pub(crate) async fn get_stats(State(state): State<AppState>) -> Json<StatsRespon
 
     let (connected_fraction, observed) = stability::measured(&state);
 
+    let mut incidents = Incidents::default();
+    let peers: Vec<_> = state.peer_states.read().await.values().cloned().collect();
+    for peer_state in peers {
+        incidents.add(&peer_state.lock().await.incidents);
+    }
+
     Json(StatsResponse {
         uptime_secs: node.uptime.as_secs(),
         ready: state.is_ready().await,
@@ -130,5 +138,6 @@ pub(crate) async fn get_stats(State(state): State<AppState>) -> Json<StatsRespon
         },
         stale_hello_replies: node.stale_hello_replies,
         traffic: state.traffic.total(),
+        incidents,
     })
 }
diff --git a/src/state.rs b/src/state.rs
index aae33f5..dfabf8f 100644
--- a/src/state.rs
+++ b/src/state.rs
@@ -144,6 +144,34 @@ pub(crate) struct PeerState {
     /// dormant). Cleared by any message from it; ignored after `dormant_probe`, so the
     /// peer becomes a neighbour candidate again and gets tried once more.
     pub(crate) unreachable_since: Option<Instant>,
+
+    /// What went wrong with this peer so far (counted only, for /peers and /admin/stats).
+    pub(crate) incidents: Incidents,
+}
+
+/// Trouble with a peer, counted since it was first listed. Lost keepalives and refused
+/// sends are in the traffic counters (`transport::traffic`): every PONG answers a PING.
+#[derive(Debug, Default, Clone, Copy, serde::Serialize, utoipa::ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct Incidents {
+    /// Our sync messages left unanswered until the reply timeout (then sent again).
+    pub(crate) reply_timeouts: u64,
+    /// Re-handshakes because our documents differed while sync was idle.
+    pub(crate) heals: u64,
+    /// Times the peer went dormant.
+    pub(crate) dormancies: u64,
+    /// Time spent dormant, in milliseconds, up to its last return (the current
+    /// dormancy, if any, isn't included).
+    pub(crate) dormant_ms: u64,
+}
+
+impl Incidents {
+    pub(crate) fn add(&mut self, other: &Incidents) {
+        self.reply_timeouts += other.reply_timeouts;
+        self.heals += other.heals;
+        self.dormancies += other.dormancies;
+        self.dormant_ms += other.dormant_ms;
+    }
 }
 
 /// Unanswered HELLOs after which a peer counts as unreachable (1 + 2 + 4 s apart).
@@ -296,6 +324,7 @@ impl PeerState {
             reported_heads: None,
             hello_attempts: 0,
             unreachable_since: None,
+            incidents: Incidents::default(),
         }
     }
 
@@ -311,7 +340,11 @@ impl PeerState {
         if self.liveness != Liveness::Active || self.hello_state != HelloState::Established {
             return None;
         }
-        self.sync.take_overdue_reply(now)
+        let overdue = self.sync.take_overdue_reply(now);
+        if overdue.is_some() {
+            self.incidents.reply_timeouts += 1;
+        }
+        overdue
     }
 
     fn quiet_for(&self, now: Instant) -> Duration {
@@ -324,7 +357,8 @@ impl PeerState {
         self.hello_attempts = 0;
         self.unreachable_since = None;
 
-        if matches!(self.liveness, Liveness::Dormant { .. }) {
+        if let Liveness::Dormant { since } = self.liveness {
+            self.incidents.dormant_ms += now.saturating_duration_since(since).as_millis() as u64;
             self.liveness = Liveness::Active;
             // Force a fresh handshake; its reply resets sync on both sides.
             if self.hello_state != HelloState::Sending {
@@ -365,6 +399,7 @@ impl PeerState {
         }
 
         self.liveness = Liveness::Dormant { since: now };
+        self.incidents.dormancies += 1;
         self.unreachable_since.get_or_insert(now);
         self.reset_sync();
 
@@ -446,6 +481,7 @@ impl PeerState {
             return false;
         }
         self.diverged_since = None;
+        self.incidents.heals += 1;
 
         self.last_resync_at = Some(now);
         self.hello_state = HelloState::NotSent; // establish_peer resets sync on the reply