add peer clock skew warning

2 files changed, 37 insertions, 1 deletions

diff --git a/src/api/stats.rs b/src/api/stats.rs
index 13610dd..d274fd9 100644
--- a/src/api/stats.rs
+++ b/src/api/stats.rs
@@ -13,9 +13,10 @@ use std::collections::BTreeMap;
 
 use serde::Serialize;
 
-use super::peers::{PeerListResponse, peer_list};
+use super::peers::{PeerListResponse, PeerLivenessStatus, peer_list};
 use crate::document::replica::EpochRecord;
 use crate::state::{AppState, Incidents};
+use crate::swarm::skew::{self, Skew};
 use crate::swarm::{members, stability, topology};
 use crate::transport::stats::TransportStats;
 use crate::transport::traffic::PeerTraffic;
@@ -44,6 +45,8 @@ pub(crate) struct StatsResponse {
     /// The peers listed in /peers together (a forgotten peer's are gone).
     incidents: Incidents,
     propagation: PropagationStats,
+    /// This node's clock against its active peers' (`swarm::skew`).
+    clock: Skew,
     /// This node's own files (`files`), by kind.
     disk: BTreeMap<&'static str, DiskStats>,
     /// How the transports find and reach peers (`Transport::stats`).
@@ -271,6 +274,16 @@ pub(crate) async fn collect(state: &AppState, list: &PeerListResponse) -> StatsR
 
     let (connected_fraction, observed) = stability::measured(state);
 
+    let clock = skew::of(
+        &list
+            .peers
+            .values()
+            .filter(|peer| !matches!(peer.status, PeerLivenessStatus::Dormant))
+            .filter_map(|peer| peer.clock)
+            .collect::<Vec<_>>(),
+        state.clock_skew_warn,
+    );
+
     let mut incidents = Incidents::default();
     let mut last_from_neighbour_secs: Option<u64> = None;
     let mut diverged_peers = 0;
@@ -347,6 +360,7 @@ pub(crate) async fn collect(state: &AppState, list: &PeerListResponse) -> StatsR
         stale_hello_replies: node.stale_hello_replies,
         traffic: state.traffic.total(),
         incidents,
+        clock,
         propagation: PropagationStats {
             count: node.propagation.count,
             sum_ms: node.propagation.sum_ms,
diff --git a/src/api/stats/prometheus.rs b/src/api/stats/prometheus.rs
index 2f33bef..6774e74 100644
--- a/src/api/stats/prometheus.rs
+++ b/src/api/stats/prometheus.rs
@@ -763,6 +763,28 @@ pub(super) fn render(stats: &StatsResponse, list: &PeerListResponse) -> String {
         propagation.ahead as f64,
     );
 
+    let clock = &stats.clock;
+    out.single(
+        "clock_peers_compared",
+        "gauge",
+        "Active peers whose clocks were compared with ours (from their PONGs).",
+        clock.peers as f64,
+    );
+    if let Some(offset_ms) = clock.offset_ms {
+        out.single(
+            "clock_offset_seconds",
+            "gauge",
+            "How far this node's clock is ahead of its peers' (behind: negative), the median.",
+            offset_ms as f64 / 1000.0,
+        );
+    }
+    out.single(
+        "clock_off",
+        "gauge",
+        "1 while this node's clock differs from most of its peers' by more than --clock-skew-warn.",
+        flag(clock.off),
+    );
+
     // Node totals, including peers no longer listed.
     let totals = traffic_series(&stats.traffic);
     out.family(