add remake stats

4 files changed, 132 insertions, 2 deletions

diff --git a/src/api/stats.rs b/src/api/stats.rs
index fa11b45..5d32d82 100644
--- a/src/api/stats.rs
+++ b/src/api/stats.rs
@@ -28,6 +28,7 @@ pub(crate) struct StatsResponse {
     epoch: EpochInfo,
     document: DocumentStats,
     catch_up: CatchUpStats,
+    remakes: RemakeStats,
     topology: TopologyStats,
     stability: StabilityStats,
     /// HELLO_REPLYs to an earlier run of ours, ignored.
@@ -102,6 +103,25 @@ struct CatchUpStats {
     sync_bytes: Option<u64>,
 }
 
+/// Remakes and their aftermath here (`remake::Summary`).
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct RemakeStats {
+    /// Remakes of the local epoch not decided yet (expired ones included).
+    undecided: usize,
+    /// Committed remakes of the local epoch: 1 while switching; more is a double commit.
+    committed: usize,
+    /// Syncing a newer epoch from scratch.
+    rejoining: bool,
+    /// Attempts at catching up with a newer epoch (0 when not catching up).
+    catch_up_attempts: u32,
+    /// Quarantined change sets (`GET /admin/quarantine`), and the oldest one's age.
+    quarantined: usize,
+    oldest_quarantine_secs: Option<u64>,
+    /// Old epochs' documents kept in memory.
+    archived_epochs: usize,
+}
+
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct TopologyStats {
@@ -150,11 +170,17 @@ pub(crate) async fn get_stats(State(state): State<AppState>, headers: HeaderMap)
 async fn collect(state: &AppState) -> StatsResponse {
     let node = state.stats.snapshot();
 
-    let (heads, tier) = {
+    let (heads, tier, remakes) = {
         let replica = state.doc.read().await;
         let heads = replica.heads().len();
-        (heads, members::tier(&replica.doc, state.local_id()))
+        let remakes = crate::remake::summary(&state.remake, &replica.doc, replica.epoch);
+        (
+            heads,
+            members::tier(&replica.doc, state.local_id()),
+            remakes,
+        )
     };
+    let now_ms = state.clock.now_millis();
     let saved = state.stats.saved_size();
 
     let (connected_fraction, observed) = stability::measured(state);
@@ -183,6 +209,17 @@ async fn collect(state: &AppState) -> StatsResponse {
             done_after_secs: node.caught_up.map(|c| c.after.as_secs_f64()),
             sync_bytes: node.caught_up.map(|c| c.sync_bytes),
         },
+        remakes: RemakeStats {
+            undecided: remakes.undecided,
+            committed: remakes.committed,
+            rejoining: remakes.rejoining,
+            catch_up_attempts: remakes.catch_up_attempts,
+            quarantined: remakes.quarantined,
+            oldest_quarantine_secs: remakes
+                .oldest_quarantine
+                .map(|created| now_ms.ms_since(created) / 1000),
+            archived_epochs: remakes.archived_epochs,
+        },
         topology: TopologyStats {
             k: state.neighbours_k,
             view: topology::summary(state).map(|s| TopologyView {
diff --git a/src/api/stats/prometheus.rs b/src/api/stats/prometheus.rs
index 3709ea4..6fd37a0 100644
--- a/src/api/stats/prometheus.rs
+++ b/src/api/stats/prometheus.rs
@@ -257,6 +257,52 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String {
         );
     }
 
+    let remakes = &stats.remakes;
+    out.single(
+        "remakes_undecided",
+        "gauge",
+        "Remakes of the local epoch not decided yet (expired ones included).",
+        remakes.undecided as f64,
+    );
+    out.single(
+        "remakes_committed",
+        "gauge",
+        "Committed remakes of the local epoch: 1 while switching; more is a double commit.",
+        remakes.committed as f64,
+    );
+    out.single(
+        "rejoining",
+        "gauge",
+        "1 while syncing a newer epoch from scratch.",
+        flag(remakes.rejoining),
+    );
+    out.single(
+        "catch_up_attempts",
+        "gauge",
+        "Attempts at catching up with a newer epoch (0 when not catching up).",
+        remakes.catch_up_attempts as f64,
+    );
+    out.single(
+        "quarantined",
+        "gauge",
+        "Quarantined change sets (in memory only).",
+        remakes.quarantined as f64,
+    );
+    if let Some(secs) = remakes.oldest_quarantine_secs {
+        out.single(
+            "oldest_quarantine_seconds",
+            "gauge",
+            "Age of the oldest quarantined change set.",
+            secs as f64,
+        );
+    }
+    out.single(
+        "archived_epochs",
+        "gauge",
+        "Old epochs' documents kept in memory.",
+        remakes.archived_epochs as f64,
+    );
+
     let topology = &stats.topology;
     out.single(
         "neighbours_target",
diff --git a/src/remake/local.rs b/src/remake/local.rs
index fd7c567..60f7184 100644
--- a/src/remake/local.rs
+++ b/src/remake/local.rs
@@ -379,6 +379,11 @@ impl RemakeState {
         self.lock().quarantine.push(quarantined);
     }
 
+    /// How many old epochs are archived.
+    pub(crate) fn archived_count(&self) -> usize {
+        self.lock().archive.len()
+    }
+
     pub(crate) fn archived_epochs(&self) -> Vec<ArchivedEpoch> {
         self.lock()
             .archive
diff --git a/src/remake/mod.rs b/src/remake/mod.rs
index ad027a7..bd04de9 100644
--- a/src/remake/mod.rs
+++ b/src/remake/mod.rs
@@ -51,6 +51,48 @@ use tokio::sync::broadcast;
 use crate::document::replica::Epoch;
 use crate::swarm::sync::ChangeSet;
 
+/// Remakes and their aftermath on this node, in numbers (`/admin/stats`).
+#[derive(Debug, Clone, Copy, Default)]
+pub(crate) struct Summary {
+    /// Remakes of the local epoch not decided yet (expired ones included).
+    pub(crate) undecided: usize,
+    /// Committed remakes of the local epoch: 1 while switching to it; more is a double
+    /// commit, on which no node switches by itself.
+    pub(crate) committed: usize,
+    pub(crate) rejoining: bool,
+    /// Attempts at catching up with a newer epoch (0 when not catching up).
+    pub(crate) catch_up_attempts: u32,
+    pub(crate) quarantined: usize,
+    /// When the oldest quarantine entry was made.
+    pub(crate) oldest_quarantine: Option<crate::clock::UnixMillis>,
+    pub(crate) archived_epochs: usize,
+}
+
+/// The `Summary` of `remake` with the records of `doc`, whose epoch is `epoch`.
+pub(crate) fn summary(remake: &RemakeState, doc: &AutoCommit, epoch: Epoch) -> Summary {
+    let local = records::EpochRecord::from(epoch);
+    let mut summary = Summary {
+        rejoining: remake.is_rejoining(),
+        catch_up_attempts: remake.catch_up().map_or(0, |catch_up| catch_up.attempts),
+        archived_epochs: remake.archived_count(),
+        ..Summary::default()
+    };
+    for (_, record) in records::load_all(doc) {
+        if record.proposal.from_epoch != local {
+            continue;
+        }
+        if record.decision.is_none() {
+            summary.undecided += 1;
+        } else if record.committed_epoch().is_some() {
+            summary.committed += 1;
+        }
+    }
+    let created = remake.list_quarantine(|quarantined| quarantined.created_at);
+    summary.quarantined = created.len();
+    summary.oldest_quarantine = created.into_iter().min();
+    summary
+}
+
 /// What remake needs from the node. Implemented by `AppState`.
 ///
 /// Used generically (`H: Host`), never as `dyn Host`, so the generic methods are fine.