diff options
Diffstat (limited to 'src/api')
| -rw-r--r-- | src/api/stats.rs | 41 | ||||
| -rw-r--r-- | src/api/stats/prometheus.rs | 46 |
2 files changed, 85 insertions, 2 deletions
diff --git a/src/api/stats.rs b/src/api/stats.rs index fa11b45..5d32d82 100644 --- a/src/api/stats.rs +++ b/src/api/stats.rs @@ -28,6 +28,7 @@ pub(crate) struct StatsResponse { epoch: EpochInfo, document: DocumentStats, catch_up: CatchUpStats, + remakes: RemakeStats, topology: TopologyStats, stability: StabilityStats, /// HELLO_REPLYs to an earlier run of ours, ignored. @@ -102,6 +103,25 @@ struct CatchUpStats { sync_bytes: Option<u64>, } +/// Remakes and their aftermath here (`remake::Summary`). +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct RemakeStats { + /// Remakes of the local epoch not decided yet (expired ones included). + undecided: usize, + /// Committed remakes of the local epoch: 1 while switching; more is a double commit. + committed: usize, + /// Syncing a newer epoch from scratch. + rejoining: bool, + /// Attempts at catching up with a newer epoch (0 when not catching up). + catch_up_attempts: u32, + /// Quarantined change sets (`GET /admin/quarantine`), and the oldest one's age. + quarantined: usize, + oldest_quarantine_secs: Option<u64>, + /// Old epochs' documents kept in memory. + archived_epochs: usize, +} + #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct TopologyStats { @@ -150,11 +170,17 @@ pub(crate) async fn get_stats(State(state): State<AppState>, headers: HeaderMap) async fn collect(state: &AppState) -> StatsResponse { let node = state.stats.snapshot(); - let (heads, tier) = { + let (heads, tier, remakes) = { let replica = state.doc.read().await; let heads = replica.heads().len(); - (heads, members::tier(&replica.doc, state.local_id())) + let remakes = crate::remake::summary(&state.remake, &replica.doc, replica.epoch); + ( + heads, + members::tier(&replica.doc, state.local_id()), + remakes, + ) }; + let now_ms = state.clock.now_millis(); let saved = state.stats.saved_size(); let (connected_fraction, observed) = stability::measured(state); @@ -183,6 +209,17 @@ async fn collect(state: &AppState) -> StatsResponse { done_after_secs: node.caught_up.map(|c| c.after.as_secs_f64()), sync_bytes: node.caught_up.map(|c| c.sync_bytes), }, + remakes: RemakeStats { + undecided: remakes.undecided, + committed: remakes.committed, + rejoining: remakes.rejoining, + catch_up_attempts: remakes.catch_up_attempts, + quarantined: remakes.quarantined, + oldest_quarantine_secs: remakes + .oldest_quarantine + .map(|created| now_ms.ms_since(created) / 1000), + archived_epochs: remakes.archived_epochs, + }, topology: TopologyStats { k: state.neighbours_k, view: topology::summary(state).map(|s| TopologyView { diff --git a/src/api/stats/prometheus.rs b/src/api/stats/prometheus.rs index 3709ea4..6fd37a0 100644 --- a/src/api/stats/prometheus.rs +++ b/src/api/stats/prometheus.rs @@ -257,6 +257,52 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String { ); } + let remakes = &stats.remakes; + out.single( + "remakes_undecided", + "gauge", + "Remakes of the local epoch not decided yet (expired ones included).", + remakes.undecided as f64, + ); + out.single( + "remakes_committed", + "gauge", + "Committed remakes of the local epoch: 1 while switching; more is a double commit.", + remakes.committed as f64, + ); + out.single( + "rejoining", + "gauge", + "1 while syncing a newer epoch from scratch.", + flag(remakes.rejoining), + ); + out.single( + "catch_up_attempts", + "gauge", + "Attempts at catching up with a newer epoch (0 when not catching up).", + remakes.catch_up_attempts as f64, + ); + out.single( + "quarantined", + "gauge", + "Quarantined change sets (in memory only).", + remakes.quarantined as f64, + ); + if let Some(secs) = remakes.oldest_quarantine_secs { + out.single( + "oldest_quarantine_seconds", + "gauge", + "Age of the oldest quarantined change set.", + secs as f64, + ); + } + out.single( + "archived_epochs", + "gauge", + "Old epochs' documents kept in memory.", + remakes.archived_epochs as f64, + ); + let topology = &stats.topology; out.single( "neighbours_target", |
