diff options
Diffstat (limited to 'src/api')
| -rw-r--r-- | src/api/stats.rs | 4 | ||||
| -rw-r--r-- | src/api/stats/prometheus.rs | 88 |
2 files changed, 92 insertions, 0 deletions
diff --git a/src/api/stats.rs b/src/api/stats.rs index 45d91da..07521ab 100644 --- a/src/api/stats.rs +++ b/src/api/stats.rs @@ -96,6 +96,9 @@ struct DocumentStats { saved_bytes_secs_ago: Option<u64>, /// How long the last measurement took (copy and save), in milliseconds. saved_bytes_took_ms: Option<u64>, + /// How long takers of the document's lock waited for it, since start: the node's + /// work on the one document queues there. + lock_wait: crate::document::lock::LockWaits, } #[derive(Debug, Serialize)] @@ -245,6 +248,7 @@ async fn collect(state: &AppState) -> StatsResponse { saved_bytes: saved.map(|s| s.bytes), saved_bytes_secs_ago: saved.map(|s| s.at.elapsed().as_secs()), saved_bytes_took_ms: saved.map(|s| s.took.as_millis() as u64), + lock_wait: state.doc.waits(), }, catch_up: CatchUpStats { sources: node.catch_up_sources, diff --git a/src/api/stats/prometheus.rs b/src/api/stats/prometheus.rs index 8e6fde7..9cbeff4 100644 --- a/src/api/stats/prometheus.rs +++ b/src/api/stats/prometheus.rs @@ -420,6 +420,94 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String { ); } + out.family( + "document_lock_wait_seconds", + "histogram", + "Time waited for the document's lock, by mode (read, write).", + ); + let lock_wait = &document.lock_wait; + for (mode, waits) in [("read", &lock_wait.read), ("write", &lock_wait.write)] { + let mut cumulative = 0; + for (le_us, count) in waits.buckets() { + cumulative += count; + let le = match le_us { + Some(us) => (us as f64 / 1e6).to_string(), + None => "+Inf".to_owned(), + }; + out.sample( + "document_lock_wait_seconds_bucket", + &[("mode", mode), ("le", &le)], + cumulative as f64, + ); + } + out.sample( + "document_lock_wait_seconds_sum", + &[("mode", mode)], + waits.sum_us as f64 / 1e6, + ); + out.sample( + "document_lock_wait_seconds_count", + &[("mode", mode)], + waits.count as f64, + ); + } + out.family( + "document_lock_wait_max_seconds", + "gauge", + "Longest wait for the document's lock since start, by mode.", + ); + for (mode, waits) in [("read", &lock_wait.read), ("write", &lock_wait.write)] { + out.sample( + "document_lock_wait_max_seconds", + &[("mode", mode)], + waits.max_us as f64 / 1e6, + ); + } + + out.family( + "document_lock_wait_seconds", + "histogram", + "Time waited for the document's lock, by mode (read, write).", + ); + let lock_wait = &document.lock_wait; + for (mode, waits) in [("read", &lock_wait.read), ("write", &lock_wait.write)] { + let mut cumulative = 0; + for (le_us, count) in waits.buckets() { + cumulative += count; + let le = match le_us { + Some(us) => (us as f64 / 1e6).to_string(), + None => "+Inf".to_owned(), + }; + out.sample( + "document_lock_wait_seconds_bucket", + &[("mode", mode), ("le", &le)], + cumulative as f64, + ); + } + out.sample( + "document_lock_wait_seconds_sum", + &[("mode", mode)], + waits.sum_us as f64 / 1e6, + ); + out.sample( + "document_lock_wait_seconds_count", + &[("mode", mode)], + waits.count as f64, + ); + } + out.family( + "document_lock_wait_max_seconds", + "gauge", + "Longest wait for the document's lock since start, by mode.", + ); + for (mode, waits) in [("read", &lock_wait.read), ("write", &lock_wait.write)] { + out.sample( + "document_lock_wait_max_seconds", + &[("mode", mode)], + waits.max_us as f64 / 1e6, + ); + } + let catch_up = &stats.catch_up; out.single( "catch_up_sources_total", |