add document lock stats

2 files changed, 92 insertions, 0 deletions

diff --git a/src/api/stats.rs b/src/api/stats.rs
index 45d91da..07521ab 100644
--- a/src/api/stats.rs
+++ b/src/api/stats.rs
@@ -96,6 +96,9 @@ struct DocumentStats {
     saved_bytes_secs_ago: Option<u64>,
     /// How long the last measurement took (copy and save), in milliseconds.
     saved_bytes_took_ms: Option<u64>,
+    /// How long takers of the document's lock waited for it, since start: the node's
+    /// work on the one document queues there.
+    lock_wait: crate::document::lock::LockWaits,
 }
 
 #[derive(Debug, Serialize)]
@@ -245,6 +248,7 @@ async fn collect(state: &AppState) -> StatsResponse {
             saved_bytes: saved.map(|s| s.bytes),
             saved_bytes_secs_ago: saved.map(|s| s.at.elapsed().as_secs()),
             saved_bytes_took_ms: saved.map(|s| s.took.as_millis() as u64),
+            lock_wait: state.doc.waits(),
         },
         catch_up: CatchUpStats {
             sources: node.catch_up_sources,
diff --git a/src/api/stats/prometheus.rs b/src/api/stats/prometheus.rs
index 8e6fde7..9cbeff4 100644
--- a/src/api/stats/prometheus.rs
+++ b/src/api/stats/prometheus.rs
@@ -420,6 +420,94 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String {
         );
     }
 
+    out.family(
+        "document_lock_wait_seconds",
+        "histogram",
+        "Time waited for the document's lock, by mode (read, write).",
+    );
+    let lock_wait = &document.lock_wait;
+    for (mode, waits) in [("read", &lock_wait.read), ("write", &lock_wait.write)] {
+        let mut cumulative = 0;
+        for (le_us, count) in waits.buckets() {
+            cumulative += count;
+            let le = match le_us {
+                Some(us) => (us as f64 / 1e6).to_string(),
+                None => "+Inf".to_owned(),
+            };
+            out.sample(
+                "document_lock_wait_seconds_bucket",
+                &[("mode", mode), ("le", &le)],
+                cumulative as f64,
+            );
+        }
+        out.sample(
+            "document_lock_wait_seconds_sum",
+            &[("mode", mode)],
+            waits.sum_us as f64 / 1e6,
+        );
+        out.sample(
+            "document_lock_wait_seconds_count",
+            &[("mode", mode)],
+            waits.count as f64,
+        );
+    }
+    out.family(
+        "document_lock_wait_max_seconds",
+        "gauge",
+        "Longest wait for the document's lock since start, by mode.",
+    );
+    for (mode, waits) in [("read", &lock_wait.read), ("write", &lock_wait.write)] {
+        out.sample(
+            "document_lock_wait_max_seconds",
+            &[("mode", mode)],
+            waits.max_us as f64 / 1e6,
+        );
+    }
+
+    out.family(
+        "document_lock_wait_seconds",
+        "histogram",
+        "Time waited for the document's lock, by mode (read, write).",
+    );
+    let lock_wait = &document.lock_wait;
+    for (mode, waits) in [("read", &lock_wait.read), ("write", &lock_wait.write)] {
+        let mut cumulative = 0;
+        for (le_us, count) in waits.buckets() {
+            cumulative += count;
+            let le = match le_us {
+                Some(us) => (us as f64 / 1e6).to_string(),
+                None => "+Inf".to_owned(),
+            };
+            out.sample(
+                "document_lock_wait_seconds_bucket",
+                &[("mode", mode), ("le", &le)],
+                cumulative as f64,
+            );
+        }
+        out.sample(
+            "document_lock_wait_seconds_sum",
+            &[("mode", mode)],
+            waits.sum_us as f64 / 1e6,
+        );
+        out.sample(
+            "document_lock_wait_seconds_count",
+            &[("mode", mode)],
+            waits.count as f64,
+        );
+    }
+    out.family(
+        "document_lock_wait_max_seconds",
+        "gauge",
+        "Longest wait for the document's lock since start, by mode.",
+    );
+    for (mode, waits) in [("read", &lock_wait.read), ("write", &lock_wait.write)] {
+        out.sample(
+            "document_lock_wait_max_seconds",
+            &[("mode", mode)],
+            waits.max_us as f64 / 1e6,
+        );
+    }
+
     let catch_up = &stats.catch_up;
     out.single(
         "catch_up_sources_total",