add files module to consistently track stats

2 files changed, 49 insertions, 34 deletions

diff --git a/src/api/stats.rs b/src/api/stats.rs
index 6c5e172..8899421 100644
--- a/src/api/stats.rs
+++ b/src/api/stats.rs
@@ -9,6 +9,8 @@ use axum::{
     http::{HeaderMap, header},
     response::{IntoResponse, Response},
 };
+use std::collections::BTreeMap;
+
 use serde::Serialize;
 
 use super::peers::EpochInfo;
@@ -35,11 +37,11 @@ pub(crate) struct StatsResponse {
     /// The peers listed in /peers together (a forgotten peer's are gone).
     incidents: Incidents,
     propagation: PropagationStats,
-    disk: DiskStats,
+    /// This node's own files (`files`), by kind.
+    disk: BTreeMap<&'static str, DiskStats>,
 }
 
-/// Files this node reads and writes for itself (the state file, a backup to recover
-/// from), since it started.
+/// Reads and writes of one kind of file, since the node started.
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct DiskStats {
@@ -47,6 +49,7 @@ struct DiskStats {
     read_bytes: u64,
     writes: u64,
     written_bytes: u64,
+    errors: u64,
 }
 
 /// How long changes written elsewhere took from their commit there until applied here,
@@ -207,11 +210,20 @@ async fn collect(state: &AppState) -> StatsResponse {
                 .map(|(le_ms, count)| PropagationBucket { le_ms, count })
                 .collect(),
         },
-        disk: DiskStats {
-            reads: node.disk.reads,
-            read_bytes: node.disk.read_bytes,
-            writes: node.disk.writes,
-            written_bytes: node.disk.written_bytes,
-        },
+        disk: state
+            .files
+            .counts()
+            .into_iter()
+            .map(|(kind, counts)| {
+                let stats = DiskStats {
+                    reads: counts.reads,
+                    read_bytes: counts.read_bytes,
+                    writes: counts.writes,
+                    written_bytes: counts.written_bytes,
+                    errors: counts.errors,
+                };
+                (kind.as_str(), stats)
+            })
+            .collect(),
     }
 }
diff --git a/src/api/stats/prometheus.rs b/src/api/stats/prometheus.rs
index ea97bd2..e267042 100644
--- a/src/api/stats/prometheus.rs
+++ b/src/api/stats/prometheus.rs
@@ -297,31 +297,34 @@ pub(super) async fn render(state: &AppState, stats: &StatsResponse) -> String {
         stats.stale_hello_replies as f64,
     );
 
-    let disk = &stats.disk;
-    out.single(
-        "disk_reads_total",
-        "counter",
-        "Files this node read for itself (state file, backup).",
-        disk.reads as f64,
-    );
-    out.single(
-        "disk_read_bytes_total",
-        "counter",
-        "Bytes of those files.",
-        disk.read_bytes as f64,
-    );
-    out.single(
-        "disk_writes_total",
-        "counter",
-        "Files this node wrote for itself (state file).",
-        disk.writes as f64,
-    );
-    out.single(
-        "disk_written_bytes_total",
-        "counter",
-        "Bytes of those files.",
-        disk.written_bytes as f64,
-    );
+    type Field = fn(&super::DiskStats) -> u64;
+    let disk: [(&str, &str, Field); 5] = [
+        ("disk_reads_total", "Reads of this node's own files.", |c| {
+            c.reads
+        }),
+        ("disk_read_bytes_total", "Bytes read from them.", |c| {
+            c.read_bytes
+        }),
+        (
+            "disk_writes_total",
+            "Writes of this node's own files.",
+            |c| c.writes,
+        ),
+        ("disk_written_bytes_total", "Bytes written to them.", |c| {
+            c.written_bytes
+        }),
+        (
+            "disk_errors_total",
+            "Failed reads and writes of them.",
+            |c| c.errors,
+        ),
+    ];
+    for (name, help, field) in disk {
+        out.family(name, "counter", help);
+        for (kind, counts) in &stats.disk {
+            out.sample(name, &[("file", *kind)], field(counts) as f64);
+        }
+    }
 
     let propagation = &stats.propagation;
     out.family(