simtest: benchmark incidents

1 files changed, 21 insertions, 6 deletions

diff --git a/src/simtest/bench.rs b/src/simtest/bench.rs
index 9ffb93c..2640381 100644
--- a/src/simtest/bench.rs
+++ b/src/simtest/bench.rs
@@ -19,7 +19,7 @@ use serde_json::json;
 use tokio::time::Instant;
 
 use super::{SYNC_REPLY_TIMEOUT, Sim, write};
-use crate::state::{AppState, Timing};
+use crate::state::{AppState, Incidents, Timing};
 use crate::swarm::bootstrap::Start;
 use crate::swarm::members;
 use crate::swarm::stats::{Propagation, Relay};
@@ -191,6 +191,8 @@ struct Totals {
     at: Instant,
     sent: ByKind,
     relay: Relay,
+    /// What went wrong with peers (`PeerState::incidents`), summed over every node's.
+    incidents: Incidents,
     propagation: Propagation,
     /// Per node, by name: its arrivals, their delays summed, and those later than
     /// `LATE_MS`.
@@ -201,11 +203,12 @@ struct Totals {
 const LATE_MS: u64 = 500;
 
 impl Totals {
-    fn of(nodes: &[AppState]) -> Self {
+    async fn of(nodes: &[AppState]) -> Self {
         let mut totals = Self {
             at: Instant::now(),
             sent: ByKind::default(),
             relay: Relay::default(),
+            incidents: Incidents::default(),
             propagation: Propagation::default(),
             by_node: Vec::new(),
         };
@@ -226,6 +229,9 @@ impl Totals {
             relay.pulls += stats.relay.pulls;
             relay.pulls_queued += stats.relay.pulls_queued;
             relay.tree_forwards += stats.relay.tree_forwards;
+            for (_, peer_state) in node.peers_snapshot().await {
+                totals.incidents.add(&peer_state.lock().await.incidents);
+            }
             let propagation = &mut totals.propagation;
             for (sum, count) in propagation
                 .buckets
@@ -339,10 +345,10 @@ async fn run(seed: u64, scenario: &'static Scenario) -> String {
     // fewer in a half of the quiet phase than in the one before (some go on for good:
     // alive stamps, registrations). Those two halves are the quiet phase.
     let settling = Instant::now();
-    let mut halves = vec![Totals::of(&nodes)];
+    let mut halves = vec![Totals::of(&nodes).await];
     loop {
         tokio::time::sleep(QUIET / 2).await;
-        halves.push(Totals::of(&nodes));
+        halves.push(Totals::of(&nodes).await);
         let count = |i: usize| halves[i + 1].sent.sync.messages - halves[i].sent.sync.messages;
         let n = halves.len();
         if (n >= 3 && count(n - 2) >= count(n - 3)) || settling.elapsed() > SETTLE {
@@ -366,7 +372,7 @@ async fn run(seed: u64, scenario: &'static Scenario) -> String {
     .into_iter()
     .enumerate()
     {
-        let start = Totals::of(&nodes);
+        let start = Totals::of(&nodes).await;
         for i in 0..WRITES {
             // The middle half, the node that goes away is gone (writing nothing).
             if let Some(away) = scenario.away {
@@ -396,7 +402,7 @@ async fn run(seed: u64, scenario: &'static Scenario) -> String {
             &mut out,
             workload,
             quiet.clone(),
-            (start, Totals::of(&nodes)),
+            (start, Totals::of(&nodes).await),
             converged,
         );
     }
@@ -469,6 +475,15 @@ fn report(
         (r1.pulls_queued - r0.pulls_queued) as f64 / writes,
         (r1.tree_forwards - r0.tree_forwards) as f64 / writes,
     );
+    let (i0, i1) = (during.0.incidents, during.1.incidents);
+    let _ = writeln!(
+        out,
+        "    incidents: {} reply timeouts ({:.2} per write), {} heals, {} dormancies",
+        i1.reply_timeouts - i0.reply_timeouts,
+        (i1.reply_timeouts - i0.reply_timeouts) as f64 / writes,
+        i1.heals - i0.heals,
+        i1.dormancies - i0.dormancies,
+    );
     let mut propagation = Propagation::default();
     let (p0, p1) = (during.0.propagation, during.1.propagation);
     for ((sum, a), b) in propagation