bench repeat

1 files changed, 81 insertions, 7 deletions

diff --git a/src/simtest/bench.rs b/src/simtest/bench.rs
index dd0d4eb..2530015 100644
--- a/src/simtest/bench.rs
+++ b/src/simtest/bench.rs
@@ -51,6 +51,27 @@ const ONLY: &str = "BENCH_SCENARIO";
 /// cores, but they share them, so delays (and what depends on timing) may come out
 /// higher than alone; message counts barely.
 const PARALLEL: &str = "BENCH_PARALLEL";
+/// How many times each scenario runs, one run after another (`BENCH_REPEAT=3`; once if
+/// unset), with a summary at the end: runs differ (timing decides much, PlumTree's
+/// trees most), so one run can't tell a small change from chance.
+const REPEAT: &str = "BENCH_REPEAT";
+
+/// How many times each scenario runs (`REPEAT`).
+fn repeat() -> usize {
+    std::env::var(REPEAT)
+        .ok()
+        .and_then(|value| value.parse().ok())
+        .filter(|times| *times > 0)
+        .unwrap_or(1)
+}
+
+/// What one workload of a run came to, for the summary over repeated runs: the mean
+/// arrival delay in ms, and the messages per write.
+#[derive(Clone, Copy)]
+struct Outcome {
+    mean_ms: u64,
+    messages_per_write: f64,
+}
 const SETTLE: Duration = Duration::from_secs(180);
 
 /// How the swarm is made.
@@ -297,18 +318,64 @@ async fn bench_propagation() {
         // All at once, each reported when all are done, in order.
         let running: Vec<_> = chosen
             .into_iter()
-            .map(|(seed, scenario)| tokio::spawn(run(seed, scenario)))
+            .map(|(seed, scenario)| tokio::spawn(runs(seed, scenario)))
             .collect();
         for report in running {
             print!("{}", report.await.expect("scenario panicked"));
         }
     } else {
         for (seed, scenario) in chosen {
-            print!("{}", run(seed, scenario).await);
+            print!("{}", runs(seed, scenario).await);
         }
     }
 }
 
+/// The scenario run `repeat()` times, one after another, each run's report in turn;
+/// after several, a summary: each workload's mean arrival delay and messages per write,
+/// run by run, and their medians.
+async fn runs(seed: u64, scenario: &'static Scenario) -> String {
+    let times = repeat();
+    let mut out = String::new();
+    let mut outcomes: Vec<Vec<(&'static str, Outcome)>> = Vec::new();
+    for _ in 0..times {
+        let (report, outcome) = run(seed, scenario).await;
+        out.push_str(&report);
+        outcomes.push(outcome);
+    }
+    if times < 2 {
+        return out;
+    }
+    let _ = writeln!(out, "\n{}: {times} runs", scenario.name);
+    let workloads: Vec<&str> = outcomes[0].iter().map(|(workload, _)| *workload).collect();
+    for workload in workloads {
+        let of: Vec<Outcome> = outcomes
+            .iter()
+            .filter_map(|run| run.iter().find(|(w, _)| *w == workload).map(|(_, o)| *o))
+            .collect();
+        let mut means: Vec<u64> = of.iter().map(|o| o.mean_ms).collect();
+        let mut messages: Vec<f64> = of.iter().map(|o| o.messages_per_write).collect();
+        let listed_means = means
+            .iter()
+            .map(u64::to_string)
+            .collect::<Vec<_>>()
+            .join(", ");
+        let listed_messages = messages
+            .iter()
+            .map(|m| format!("{m:.0}"))
+            .collect::<Vec<_>>()
+            .join(", ");
+        means.sort_unstable();
+        messages.sort_by(f64::total_cmp);
+        let _ = writeln!(
+            out,
+            "  {workload}: mean arrival {listed_means} ms (median {}); messages per write {listed_messages} (median {:.0})",
+            means[means.len() / 2],
+            messages[messages.len() / 2],
+        );
+    }
+    out
+}
+
 /// What all nodes have sent, counted, and their relay counters and propagation delays
 /// summed: at one moment.
 #[derive(Clone)]
@@ -471,7 +538,8 @@ fn counters_mut(by: &mut ByKind) -> [&mut Counter; 11] {
 }
 
 /// Runs `scenario`; returns its report.
-async fn run(seed: u64, scenario: &'static Scenario) -> String {
+async fn run(seed: u64, scenario: &'static Scenario) -> (String, Vec<(&'static str, Outcome)>) {
+    let mut outcomes = Vec::new();
     let mut out = String::new();
     let sim = Sim::with_identity(seed);
     let ids: Vec<String> = (0..scenario.nodes).map(|i| format!("n{i:03}")).collect();
@@ -606,13 +674,14 @@ async fn run(seed: u64, scenario: &'static Scenario) -> String {
         let all: Vec<&AppState> = nodes.iter().collect();
         sim.wait_converged(&all, SETTLE).await;
         let converged = written.elapsed();
-        report(
+        let outcome = report(
             &mut out,
             workload,
             quiet.clone(),
             (start, Totals::of(&nodes).await),
             converged,
         );
+        outcomes.push((workload, outcome));
         trees(&mut out, &nodes, &sim.net).await;
         sent_kept(&mut out, &nodes).await;
         eager_links(&mut out, &nodes, &sim.net);
@@ -620,7 +689,7 @@ async fn run(seed: u64, scenario: &'static Scenario) -> String {
             slowest(&mut out, &nodes, &nodes[writer], &sim.net).await;
         }
     }
-    out
+    (out, outcomes)
 }
 
 /// Every node's own writer's tree (`swarm::tree::own_shape`, `/admin/stats`), as it
@@ -1050,7 +1119,7 @@ fn report(
     quiet: (Totals, Totals),
     during: (Totals, Totals),
     converged: Duration,
-) {
+) -> Outcome {
     let took = during.1.at - during.0.at;
     let scale = took.as_secs_f64() / (quiet.1.at - quiet.0.at).as_secs_f64();
     let writes = writes() as f64;
@@ -1234,11 +1303,12 @@ fn report(
         super::soak::percentile_bound(&propagation, quantile)
             .map_or("more than an hour".to_owned(), |ms| format!("{ms} ms"))
     };
+    let mean_ms = propagation.sum_ms / propagation.count.max(1);
     let _ = writeln!(
         out,
         "    propagation: {} arrivals, mean {} ms, half within {}, 99% within {}",
         propagation.count,
-        propagation.sum_ms / propagation.count.max(1),
+        mean_ms,
         within(0.5),
         within(0.99),
     );
@@ -1332,6 +1402,10 @@ fn report(
             mostly_late.len(),
         );
     }
+    Outcome {
+        mean_ms,
+        messages_per_write: all_messages / writes,
+    }
 }
 
 /// Values written in `bench_bridge`, unless `BENCH_BRIDGE_WRITES` says otherwise.