announcement round counters

7 files changed, 53 insertions, 10 deletions

diff --git a/docs/measurements.md b/docs/measurements.md
index 9cdc265..d878ba7 100644
--- a/docs/measurements.md
+++ b/docs/measurements.md
@@ -81,6 +81,8 @@ or caught up over many small messages.
 | `sync.divergedPeers` | `diverged_peers` | Peers whose heads differ from ours with no sync message in between. |
 | `sync.largestReceivedBytes` | `largest_sync_received_bytes` | The largest sync message received since start. Larger transfers come as deltas (blobs of 16 chunks or more, [`peers.md`](peers.md)), so this stays below 16 × `--blob-chunk` from nodes of this version. |
 | `relay.announceRounds` | `announce_rounds_total` | Announcement rounds: our new heads to the neighbours after applying their changes (the quick round to those writers' trees leave out, or to all for changes no tree carries, and the slow round to the others; [`peers.md`](peers.md)). |
+| `relay.treelessRounds` | `treeless_rounds_total` | Of those, quick rounds to all neighbours: some change came that no writer's tree carries (pulled, a delta, from a node with no part in its tree). Many, with every node on the member list: trees miss often. |
+| `relay.slowRounds` | `slow_rounds_total` | Of those, slow rounds (2 s) after changes passed on along writers' trees: at most one per 2 s while changes keep coming. |
 | `relay.pulls` | `pulls_total` | Pulls of changes we lacked from a neighbour that announced them. |
 | `relay.pullsQueued` | `pulls_queued_total` | Announcements of heads we lacked that waited for the pull in progress. Many compared with `pulls`: changes arrive from several sides at once. |
 | `relay.treeForwards` | `tree_forwards_total` | Changes passed on along writers' trees ([`peers.md`](peers.md)): a sync message per child, at once, instead of an announcement and a pull. |
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index b73827c..b7cd3ab 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -1976,10 +1976,14 @@ components:
               type: integer
         relay:
           type: object
-          required: [announceRounds, pulls, pullsQueued, treeForwards, blobRequestsDropped]
+          required: [announceRounds, treelessRounds, slowRounds, pulls, pullsQueued, treeForwards, blobRequestsDropped]
           properties:
             announceRounds:
               type: integer
+            treelessRounds:
+              type: integer
+            slowRounds:
+              type: integer
             pulls:
               type: integer
             pullsQueued:
diff --git a/src/api/stats.rs b/src/api/stats.rs
index 96e8ad3..0bdb1b6 100644
--- a/src/api/stats.rs
+++ b/src/api/stats.rs
@@ -137,6 +137,10 @@ struct SyncStats {
 #[serde(rename_all = "camelCase")]
 struct RelayStats {
     announce_rounds: u64,
+    /// Of those, quick ones to all neighbours (some change had no writer's tree), and
+    /// slow ones (`swarm::relay`).
+    treeless_rounds: u64,
+    slow_rounds: u64,
     pulls: u64,
     /// Announcements of heads we lacked that waited for the pull in progress.
     pulls_queued: u64,
@@ -340,6 +344,8 @@ pub(crate) async fn collect(state: &AppState, list: &PeerListResponse) -> StatsR
         },
         relay: RelayStats {
             announce_rounds: node.relay.announce_rounds,
+            treeless_rounds: node.relay.treeless_rounds,
+            slow_rounds: node.relay.slow_rounds,
             pulls: node.relay.pulls,
             pulls_queued: node.relay.pulls_queued,
             tree_forwards: node.relay.tree_forwards,
diff --git a/src/api/stats/prometheus.rs b/src/api/stats/prometheus.rs
index 03643ee..5b1fc8c 100644
--- a/src/api/stats/prometheus.rs
+++ b/src/api/stats/prometheus.rs
@@ -639,6 +639,18 @@ pub(super) fn render(stats: &StatsResponse, list: &PeerListResponse) -> String {
         relay.announce_rounds as f64,
     );
     out.single(
+        "treeless_rounds_total",
+        "counter",
+        "Of the announcement rounds, quick ones to all neighbours: some change had no writer's tree.",
+        relay.treeless_rounds as f64,
+    );
+    out.single(
+        "slow_rounds_total",
+        "counter",
+        "Of the announcement rounds, slow ones after changes passed on along writers' trees.",
+        relay.slow_rounds as f64,
+    );
+    out.single(
         "pulls_total",
         "counter",
         "Pulls of changes we lacked, started.",
diff --git a/src/simtest/bench.rs b/src/simtest/bench.rs
index d4a254f..a263e64 100644
--- a/src/simtest/bench.rs
+++ b/src/simtest/bench.rs
@@ -195,6 +195,8 @@ impl Totals {
             let stats = node.stats.snapshot();
             let relay = &mut totals.relay;
             relay.announce_rounds += stats.relay.announce_rounds;
+            relay.treeless_rounds += stats.relay.treeless_rounds;
+            relay.slow_rounds += stats.relay.slow_rounds;
             relay.pulls += stats.relay.pulls;
             relay.pulls_queued += stats.relay.pulls_queued;
             relay.tree_forwards += stats.relay.tree_forwards;
@@ -404,8 +406,10 @@ fn report(
     let (r0, r1) = (during.0.relay, during.1.relay);
     let _ = writeln!(
         out,
-        "    relay per write: {:.1} announcement rounds, {:.1} pulls ({:.1} queued), {:.1} along writers' trees",
+        "    relay per write: {:.1} announcement rounds ({:.1} without a tree, {:.1} slow), {:.1} pulls ({:.1} queued), {:.1} along writers' trees",
         (r1.announce_rounds - r0.announce_rounds) as f64 / writes,
+        (r1.treeless_rounds - r0.treeless_rounds) as f64 / writes,
+        (r1.slow_rounds - r0.slow_rounds) as f64 / writes,
         (r1.pulls - r0.pulls) as f64 / writes,
         (r1.pulls_queued - r0.pulls_queued) as f64 / writes,
         (r1.tree_forwards - r0.tree_forwards) as f64 / writes,
diff --git a/src/swarm/relay.rs b/src/swarm/relay.rs
index 29da2eb..a8560a5 100644
--- a/src/swarm/relay.rs
+++ b/src/swarm/relay.rs
@@ -104,7 +104,7 @@ pub(crate) async fn announce(state: &AppState, from: &str, writer: Option<u64>)
         schedule
     };
     if schedule {
-        round(state, &state.announce_pending, ANNOUNCE_DELAY);
+        round(state, &state.announce_pending, ANNOUNCE_DELAY, false);
     }
     if !tree {
         return;
@@ -121,15 +121,17 @@ pub(crate) async fn announce(state: &AppState, from: &str, writer: Option<u64>)
         schedule
     };
     if schedule {
-        round(state, &state.announce_slow, SLOW_ANNOUNCE_DELAY);
+        round(state, &state.announce_slow, SLOW_ANNOUNCE_DELAY, true);
     }
 }
 
-/// An announcement round of what `pending` collects, `delay` (and a jitter) from now.
+/// An announcement round of what `pending` collects, `delay` (and a jitter) from now:
+/// the `slow` one, or the quick one.
 fn round(
     state: &AppState,
     pending: &std::sync::Arc<std::sync::Mutex<Option<Pending>>>,
     delay: Duration,
+    slow: bool,
 ) {
     let state = state.clone();
     let pending = std::sync::Arc::clone(pending);
@@ -140,14 +142,14 @@ fn round(
                 .max(1);
         tokio::time::sleep(delay + Duration::from_millis(jitter)).await;
         let pending = pending.lock_or_recover().take().unwrap_or_default();
-        announce_now(&state, &pending).await;
+        announce_now(&state, &pending, slow).await;
     });
 }
 
 /// One announcement round. The peer whose changes we announce has them already, so a
 /// round with a single sender skips it; with several, each lacks the others' changes,
 /// so every neighbour is told.
-async fn announce_now(state: &AppState, pending: &Pending) {
+async fn announce_now(state: &AppState, pending: &Pending, slow: bool) {
     let mut neighbours = unreached(state, &pending.senders, &pending.sent).await;
     if !pending.all {
         neighbours.retain(|peer| pending.targets.contains(peer));
@@ -155,7 +157,7 @@ async fn announce_now(state: &AppState, pending: &Pending) {
     if neighbours.is_empty() {
         return;
     }
-    state.stats.announce_round();
+    state.stats.announce_round(pending.all, slow);
     for peer in neighbours {
         if let Err(error) = announce_heads(state, &peer, false).await {
             debug!(peer = %peer, error = %format!("{error:#}"), "Announcing our heads failed");
diff --git a/src/swarm/stats.rs b/src/swarm/stats.rs
index 8a00629..9d32baa 100644
--- a/src/swarm/stats.rs
+++ b/src/swarm/stats.rs
@@ -98,6 +98,11 @@ pub(crate) struct SavedSize {
 pub(crate) struct Relay {
     /// Announcement rounds (each a STATUS to the neighbours).
     pub(crate) announce_rounds: u64,
+    /// Of those, quick rounds to all neighbours, as some change came with no writer's
+    /// tree to go along (pulled, a delta, from a node with no part in it), and slow rounds
+    /// after changes that did (`swarm::relay`).
+    pub(crate) treeless_rounds: u64,
+    pub(crate) slow_rounds: u64,
     /// Pulls started.
     pub(crate) pulls: u64,
     /// Announcements of heads we lacked that had to wait for the pull in progress.
@@ -175,8 +180,16 @@ impl NodeStats {
         self.lock().stale_hello_replies += 1;
     }
 
-    pub(crate) fn announce_round(&self) {
-        self.lock().relay.announce_rounds += 1;
+    /// An announcement round: `treeless` (quick, to all), `slow`, or else quick to
+    /// those writers' trees leave out.
+    pub(crate) fn announce_round(&self, treeless: bool, slow: bool) {
+        let mut inner = self.lock();
+        inner.relay.announce_rounds += 1;
+        if slow {
+            inner.relay.slow_rounds += 1;
+        } else if treeless {
+            inner.relay.treeless_rounds += 1;
+        }
     }
 
     pub(crate) fn pull_started(&self) {