diff options
| -rw-r--r-- | docs/measurements.md | 1 | ||||
| -rw-r--r-- | docs/openapi.yaml | 5 | ||||
| -rw-r--r-- | src/api/stats/prometheus.rs | 7 | ||||
| -rw-r--r-- | src/simtest/bench.rs | 4 | ||||
| -rw-r--r-- | src/state/peer.rs | 5 |
5 files changed, 19 insertions, 3 deletions
diff --git a/docs/measurements.md b/docs/measurements.md index cff7add..b5c3d6e 100644 --- a/docs/measurements.md +++ b/docs/measurements.md @@ -261,6 +261,7 @@ forgotten peer's are gone). |---|---|---| | `incidents.replyTimeouts` | `peer_reply_timeouts_total` | Our sync messages the peer didn't answer in time (10 s, 30 s with Tor enabled, doubling to 30 times that); resent. Lost messages, or a slow peer. | | `incidents.heals` | `peer_heals_total` | Re-handshakes because our heads and the peer's differed for `resync_after` with no sync between ([`peers.md`](peers.md), "Keepalive and self-healing"). Rare in a healthy swarm; frequent means sync state keeps getting out of step. | +| `incidents.askedAgain` | `peer_asked_again_total` | Times nothing answered what we sent (a keepalive, or a sync message that awaits an answer) for half the time it may before the peer counts as unreached (`unreached_after`, twice the first sync reply timeout), so a PING asked again ([`peers.md`](peers.md)). Each is a PING and its PONG on top of the keepalives. | | `incidents.dormancies` | `peer_dormancies_total` | Times the peer went dormant (nothing received for `dormant_after`). | | `incidents.dormantMs` | `peer_dormant_seconds_total` | Time it spent dormant, up to its last return. | diff --git a/docs/openapi.yaml b/docs/openapi.yaml index db73ede..f45fae7 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1743,7 +1743,7 @@ components: Incidents: type: object - required: [replyTimeouts, heals, dormancies, dormantMs] + required: [replyTimeouts, heals, askedAgain, dormancies, dormantMs] properties: replyTimeouts: type: integer @@ -1751,6 +1751,9 @@ components: heals: type: integer description: Re-handshakes because documents differed while sync was idle. + askedAgain: + type: integer + description: Times nothing answered our keepalives or sync messages for a while, so a PING asked again. dormancies: type: integer dormantMs: diff --git a/src/api/stats/prometheus.rs b/src/api/stats/prometheus.rs index ffe6593..fb8b890 100644 --- a/src/api/stats/prometheus.rs +++ b/src/api/stats/prometheus.rs @@ -1068,7 +1068,7 @@ pub(super) fn render(stats: &StatsResponse, list: &PeerListResponse) -> String { } } - let incidents: [IncidentSeries; 4] = [ + let incidents: [IncidentSeries; 5] = [ ( "peer_reply_timeouts_total", "Sync messages unanswered until the reply timeout.", @@ -1080,6 +1080,11 @@ pub(super) fn render(stats: &StatsResponse, list: &PeerListResponse) -> String { |i| i.heals as f64, ), ( + "peer_asked_again_total", + "Times nothing answered for a while, so a PING asked again.", + |i| i.asked_again as f64, + ), + ( "peer_dormancies_total", "Times the peer went dormant.", |i| i.dormancies as f64, diff --git a/src/simtest/bench.rs b/src/simtest/bench.rs index 06b557e..3b1b3e6 100644 --- a/src/simtest/bench.rs +++ b/src/simtest/bench.rs @@ -622,10 +622,12 @@ fn report( let (i0, i1) = (during.0.incidents, during.1.incidents); let _ = writeln!( out, - " incidents: {} reply timeouts ({:.2} per write), {} heals, {} dormancies", + " incidents: {} reply timeouts ({:.2} per write), {} heals, {} asked again ({:.1} per write), {} dormancies", i1.reply_timeouts - i0.reply_timeouts, (i1.reply_timeouts - i0.reply_timeouts) as f64 / writes, i1.heals - i0.heals, + i1.asked_again - i0.asked_again, + (i1.asked_again - i0.asked_again) as f64 / writes, i1.dormancies - i0.dormancies, ); let mut propagation = Propagation::default(); diff --git a/src/state/peer.rs b/src/state/peer.rs index 787a772..a84e069 100644 --- a/src/state/peer.rs +++ b/src/state/peer.rs @@ -233,6 +233,9 @@ pub(crate) struct Incidents { pub(crate) reply_timeouts: u64, /// Re-handshakes because our documents differed while sync was idle. pub(crate) heals: u64, + /// Times nothing answered us for half the time it may (`AwaitingAnswer`), so we + /// asked again, with a PING (or the keepalive then due). + pub(crate) asked_again: u64, /// Times the peer went dormant. pub(crate) dormancies: u64, /// Time spent dormant, in milliseconds, up to its last return (the current @@ -244,6 +247,7 @@ impl Incidents { pub(crate) fn add(&mut self, other: &Incidents) { self.reply_timeouts += other.reply_timeouts; self.heals += other.heals; + self.asked_again += other.asked_again; self.dormancies += other.dormancies; self.dormant_ms += other.dormant_ms; } @@ -439,6 +443,7 @@ impl PeerState { match &mut self.awaiting_answer { Some(awaiting) if awaiting.ask_again_at.is_some_and(|at| now >= at) => { awaiting.ask_again_at = None; + self.incidents.asked_again += 1; true } _ => false, |