heal from noticed divergence

Wait 2*keepalive but at least 20 s to trigger resync so that lost
messages can be resent first.

1 files changed, 27 insertions, 6 deletions

diff --git a/src/state.rs b/src/state.rs
index a6c2356..6d95bb5 100644
--- a/src/state.rs
+++ b/src/state.rs
@@ -111,6 +111,10 @@ pub(crate) struct PeerState {
     /// Last self-healing re-handshake (cooldown).
     pub(crate) last_resync_at: Option<Instant>,
 
+    /// Since when keepalives show the peer's heads differ from ours with no sync message
+    /// from it in between (None while they match).
+    pub(crate) diverged_since: Option<Instant>,
+
     /// Epoch the peer advertised in its most recent message.
     pub(crate) epoch: Option<Epoch>,
 
@@ -228,6 +232,7 @@ impl PeerState {
             last_ping_at: None,
             last_sync_rx: None,
             last_resync_at: None,
+            diverged_since: None,
             epoch: None,
             reports_ready: None,
             syncs_with_us: None,
@@ -324,25 +329,39 @@ impl PeerState {
         timing: &Timing,
         now: Instant,
     ) -> bool {
+        if local == remote {
+            self.diverged_since = None;
+            return false;
+        }
+
+        // How long the heads have differed without sync making progress. Measured from
+        // when we noticed, not from the last sync message: in a quiet swarm that one is
+        // long past, and the first keepalive after a lost message would heal at once,
+        // before the sender's reply timeout could resend it. Any sync message restarts
+        // the clock.
+        let noticed = match self.diverged_since {
+            Some(since) if self.last_sync_rx.is_none_or(|rx| rx <= since) => since,
+            _ => *self.diverged_since.insert(now),
+        };
+
         // While an answer to our last sync message is due, a lost message is the reply
         // timeout's to resend; healing is for when neither side has anything outstanding.
-        if local == remote
-            || self.liveness != Liveness::Active
+        if self.liveness != Liveness::Active
             || self.hello_state != HelloState::Established
             || self.sync.reply.is_some()
         {
             return false;
         }
 
-        let sync_idle = now.saturating_duration_since(self.last_sync_rx.unwrap_or(self.added_at))
-            >= timing.resync_after;
+        let persisted = now.saturating_duration_since(noticed) >= timing.resync_after;
         let cooled_down = self
             .last_resync_at
             .is_none_or(|t| now.saturating_duration_since(t) >= timing.resync_after);
 
-        if !(sync_idle && cooled_down) {
+        if !(persisted && cooled_down) {
             return false;
         }
+        self.diverged_since = None;
 
         self.last_resync_at = Some(now);
         self.hello_state = HelloState::NotSent; // establish_peer resets sync on the reply
@@ -381,7 +400,9 @@ impl Timing {
         Self {
             keepalive,
             dormant_after: keepalive * 4, // dormant_after >= 4* keepalive to tolerate three lost PINGs in a row
-            resync_after: keepalive * 2, // resync_after should be >= 2* keepalive to guarantee at least one digest exchange while sync is idle
+            // >= 2 keepalives, so a second exchange confirms the difference; and >= twice
+            // the first sync reply timeout, so a lost message's sender resends it first.
+            resync_after: (keepalive * 2).max(SYNC_REPLY_TIMEOUT * 2),
             dormant_probe: keepalive * 10, // >= 10*: dormant peers cost almost nothing, but still rejoin within minutes
             // Seconds, not a keepalive interval: a dead source should cost little time.
             // Only the source gets these, and only while catching up.