remake: decouple grace time from node's keepalive

4 files changed, 25 insertions, 4 deletions

diff --git a/src/remake/api.rs b/src/remake/api.rs
index 7750242..8fd0c5b 100644
--- a/src/remake/api.rs
+++ b/src/remake/api.rs
@@ -453,6 +453,7 @@ async fn start_kind<H: Host>(
         proposed_at: now,
         settle_until: now.plus(settle),
         switch_delay_ms: switch_delay.map(|delay| delay.as_millis() as u64),
+        grace_ms: Some(grace),
         // Filled in when the proposal is written, from the document at that moment.
         electorate: Vec::new(),
         quorum: 0,
diff --git a/src/remake/proposal.rs b/src/remake/proposal.rs
index d9cd893..7695b4c 100644
--- a/src/remake/proposal.rs
+++ b/src/remake/proposal.rs
@@ -23,7 +23,7 @@ pub(super) fn abortable_at(record: &Record, grace: u64) -> UnixMillis {
     record
         .proposal
         .settle_until
-        .plus_ms(grace.saturating_mul(2))
+        .plus_ms(record.grace(grace).saturating_mul(2))
 }
 
 /// Why this node may not abort expired `record` now, and the `force` name that would
@@ -157,7 +157,7 @@ pub(super) async fn propose<H: Host>(
                             coordinator_active: supersede
                                 .active_peers
                                 .contains(&record.proposal.coordinator),
-                            since: record.proposal.settle_until.plus_ms(grace),
+                            since: record.proposal.settle_until.plus_ms(record.grace(grace)),
                             abortable_at: abortable_at(record, grace),
                             tally: vote::tally(id, record, &all, &register),
                         })
diff --git a/src/remake/records.rs b/src/remake/records.rs
index 55fa8a0..0c4c444 100644
--- a/src/remake/records.rs
+++ b/src/remake/records.rs
@@ -122,6 +122,11 @@ pub(crate) struct Proposal {
     /// interval if absent.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub(crate) switch_delay_ms: Option<u64>,
+    /// The coordinator's grace period (ms, its `dormant_after`): every node judges this
+    /// remake's expiry, and when others may abort it, by it, whatever its own keepalive
+    /// (`Record::grace`). Absent in records from before: each node's own.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub(crate) grace_ms: Option<u64>,
     /// Members whose votes count (sorted), fixed at proposal time (`vote`).
     pub(crate) electorate: Vec<String>,
     /// Votes needed to commit: a majority of the electorate.
@@ -311,14 +316,23 @@ impl Record {
         })
     }
 
-    pub(crate) fn phase(&self, now: UnixMillis, grace_ms: u64) -> Phase {
+    /// The phase at `now`, expiry judged by the record's grace period (`grace`; `own`,
+    /// this node's, for a record without one).
+    pub(crate) fn phase(&self, now: UnixMillis, own: u64) -> Phase {
         match &self.decision {
             Some(Decision::Committed { .. }) => Phase::Committed,
             Some(Decision::Aborted { .. }) => Phase::Aborted,
-            None if now > self.proposal.settle_until.plus_ms(grace_ms) => Phase::Expired,
+            None if now > self.proposal.settle_until.plus_ms(self.grace(own)) => Phase::Expired,
             None => Phase::Proposed,
         }
     }
+
+    /// The grace period this remake is judged by (ms): its coordinator's, the same on
+    /// every node, so nodes with different keepalives agree on when it expired; `own`
+    /// for a record from before it was written down.
+    pub(crate) fn grace(&self, own: u64) -> u64 {
+        self.proposal.grace_ms.unwrap_or(own)
+    }
 }
 
 // --- document access ---------------------------------------------------------
@@ -564,6 +578,7 @@ mod tests {
             proposed_at: UnixMillis(settle_until.saturating_sub(1_000)),
             settle_until: UnixMillis(settle_until),
             switch_delay_ms: None,
+            grace_ms: None,
             electorate: vec!["node-a".into(), "node-b".into()],
             quorum: 2,
             kind: DEFAULT_KIND.to_owned(),
@@ -654,6 +669,10 @@ mod tests {
         assert_eq!(record.phase(UnixMillis(10_000), 500), Phase::Proposed);
         assert_eq!(record.phase(UnixMillis(10_500), 500), Phase::Proposed);
         assert_eq!(record.phase(UnixMillis(10_501), 500), Phase::Expired);
+        // The coordinator's grace period, written down, wins over this node's.
+        record.proposal.grace_ms = Some(2_000);
+        assert_eq!(record.phase(UnixMillis(10_501), 500), Phase::Proposed);
+        assert_eq!(record.phase(UnixMillis(12_001), 500), Phase::Expired);
         record.decision = Some(Decision::Aborted {
             reason: "test".into(),
             decided_at: UnixMillis(1),
diff --git a/src/remake/vote.rs b/src/remake/vote.rs
index 6604ff6..12c1333 100644
--- a/src/remake/vote.rs
+++ b/src/remake/vote.rs
@@ -389,6 +389,7 @@ mod tests {
                 proposed_at,
                 settle_until: proposed_at.plus_ms(1_000),
                 switch_delay_ms: None,
+                grace_ms: None,
                 electorate: electorate.iter().map(|m| (*m).to_owned()).collect(),
                 quorum: quorum(electorate.len()),
                 kind: "compact".to_owned(),