make remake switch delay configurable

3 files changed, 44 insertions, 2 deletions

diff --git a/src/remake/api.rs b/src/remake/api.rs
index 7db7acb..d3d1912 100644
--- a/src/remake/api.rs
+++ b/src/remake/api.rs
@@ -19,7 +19,10 @@
 //! `{ "kind": "compact", "spec": null, "baseHeads": ["<hex>", …],
 //!    "replayWindow": {"epochs": 1, "seconds": 604800},
 //!    "archiveRetention": {"epochs": 2, "seconds": 2592000},
-//!    "settleSecs": 480 }`
+//!    "settleSecs": 480, "switchDelaySecs": 60 }`
+//!
+//! `?force=true` accepts `settleSecs` below the dormancy threshold and `switchDelaySecs`
+//! below one keepalive interval: for tests and debugging, not for real clusters.
 //!
 //! `kind` is `compact` (no spec) or `replace`, whose spec is
 //! `{ "document": {…store root…}, "rules": [{"from": "a", "to": "b"}, {"from": "c", "drop": true}] }`
@@ -78,6 +81,9 @@ struct StartRequest {
     replay_window: Option<Retention>,
     archive_retention: Option<Retention>,
     settle_secs: Option<u64>,
+    /// How long nodes wait after seeing the commit before switching; one keepalive
+    /// interval if absent. Shorter only with `?force=true`.
+    switch_delay_secs: Option<u64>,
 }
 
 #[derive(Debug, Serialize)]
@@ -285,6 +291,31 @@ async fn start_kind<H: Host>(
         );
     }
 
+    // The delay lets the commit reach every node through ordinary sync before any of
+    // them stops syncing the old epoch.
+    let keepalive = dormant_after / 4;
+    let switch_delay = request.switch_delay_secs.map(Duration::from_secs);
+    match switch_delay {
+        Some(delay) if delay < keepalive && query.force => {
+            warn!(
+                switch_delay_secs = delay.as_secs(),
+                keepalive_secs = keepalive.as_secs(),
+                "Forced switch delay shorter than a keepalive interval; slow nodes may have to rejoin"
+            );
+        }
+        Some(delay) if delay < keepalive => {
+            return error_response(
+                StatusCode::BAD_REQUEST,
+                format!(
+                    "switchDelaySecs must be at least {} (one keepalive interval), so the commit reaches every node before they switch; add ?force=true to override",
+                    keepalive.as_secs()
+                ),
+                None,
+            );
+        }
+        _ => {}
+    }
+
     // Fail fast instead of proposing something that will abort anyway.
     let peers = host.peer_summaries().await;
     let dormant: Vec<String> = peers
@@ -323,6 +354,7 @@ async fn start_kind<H: Host>(
         settle_ms,
         proposed_at: now,
         settle_until: now + settle_ms,
+        switch_delay_ms: switch_delay.map(|delay| delay.as_millis() as u64),
         peers_considered,
         kind: kind.name().to_owned(),
         content: prepared.content,
diff --git a/src/remake/records.rs b/src/remake/records.rs
index 9edca6e..600d4dc 100644
--- a/src/remake/records.rs
+++ b/src/remake/records.rs
@@ -123,6 +123,10 @@ pub(crate) struct Proposal {
     pub(crate) settle_ms: u64,
     pub(crate) proposed_at: u64,
     pub(crate) settle_until: u64,
+    /// How long nodes wait after seeing the commit before switching (ms); one keepalive
+    /// interval if absent.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub(crate) switch_delay_ms: Option<u64>,
     /// Peers that must stay active during the settle period.
     pub(crate) peers_considered: Vec<String>,
     /// The remake kind (`compact` for records written before kinds existed).
@@ -355,6 +359,7 @@ mod tests {
             settle_ms: 1_000,
             proposed_at: settle_until.saturating_sub(1_000),
             settle_until,
+            switch_delay_ms: None,
             peers_considered: vec!["node-b".into()],
             kind: DEFAULT_KIND.to_owned(),
             content: JsonValue::Null,
diff --git a/src/remake/switch.rs b/src/remake/switch.rs
index a5031fc..2670f14 100644
--- a/src/remake/switch.rs
+++ b/src/remake/switch.rs
@@ -111,7 +111,12 @@ async fn check<H: Host>(host: &H) {
         return catchup::check(host).await;
     }
 
-    let switch_delay = host.dormant_after() / 4; // one keepalive interval
+    // As proposed, or one keepalive interval.
+    let switch_delay = record
+        .proposal
+        .switch_delay_ms
+        .map(Duration::from_millis)
+        .unwrap_or(host.dormant_after() / 4);
     if host.remake().first_seen(&id).elapsed() < switch_delay {
         return;
     }