remake needs sane clock

4 files changed, 31 insertions, 4 deletions

diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index a5780e3..96132cf 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -715,7 +715,8 @@ paths:
             each with `id`, `coordinator`, `coordinatorActive`, `expiredAt`,
             `abortableAt` and `tally`; `force` names the override, if any); or this
             node can't vote for a new one: it voted for another remake of this epoch,
-            or may have before it restarted.
+            or may have before it restarted; or this node's clock is off from most of
+            its peers' (`force` is `clock`).
           content:
             application/json:
               schema:
@@ -1102,8 +1103,9 @@ components:
         Checks to override: `true` for all, or a comma-separated list of
         `switchDelay` (accept one below a keepalive interval; tests only) and
         `supersede` (abort expired remakes of this epoch that still hold votes, and
-        propose in their place) and `coordinatorActive` (supersede even though this
-        node sees their coordinator as an active peer).
+        propose in their place), `coordinatorActive` (supersede even though this
+        node sees their coordinator as an active peer) and `clock` (propose although
+        this node's clock is off from most of its peers').
       schema:
         type: string
       examples:
diff --git a/src/remake/api.rs b/src/remake/api.rs
index f919be7..75d0294 100644
--- a/src/remake/api.rs
+++ b/src/remake/api.rs
@@ -32,6 +32,8 @@
 //!   not for real swarms);
 //! * `supersede`: abort expired remakes of this epoch and propose in their place.
 //!   Without it, an expired remake still holding votes answers `409`;
+//! * `clock`: propose although this node's clock is off from its peers'
+//!   (`swarm::skew`): it would judge the remake's deadlines wrongly. Without it, `409`;
 //! * for `DELETE`: `coordinatorActive` (abort an expired remake whose coordinator this
 //!   node sees as active) and `coordinatorInactive` (ask for an abort of an undecided
 //!   one whose coordinator this node doesn't reach).
@@ -200,7 +202,7 @@ impl From<RetentionRequest> for Retention {
 }
 
 /// What `?force=` may override on a proposal (`force`).
-const FORCE: &[&str] = &["switchDelay", "supersede", "coordinatorActive"];
+const FORCE: &[&str] = &["switchDelay", "supersede", "coordinatorActive", "clock"];
 /// What `?force=` may override on an abort.
 const ABORT_FORCE: &[&str] = &["coordinatorActive", "coordinatorInactive"];
 
@@ -295,6 +297,21 @@ async fn start_kind<H: Host>(
         Ok(force) => force,
         Err(reason) => return error_response(StatusCode::BAD_REQUEST, reason, None),
     };
+    // The coordinator stamps and judges the voting period and the deadlines by its
+    // clock (`trust.md`, "What depends on clocks").
+    let skew = host.clock_skew().await;
+    if skew.off && !force.allows("clock") {
+        let offset = skew.offset_ms.unwrap_or(0);
+        return error_response(
+            StatusCode::CONFLICT,
+            format!(
+                "this node's clock is {} {} most of its peers' (more than --clock-skew-warn), so it would judge the remake's deadlines wrongly; fix its time synchronisation, or add ?force=clock",
+                crate::units::format_duration(Duration::from_millis(offset.unsigned_abs())),
+                if offset < 0 { "behind" } else { "ahead of" }
+            ),
+            Some(json!({ "force": "clock" })),
+        );
+    }
 
     let request: StartRequest = if is_octet_stream(headers) {
         match query.import_request(&body) {
diff --git a/src/remake/mod.rs b/src/remake/mod.rs
index 5042ce9..c77d56d 100644
--- a/src/remake/mod.rs
+++ b/src/remake/mod.rs
@@ -123,6 +123,10 @@ pub(crate) trait Host: Clone + Send + Sync + 'static {
     /// Default basis for the voting period, and the grace margin after it.
     fn dormant_after(&self) -> Duration;
 
+    /// This node's clock against its peers' (`swarm::skew`): a coordinator judges the
+    /// remake's deadlines by it.
+    fn clock_skew(&self) -> impl Future<Output = crate::swarm::skew::Skew> + Send;
+
     /// The electorate of a remake proposed now (`vote`): the members that said they are
     /// alive recently enough, read from `doc`.
     fn electorate(&self, doc: &AutoCommit, now: crate::clock::UnixMillis) -> Vec<String>;
diff --git a/src/state/host.rs b/src/state/host.rs
index 9d77f35..912a187 100644
--- a/src/state/host.rs
+++ b/src/state/host.rs
@@ -82,6 +82,10 @@ impl Host for AppState {
         self.timing.dormant_after
     }
 
+    async fn clock_skew(&self) -> crate::swarm::skew::Skew {
+        crate::swarm::skew::measure(self).await
+    }
+
     fn electorate(&self, doc: &AutoCommit, now: crate::clock::UnixMillis) -> Vec<String> {
         crate::swarm::members::electorate(doc, now.as_u64(), self.electorate_window)
     }