serve stale reads during rejoinHEADmaster

1 files changed, 16 insertions, 2 deletions

diff --git a/src/compaction/catchup.rs b/src/compaction/catchup.rs
index c71bb57..d8bac74 100644
--- a/src/compaction/catchup.rs
+++ b/src/compaction/catchup.rs
@@ -4,7 +4,8 @@
 //! returning, or one whose own build failed) can't follow through ordinary sync. Once
 //! an active peer advertises a newer generation, the node
 //!
-//! 1. stops serving requests and keeps its document aside;
+//! 1. stops being ready and keeps its document aside; reads are answered from a copy
+//!    of it, marked stale (`X-Continuum-Stale`), writes get `503`;
 //! 2. switches to an empty document labelled with the newer epoch: an empty history
 //!    conflicts with nothing, so ordinary sync fills it from peers on that epoch,
 //!    exactly like a new node joining;
@@ -16,7 +17,8 @@
 //!    * within that epoch's replay window they are replayed (state per path, as in a
 //!      switch), otherwise they are quarantined for an operator
 //!      (`/admin/quarantine`), as are changes that can't be told apart;
-//! 5. archives the old document and serves requests again.
+//! 5. archives the old document and serves requests again. A node that was never
+//!    ready (a new node) is made ready by bootstrap, which waits for the rejoin.
 //!
 //! If no peer completes a sync in time, or the check fails, the node restores its old
 //! document and epoch and tries again later.
@@ -117,6 +119,12 @@ async fn start<H: Host>(host: &H, local: Epoch, target: Epoch, attempts: u32) {
         error: None,
     }));
 
+    // Keep answering reads, from a copy of the current document marked stale, before
+    // the node stops being ready: no window in which reads get 503.
+    if host.is_node_ready().await {
+        let copy = host.read_doc(|doc| doc.clone()).await;
+        host.set_stale_view(Some((local, copy)));
+    }
     let was_ready = host.suspend().await;
     let old = match host
         .switch_epoch(target, |_live| Ok::<_, Infallible>(AutoCommit::new()))
@@ -214,6 +222,7 @@ async fn finish<H: Host>(host: &H) {
             if was_ready {
                 host.resume().await;
             }
+            host.set_stale_view(None);
             return;
         }
     };
@@ -277,6 +286,8 @@ async fn finish<H: Host>(host: &H) {
     if was_ready {
         host.resume().await;
     }
+    // After resuming, so reads move from the stale view to the live document directly.
+    host.set_stale_view(None);
 
     info!(
         from = %previous,
@@ -386,9 +397,12 @@ async fn abandon<H: Host>(host: &H, rejoin: Rejoin, reason: String) {
     {
         match never {}
     }
+    // A node that was never ready must sync again before bootstrap may finish.
+    host.reset_initial_sync();
     if was_ready {
         host.resume().await;
     }
+    host.set_stale_view(None);
 
     let attempts = host
         .compaction()