serve stale reads during rejoinHEADmaster

2 files changed, 20 insertions, 2 deletions

diff --git a/src/compaction/catchup.rs b/src/compaction/catchup.rs
index c71bb57..d8bac74 100644
--- a/src/compaction/catchup.rs
+++ b/src/compaction/catchup.rs
@@ -4,7 +4,8 @@
 //! returning, or one whose own build failed) can't follow through ordinary sync. Once
 //! an active peer advertises a newer generation, the node
 //!
-//! 1. stops serving requests and keeps its document aside;
+//! 1. stops being ready and keeps its document aside; reads are answered from a copy
+//!    of it, marked stale (`X-Continuum-Stale`), writes get `503`;
 //! 2. switches to an empty document labelled with the newer epoch: an empty history
 //!    conflicts with nothing, so ordinary sync fills it from peers on that epoch,
 //!    exactly like a new node joining;
@@ -16,7 +17,8 @@
 //!    * within that epoch's replay window they are replayed (state per path, as in a
 //!      switch), otherwise they are quarantined for an operator
 //!      (`/admin/quarantine`), as are changes that can't be told apart;
-//! 5. archives the old document and serves requests again.
+//! 5. archives the old document and serves requests again. A node that was never
+//!    ready (a new node) is made ready by bootstrap, which waits for the rejoin.
 //!
 //! If no peer completes a sync in time, or the check fails, the node restores its old
 //! document and epoch and tries again later.
@@ -117,6 +119,12 @@ async fn start<H: Host>(host: &H, local: Epoch, target: Epoch, attempts: u32) {
         error: None,
     }));
 
+    // Keep answering reads, from a copy of the current document marked stale, before
+    // the node stops being ready: no window in which reads get 503.
+    if host.is_node_ready().await {
+        let copy = host.read_doc(|doc| doc.clone()).await;
+        host.set_stale_view(Some((local, copy)));
+    }
     let was_ready = host.suspend().await;
     let old = match host
         .switch_epoch(target, |_live| Ok::<_, Infallible>(AutoCommit::new()))
@@ -214,6 +222,7 @@ async fn finish<H: Host>(host: &H) {
             if was_ready {
                 host.resume().await;
             }
+            host.set_stale_view(None);
             return;
         }
     };
@@ -277,6 +286,8 @@ async fn finish<H: Host>(host: &H) {
     if was_ready {
         host.resume().await;
     }
+    // After resuming, so reads move from the stale view to the live document directly.
+    host.set_stale_view(None);
 
     info!(
         from = %previous,
@@ -386,9 +397,12 @@ async fn abandon<H: Host>(host: &H, rejoin: Rejoin, reason: String) {
     {
         match never {}
     }
+    // A node that was never ready must sync again before bootstrap may finish.
+    host.reset_initial_sync();
     if was_ready {
         host.resume().await;
     }
+    host.set_stale_view(None);
 
     let attempts = host
         .compaction()
diff --git a/src/compaction/mod.rs b/src/compaction/mod.rs
index 4685b86..6de22c5 100644
--- a/src/compaction/mod.rs
+++ b/src/compaction/mod.rs
@@ -69,6 +69,10 @@ pub(crate) trait Host: Clone + Send + Sync + 'static {
     /// Serves requests again after [`Host::suspend`].
     fn resume(&self) -> impl Future<Output = ()> + Send;
 
+    /// Serves reads from this document (read-only, marked stale) while the node is not
+    /// ready; `None` ends it.
+    fn set_stale_view(&self, view: Option<(Epoch, AutoCommit)>);
+
     /// Whether the document caught up with a peer since the last reset.
     fn initial_sync_done(&self) -> bool;