diff options
| -rw-r--r-- | src/simtest/remakes.rs | 60 |
1 files changed, 59 insertions, 1 deletions
diff --git a/src/simtest/remakes.rs b/src/simtest/remakes.rs index ab04efe..b109f0d 100644 --- a/src/simtest/remakes.rs +++ b/src/simtest/remakes.rs @@ -1,6 +1,7 @@ //! Remakes on whole nodes (`docs/remakes.md`): a compaction everywhere, competing //! proposals across a partition, a node that misses the switch, a coordinator that is -//! gone, a skewed clock, and writes while a node starts to rejoin. +//! gone, a skewed clock, writes while a node starts to rejoin, and two remakes committed +//! for one epoch (a coordinator cut off after it committed). //! //! The simulated nodes' timings make a remake quick: settle 2 s (two `dormant_after`), //! switch delay a keepalive, votes within half a keepalive. @@ -312,3 +313,60 @@ async fn writes_while_a_rejoin_starts_are_kept_or_refused() { }) .await; } + +/// A coordinator cut off right after it committed: the others never hear of the commit, +/// abort the remake once it expired and commit another. The coordinator switches to its +/// own; once back, it sees more peers on the other epoch of that generation, rejoins +/// that one sideways (`catchup::sideways`), and the swarm is on one epoch again. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn coordinator_cut_off_after_committing_rejoins_the_others() { + let sim = Sim::new(37); + let (a, b, c) = three(&sim).await; + write(&b, "data", json!(1)).await; + sim.wait_converged(&[&a, &b, &c], SOON).await; + + let cut = LinkParams { + loss: 1.0, + ..LinkParams::default() + }; + let lost = propose(&a, json!({ "kind": "compact" })).await; + let (a_ref, b_ref, c_ref, lost_ref) = (&a, &b, &c, lost.as_str()); + sim.wait_until("b and c have the proposal", SOON, move || async move { + testing::report(b_ref, lost_ref).await.is_some() + && testing::report(c_ref, lost_ref).await.is_some() + }) + .await; + // Their votes still reach a; nothing from a reaches them. + for other in ["b", "c"] { + sim.net.set_link("a", other, cut.clone()); + } + sim.wait_until("a committed", SOON, move || async move { + testing::report(a_ref, lost_ref) + .await + .is_some_and(|report| report["phase"] == "committed") + }) + .await; + sim.net.take_down("a"); + on_generation(&sim, &[&a], 1).await; + + // For b and c, the remake expired undecided. + sim.wait_until("b can abort the lost remake", REMAKE, move || async move { + let (status, _) = testing::abort(b_ref.clone(), lost_ref, None).await; + status == StatusCode::OK + }) + .await; + propose(&b, json!({ "kind": "compact" })).await; + on_generation(&sim, &[&b, &c], 1).await; + let theirs = same_epoch(&[&b, &c]); + assert_ne!(a.local_epoch(), theirs, "two epochs of generation 1"); + + for other in ["b", "c"] { + sim.net.set_link("a", other, LinkParams::default()); + } + sim.net.bring_up("a"); + sim.wait_until("a is on b's and c's epoch", REMAKE, move || async move { + a_ref.local_epoch() == theirs && a_ref.is_ready().await + }) + .await; + assert_eq!(read(&a, "data").await, Some(json!(1))); +} |