diff options
| -rw-r--r-- | src/simtest/remakes.rs | 82 |
1 files changed, 81 insertions, 1 deletions
diff --git a/src/simtest/remakes.rs b/src/simtest/remakes.rs index 1c76520..6a1f0a9 100644 --- a/src/simtest/remakes.rs +++ b/src/simtest/remakes.rs @@ -2,7 +2,8 @@ //! proposals across a partition, a node that misses the switch, a coordinator that is //! gone, a skewed clock, writes while a node starts to rejoin, and two remakes committed //! for one epoch (a coordinator cut off after it committed; electorates of a partition -//! older than the electorate window), and a node with its clock ahead that would abort +//! older than the electorate window; a vote lost with a restart), and a node with its +//! clock ahead that would abort //! a remake as expired while its coordinator still runs it. //! //! The simulated nodes' timings make a remake quick: settle 2 s (two `dormant_after`), @@ -494,3 +495,82 @@ async fn abort_from_a_node_with_its_clock_off_is_refused() { .is_some_and(|report| report["phase"] != "aborted") ); } + +/// A vote lost with a restart: v votes on one side of a partition, restarts without a +/// document file, joins the other side, which never saw the remake, and votes there +/// for another. Each side reaches its majority with v, and both commit. Once the +/// partition heals, the side fewer nodes are on rejoins the other's epoch sideways. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_vote_lost_with_a_restart_and_the_swarm_converges() { + let sim = Sim::new(40); + let a = sim.start("a", &[], Start::Genesis); + let [b, v, d, e] = ["b", "v", "d", "e"].map(|id| sim.start(id, &["a"], Start::Join)); + sim.wait_ready(&[&a, &b, &v, &d, &e], SOON).await; + let nodes = [a.clone(), b.clone(), v.clone(), d.clone(), e.clone()]; + sim.wait_until("every node is registered", SOON, || { + let nodes = nodes.clone(); + async move { + for node in &nodes { + if crate::swarm::members::members(&node.doc.read().await.doc).len() < 5 { + return false; + } + } + true + } + }) + .await; + write(&a, "data", json!(1)).await; + sim.wait_converged(&[&a, &b, &v, &d, &e], SOON).await; + + let cut = LinkParams { + loss: 1.0, + ..LinkParams::default() + }; + let link = |side: &[&str], other: &[&str], params: &LinkParams| { + for one in side { + for two in other { + sim.net.set_link(one, two, params.clone()); + sim.net.set_link(two, one, params.clone()); + } + } + }; + link(&["a", "b", "v"], &["d", "e"], &cut); + + // a, b and v: three of five, X commits. + let x = propose(&a, json!({ "kind": "compact" })).await; + let (a_ref, x_ref) = (&a, x.as_str()); + sim.wait_until("X committed", REMAKE, move || async move { + testing::report(a_ref, x_ref) + .await + .is_some_and(|report| report["phase"] == "committed") + }) + .await; + + // v restarts on the other side, with nothing of what it had: no record of X. + link(&["v"], &["a", "b"], &cut); + link(&["v"], &["d", "e"], &LinkParams::default()); + let v = sim.restart_with("v", &["d"], Start::Join, |_| {}); + sim.wait_ready(&[&v], SOON).await; + + // d, e and v: three of five, Y commits too. + propose(&d, json!({ "kind": "compact" })).await; + on_generation(&sim, &[&a, &b], 1).await; + on_generation(&sim, &[&d, &e, &v], 1).await; + let more = same_epoch(&[&d, &e, &v]); + assert_ne!(same_epoch(&[&a, &b]), more, "two epochs of generation 1"); + + link(&["a", "b"], &["v", "d", "e"], &LinkParams::default()); + let (b_ref, more_ref) = (&b, more); + sim.wait_until( + "a and b are on the epoch more nodes are on", + REMAKE, + move || async move { + a_ref.local_epoch() == more_ref + && b_ref.local_epoch() == more_ref + && a_ref.is_ready().await + && b_ref.is_ready().await + }, + ) + .await; + assert_eq!(read(&b, "data").await, Some(json!(1))); +} |