diff options
| author | tv <tv@krebsco.de> | 2026-11-18 20:08:28 +0000 |
|---|---|---|
| committer | tv <tv@krebsco.de> | 2026-11-18 21:17:40 +0100 |
| commit | 79614926332977718956f35e1adf80ae67e219f1 (patch) | |
| tree | 011d1b66381233f61f04c77360e1f9f8f704c5c6 /src/simtest | |
| parent | 8369d2ef1ce1f5856cbff2bd33b1c39dfe80e445 (diff) | |
store: a stopped node leaves out none of what it holds back
A store that stopped can't tell which of what it holds back will be taken, nor which of
what waits: `Store::held_back` gives both, each after what it builds on. A node leaves
out none of it: it puts it again where its document lacks it (`Holding::lacking`), a
remake's snapshot holds it (`Holding::at_base`), and after a switch or a rejoin it puts
again what of it the document it left held (`Holding::left`). A remake while nodes are
stopped no longer drops what was put since the stop, nor the event they stopped at.
Tested: a node keeps what it holds back, what waited among it, through a snapshot, and a
store read anew from that stops there again; and a simulation: a swarm stopped at a
version compacts, a node away meanwhile rejoins and one started again with nothing reads
the new epoch, each stopping there, holding back what came after, until their operators
have them drop that event.
Diffstat (limited to 'src/simtest')
| -rw-r--r-- | src/simtest/holding.rs | 117 |
1 files changed, 117 insertions, 0 deletions
diff --git a/src/simtest/holding.rs b/src/simtest/holding.rs index f7ac715..74086e2 100644 --- a/src/simtest/holding.rs +++ b/src/simtest/holding.rs @@ -1139,6 +1139,123 @@ async fn store_attack_16() { assert!(read.iter().all(|counts| counts == &read[0]), "{read:?}"); } +/// Whether `node`'s store stopped (`Fold::stops`), and at which event. +fn stopped(node: &AppState) -> Option<Id> { + read(node, |store| store.stopped().map(|(id, ..)| *id)).flatten() +} + +/// Whether `node`'s store holds back every event of `ids`. +fn holds_back(node: &AppState, ids: &[Id]) -> bool { + read(node, |store| { + let held: BTreeSet<Id> = store.held_back().into_iter().map(|(id, _)| id).collect(); + ids.iter().all(|id| held.contains(id)) + }) == Some(true) +} + +/// A version of a federation this continuum doesn't know passes, and a version event +/// writes it: every node stops there, and holds back what is put meanwhile. A compaction +/// leaves none of it out: the new document holds that event and what came after. c, away +/// meanwhile, rejoins the new epoch and stops there too, and so does b, started again with +/// nothing. Their operators have them drop that event: they take again, all alike. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_stopped_store_keeps_what_it_holds_back_through_a_remake_and_a_rejoin() { + let sim = Sim::with_identity(98); + let (tv, lass, makefu) = three(); + let (first, root, id) = federation(&[(&tv, &["a"]), (&lass, &["b"]), (&makefu, &["c"])]); + let nodes = started( + &sim, + &root, + vec![ + ("a", holding("tv")), + ("b", holding("lass")), + ("c", holding("makefu")), + ], + ) + .await; + let on = first.hash(); + let text = String::from_utf8(first.after(on).json().bytes()) + .expect("UTF-8") + .replace(r#""federation":1"#, r#""federation":2"#); + let unknown = crate::federation::jcs::read(text.as_bytes()).expect("JCS"); + let hash = Id::of(&unknown.bytes()); + let proposal = tv.propose(&[id], &unknown); + let votes = [&tv, &lass].map(|party| party.vote(&[Id::of(&proposal)], on, hash, true)); + let written = harness::version_event(&[Id::of(&votes[0]), Id::of(&votes[1])], hash); + let at = Id::of(&written); + put(&nodes[0], &[&proposal, &votes[0], &votes[1], &written]).await; + let all = &nodes; + sim.wait_until("every node stopped there", LONG, move || async move { + all.iter().all(|node| stopped(node) == Some(at)) + }) + .await; + + sim.net.take_down("c"); + let meanwhile = tv.write(&[id], "home/tv/a", s("1")); + put(&nodes[0], &[&meanwhile]).await; + let meanwhile = Id::of(&meanwhile); + sim.wait_until("a and b hold it back", LONG, move || async move { + all[..2] + .iter() + .all(|node| holds_back(node, &[at, meanwhile])) + }) + .await; + compact(&nodes[0]).await; + on_generation(&sim, &nodes[..2], 1, Duration::from_secs(60)).await; + { + let replica = nodes[0].doc.read().await; + let held = events::ids(&replica.doc); + assert!(held.contains(&at) && held.contains(&meanwhile)); + } + for node in &nodes[..2] { + assert_eq!(stopped(node), Some(at)); + assert!(holds_back(node, &[meanwhile])); + } + + sim.net.bring_up("c"); + let c = &nodes[2]; + sim.wait_until( + "c rejoins the new epoch, and stops there too", + Duration::from_secs(60), + move || async move { + c.local_epoch().generation == 1 + && c.is_ready().await + && stopped(c) == Some(at) + && holds_back(c, &[meanwhile]) + }, + ) + .await; + let b = sim.restart_with("b", &["a"], Start::Join, move |config| { + of_the_store(config, id, None, holding("lass")) + }); + let b = &b; + sim.wait_until( + "b, started again with nothing, reads the new epoch and stops there", + Duration::from_secs(60), + move || async move { + b.local_epoch().generation == 1 + && b.is_ready().await + && stopped(b) == Some(at) + && holds_back(b, &[meanwhile]) + }, + ) + .await; + + let every = [nodes[0].clone(), b.clone(), nodes[2].clone()]; + for node in &every { + let dropped = node.holding.drop_taken(&[at]).expect("a store"); + assert_eq!(dropped, Ok(vec![at])); + } + let every = &every; + sim.wait_until("every node takes again", LONG, move || async move { + every + .iter() + .all(|node| stopped(node).is_none() && holds(node, &[meanwhile])) + }) + .await; + let read: Vec<Option<Counts>> = every.iter().map(counts).collect(); + assert!(read.iter().all(|counts| counts == &read[0]), "{read:?}"); +} + /// A compaction whose base comes before a checkpoint, switched after nodes moved to it: /// they switch, keep their checkpoint, read none of what it forgot, and put again from /// the old document what a store starting at the snapshot's checkpoint takes to reach |
