From 79614926332977718956f35e1adf80ae67e219f1 Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 18 Nov 2026 20:08:28 +0000 Subject: store: a stopped node leaves out none of what it holds back A store that stopped can't tell which of what it holds back will be taken, nor which of what waits: `Store::held_back` gives both, each after what it builds on. A node leaves out none of it: it puts it again where its document lacks it (`Holding::lacking`), a remake's snapshot holds it (`Holding::at_base`), and after a switch or a rejoin it puts again what of it the document it left held (`Holding::left`). A remake while nodes are stopped no longer drops what was put since the stop, nor the event they stopped at. Tested: a node keeps what it holds back, what waited among it, through a snapshot, and a store read anew from that stops there again; and a simulation: a swarm stopped at a version compacts, a node away meanwhile rejoins and one started again with nothing reads the new epoch, each stopping there, holding back what came after, until their operators have them drop that event. --- src/simtest/holding.rs | 117 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) (limited to 'src/simtest') diff --git a/src/simtest/holding.rs b/src/simtest/holding.rs index f7ac715..74086e2 100644 --- a/src/simtest/holding.rs +++ b/src/simtest/holding.rs @@ -1139,6 +1139,123 @@ async fn store_attack_16() { assert!(read.iter().all(|counts| counts == &read[0]), "{read:?}"); } +/// Whether `node`'s store stopped (`Fold::stops`), and at which event. +fn stopped(node: &AppState) -> Option { + read(node, |store| store.stopped().map(|(id, ..)| *id)).flatten() +} + +/// Whether `node`'s store holds back every event of `ids`. +fn holds_back(node: &AppState, ids: &[Id]) -> bool { + read(node, |store| { + let held: BTreeSet = store.held_back().into_iter().map(|(id, _)| id).collect(); + ids.iter().all(|id| held.contains(id)) + }) == Some(true) +} + +/// A version of a federation this continuum doesn't know passes, and a version event +/// writes it: every node stops there, and holds back what is put meanwhile. A compaction +/// leaves none of it out: the new document holds that event and what came after. c, away +/// meanwhile, rejoins the new epoch and stops there too, and so does b, started again with +/// nothing. Their operators have them drop that event: they take again, all alike. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_stopped_store_keeps_what_it_holds_back_through_a_remake_and_a_rejoin() { + let sim = Sim::with_identity(98); + let (tv, lass, makefu) = three(); + let (first, root, id) = federation(&[(&tv, &["a"]), (&lass, &["b"]), (&makefu, &["c"])]); + let nodes = started( + &sim, + &root, + vec![ + ("a", holding("tv")), + ("b", holding("lass")), + ("c", holding("makefu")), + ], + ) + .await; + let on = first.hash(); + let text = String::from_utf8(first.after(on).json().bytes()) + .expect("UTF-8") + .replace(r#""federation":1"#, r#""federation":2"#); + let unknown = crate::federation::jcs::read(text.as_bytes()).expect("JCS"); + let hash = Id::of(&unknown.bytes()); + let proposal = tv.propose(&[id], &unknown); + let votes = [&tv, &lass].map(|party| party.vote(&[Id::of(&proposal)], on, hash, true)); + let written = harness::version_event(&[Id::of(&votes[0]), Id::of(&votes[1])], hash); + let at = Id::of(&written); + put(&nodes[0], &[&proposal, &votes[0], &votes[1], &written]).await; + let all = &nodes; + sim.wait_until("every node stopped there", LONG, move || async move { + all.iter().all(|node| stopped(node) == Some(at)) + }) + .await; + + sim.net.take_down("c"); + let meanwhile = tv.write(&[id], "home/tv/a", s("1")); + put(&nodes[0], &[&meanwhile]).await; + let meanwhile = Id::of(&meanwhile); + sim.wait_until("a and b hold it back", LONG, move || async move { + all[..2] + .iter() + .all(|node| holds_back(node, &[at, meanwhile])) + }) + .await; + compact(&nodes[0]).await; + on_generation(&sim, &nodes[..2], 1, Duration::from_secs(60)).await; + { + let replica = nodes[0].doc.read().await; + let held = events::ids(&replica.doc); + assert!(held.contains(&at) && held.contains(&meanwhile)); + } + for node in &nodes[..2] { + assert_eq!(stopped(node), Some(at)); + assert!(holds_back(node, &[meanwhile])); + } + + sim.net.bring_up("c"); + let c = &nodes[2]; + sim.wait_until( + "c rejoins the new epoch, and stops there too", + Duration::from_secs(60), + move || async move { + c.local_epoch().generation == 1 + && c.is_ready().await + && stopped(c) == Some(at) + && holds_back(c, &[meanwhile]) + }, + ) + .await; + let b = sim.restart_with("b", &["a"], Start::Join, move |config| { + of_the_store(config, id, None, holding("lass")) + }); + let b = &b; + sim.wait_until( + "b, started again with nothing, reads the new epoch and stops there", + Duration::from_secs(60), + move || async move { + b.local_epoch().generation == 1 + && b.is_ready().await + && stopped(b) == Some(at) + && holds_back(b, &[meanwhile]) + }, + ) + .await; + + let every = [nodes[0].clone(), b.clone(), nodes[2].clone()]; + for node in &every { + let dropped = node.holding.drop_taken(&[at]).expect("a store"); + assert_eq!(dropped, Ok(vec![at])); + } + let every = &every; + sim.wait_until("every node takes again", LONG, move || async move { + every + .iter() + .all(|node| stopped(node).is_none() && holds(node, &[meanwhile])) + }) + .await; + let read: Vec> = every.iter().map(counts).collect(); + assert!(read.iter().all(|counts| counts == &read[0]), "{read:?}"); +} + /// A compaction whose base comes before a checkpoint, switched after nodes moved to it: /// they switch, keep their checkpoint, read none of what it forgot, and put again from /// the old document what a store starting at the snapshot's checkpoint takes to reach -- cgit v1.3.1