//! Swarms holding a store (`docs/store.md`, Members), the federation its fold: nodes //! started from its root, or joining by its ID, sync with the members its store names; //! the events put on any of them reach all of them, and none reaches a node its party //! doesn't list; and the attack rows marked sim, each by its test. use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use std::sync::Arc; use std::time::Duration; use super::{SOON, Sim, sim_key}; use crate::document::events; use crate::federation::client::Record; use crate::federation::event::{Checkpoint, Does, Event, Signed}; use crate::federation::fold::Federation; use crate::federation::fork; use crate::federation::harness::{self, Party, Spec}; use crate::federation::held::{self, Holding, Voter}; use crate::federation::jcs::Json; use crate::federation::store::memory::Store; use crate::federation::store::{Fold, Id}; use crate::federation::version::Version; use crate::federation::vote::Rules; use crate::files::Files; use crate::node::NodeConfig; use crate::state::{AppState, HelloState}; use crate::swarm::bootstrap::{Origin, Start}; use crate::swarm::topology; use crate::transport::MessageKind; use crate::transport::sim::LinkParams; /// How long a test waits for what takes a few rounds of syncs, writes and reads. const LONG: Duration = Duration::from_secs(30); /// A federation of `parties`, each listing the simulated nodes named with it: its first /// version, its root, and its ID. fn federation(parties: &[(&Party, &[&str])]) -> (Spec, Vec, Id) { let named: Vec<&Party> = parties.iter().map(|(party, _)| *party).collect(); let mut first = Spec::first(&named, &[]); for (spec, (_, nodes)) in first.parties.iter_mut().zip(parties) { spec.nodes = nodes.iter().map(|node| sim_key(node).id()).collect(); } let root = first.root(); let id = Id::of(&root); (first, root, id) } /// tv, lass and makefu. fn three() -> (Party, Party, Party) { ( Party::new("tv", 1), Party::new("lass", 2), Party::new("makefu", 3), ) } fn s(text: &str) -> Json { harness::str(text) } /// What a node of `party` keeps of its store, without a file (`--store-party`). fn holding(party: &str) -> Holding { Holding::open( None, Arc::new(Files::default()), None, Some(party.to_owned()), ) .expect("no file to read") } /// What a node of `party` keeps of its store, starting from the checkpoint `pin`, found in /// the document (`--store-checkpoint`). fn pinned(party: &str, pin: Id) -> Holding { Holding::open( None, Arc::new(Files::default()), Some(pin), Some(party.to_owned()), ) .expect("no file to read") } /// What a node of `party` keeps of its store, voting for its party by itself by `rules` /// with its key, its party's record in `dir`, begun before the federation `first` spells /// out: so it casts on the first version too. fn voting(party: &Party, rules: Rules, dir: &Path, first: &Spec) -> Holding { let record = dir.join(format!("{}.json", party.name)); let version = Version::read(&first.json()).expect("a version"); let mut begun = Record::begin(Id::of(&first.root()), &party.name, &version); begun.began = Id::of(b"before the federation").to_string(); begun.save(&record).expect("a record"); holding(&party.name).voting(Some(Voter { key: party.key.clone(), record, rules, })) } /// `config`, of a node of the swarm holding the store `id`, keeping `holding`: started /// from the store's `root` where given (`--store-root`), else joining by its ID /// (`--store-id`). fn of_the_store(config: &mut NodeConfig, id: Id, root: Option<&[u8]>, holding: Holding) { config.store = Some(id); config.holding = holding; if let Some(root) = root { config.initial_doc = events::started(Federation::NAME, root, &[]).expect("a root"); } } /// Starts `nodes`, each by its name with what it keeps of its store, from the store's /// `root`, the first one the others' seed, and waits until each holds the store, and /// until they synced: each took the others' handshakes, and holds their entries. async fn started(sim: &Sim, root: &[u8], nodes: Vec<(&str, Holding)>) -> Vec { started_with(sim, root, nodes, |_| {}).await } /// [`started`], each node's configuration adjusted by `adjust` too. async fn started_with( sim: &Sim, root: &[u8], nodes: Vec<(&str, Holding)>, adjust: fn(&mut NodeConfig), ) -> Vec { let id = Id::of(root); let first = nodes[0].0.to_owned(); let states: Vec = nodes .into_iter() .map(|(name, holding)| { let seeds: Vec<&str> = if name == first { Vec::new() } else { vec![first.as_str()] }; let root = root.to_vec(); sim.start_with( name, &seeds, Start::FromDocument(Origin::Root), move |config| { of_the_store(config, id, Some(&root), holding); adjust(config); }, ) }) .collect(); let all: Vec<&AppState> = states.iter().collect(); sim.wait_ready(&all, SOON).await; let all = &all; sim.wait_until("every node holds the store", SOON, move || async move { all.iter().all(|node| node.holding.tells_members()) }) .await; sim.wait_converged(all, LONG).await; states } /// Puts `events` on `node`, each in a change of its own. async fn put(node: &AppState, events: &[&[u8]]) { for event in events { held::write(node, None, event.to_vec()) .await .expect("an event put"); } } /// What `f` reads of `node`'s store, while it holds one. fn read(node: &AppState, f: impl FnOnce(&Store) -> T) -> Option { node.holding.with(f) } /// Whether `node` holds every event of `ids`. fn holds(node: &AppState, ids: &[Id]) -> bool { read(node, |store| ids.iter().all(|id| store.holds(id))) == Some(true) } /// The version current, as `node`'s store reads, by its hash. fn current(node: &AppState) -> Option { read(node, |store| store.state().current().hash) } /// The ID of the checkpoint `node`'s store holds. fn checkpoint(node: &AppState) -> Option { read(node, |store| Id::of(store.checkpoint())) } /// The version event writing `version` that `node`'s store took after its checkpoint. fn written(node: &AppState, version: Id) -> Option { read(node, |store| { store.taken().iter().copied().find(|id| { matches!( store.get(id).and_then(Event::read_taken), Some(Event::Version { version: writes, .. }) if writes == version ) }) }) .flatten() } /// The proposal on the current version that `party` proposed, as `node`'s store holds it: /// its hash, and the parties voting `true` on it, and `false`. type Ballot = (Id, BTreeSet, BTreeSet); fn ballot(node: &AppState, party: &str) -> Option { read(node, |store| { let state = store.state(); let signed = |id: &Id| match store.get(id).and_then(Event::read_taken) { Some(Event::Signed(signed)) => Some(signed), _ => None, }; state .ballots(&state.current().hash) .into_iter() .find_map(|(hash, ballot)| { let proposed = ballot .proposals .iter() .filter_map(signed) .any(|proposing| proposing.party == party); let (mut yes, mut no) = (BTreeSet::new(), BTreeSet::new()); for vote in ballot.votes.iter().filter_map(signed) { if let Does::Vote { yes: true, .. } = vote.does { yes.insert(vote.party); } else { no.insert(vote.party); } } proposed.then_some((hash, yes, no)) }) }) .flatten() } fn set(names: &[&str]) -> BTreeSet { names.iter().map(|name| (*name).to_owned()).collect() } /// What counts, as `node`'s store reads: the current version, the values at places, and /// the members; and the events it took after its checkpoint. type Counts = ( Id, BTreeMap, Vec<(String, String)>, BTreeSet, ); fn counts(node: &AppState) -> Option { read(node, |store| { let state = store.state(); ( state.current().hash, state.values(), state.members(), store.taken().iter().copied().collect(), ) }) } /// tv lists a and b, which start from the root; lass lists c, which joins by the store's /// ID; d is of lass too, which doesn't list it. Events put on a and on c reach a, b and /// c; d, which no member syncs with, gets nothing. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn the_events_put_on_a_member_reach_every_member_alone() { let sim = Sim::with_identity(81); let (tv, lass, _) = three(); let (_, root, id) = federation(&[(&tv, &["a", "b"]), (&lass, &["c"])]); let mut nodes = started( &sim, &root, vec![("a", holding("tv")), ("b", holding("tv"))], ) .await; nodes.push(sim.start_with("c", &["a"], Start::Join, |config| { of_the_store(config, id, None, holding("lass")) })); let [a, b, c] = [&nodes[0], &nodes[1], &nodes[2]]; let members = [a, b, c]; sim.wait_ready(&members, SOON).await; sim.wait_until("a, b and c hold the store", SOON, move || async move { members.iter().all(|node| node.holding.tells_members()) }) .await; let d = sim.start_with("d", &["a"], Start::Join, |config| { of_the_store(config, id, None, holding("lass")) }); let on_a = tv.write(&[id], "home/tv/x", s("1")); let on_c = lass.write(&[id], "home/lass/y", s("2")); let ids = [Id::of(&on_a), Id::of(&on_c)]; put(a, &[&on_a]).await; put(c, &[&on_c]).await; sim.wait_until("a, b and c hold both events", SOON, move || async move { members.iter().all(|node| holds(node, &ids)) }) .await; sim.wait_converged(&members, SOON).await; for node in members { assert_eq!(read(node, |store| store.state().values().len()), Some(2)); } // Long enough for d to have joined, were it let in. tokio::time::sleep(Duration::from_secs(3)).await; assert!(!d.is_ready().await); assert!(!d.holding.tells_members()); assert!(events::ids(&d.doc.read().await.doc).is_empty()); } /// b's HELLO reaches a, a's reply doesn't: a takes b's handshake and syncs with it, and b, /// which hasn't taken a's, reads none of it, but says HELLO again; once a's replies come /// through, they sync. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_member_whose_reply_was_lost_is_asked_for_its_handshake() { let sim = Sim::with_identity(91); let (tv, lass, _) = three(); let (_, root, id) = federation(&[(&tv, &["a"]), (&lass, &["b"])]); sim.net.set_link( "a", "b", LinkParams { drop: vec![MessageKind::HelloReply], ..LinkParams::default() }, ); let a = sim.start_with("a", &[], Start::FromDocument(Origin::Root), |config| { of_the_store(config, id, Some(&root), holding("tv")) }); let b = sim.start_with("b", &["a"], Start::FromDocument(Origin::Root), |config| { of_the_store(config, id, Some(&root), holding("lass")) }); sim.wait_ready(&[&a, &b], SOON).await; let (on_a, b_id) = (&a, b.local_id().to_owned()); sim.wait_until("a took b's handshake", SOON, move || { let b_id = b_id.clone(); async move { match on_a.get_peer_state(&b_id).await { Some(peer) => peer.lock().await.hello_state == HelloState::Established, None => false, } } }) .await; // What a sends meanwhile, b doesn't read. tokio::time::sleep(Duration::from_secs(2)).await; sim.net.set_link("a", "b", LinkParams::default()); sim.wait_converged(&[&a, &b], LONG).await; } /// Different events to different peers: tv, lass and makefu, a node each, apart while /// each is given its own: two writes of tv's at one place, a write of lass's, and a /// proposal and the votes that pass it, each on another node. Together, they read the /// same, the version that passed current, written by whichever held it passed. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn federation_attack_28() { let sim = Sim::with_identity(82); let (tv, lass, makefu) = three(); let (first, root, id) = federation(&[(&tv, &["a"]), (&lass, &["b"]), (&makefu, &["c"])]); let nodes = started( &sim, &root, vec![ ("a", holding("tv")), ("b", holding("lass")), ("c", holding("makefu")), ], ) .await; for name in ["a", "b", "c"] { sim.net.take_down(name); } let on = first.hash(); let mut next = first.after(on); next.rules.push(("names/x".to_owned(), s("tv"))); let proposal = makefu.propose(&[id], &next.json()); let given = [ [tv.write(&[id], "home/tv/x", s("a")), proposal.clone()], [ tv.write(&[id], "home/tv/x", s("b")), tv.vote(&[Id::of(&proposal)], on, next.hash(), true), ], [ lass.write(&[id], "home/lass/y", s("c")), lass.vote(&[Id::of(&proposal)], on, next.hash(), true), ], ]; for (node, events) in nodes.iter().zip(&given) { put(node, &[&events[0], &events[1]]).await; } let (all, writes) = (&nodes, [0, 1, 2].map(|i| Id::of(&given[i][0]))); sim.wait_until( "each holds the write it was given", SOON, move || async move { all.iter() .zip(writes) .all(|(node, write)| holds(node, &[write])) }, ) .await; // Apart, a and b read their own write at one place. let at = |node: &AppState| read(node, |store| store.state().value("home/tv/x").cloned()); assert_eq!(at(&nodes[0]), Some(Some(s("a")))); assert_eq!(at(&nodes[1]), Some(Some(s("b")))); for name in ["a", "b", "c"] { sim.net.bring_up(name); } let passed = next.hash(); sim.wait_until("every node reads the same", LONG, move || async move { let first = counts(&all[0]); first.as_ref().is_some_and(|counts| counts.0 == passed) && all.iter().all(|node| counts(node) == first) }) .await; let counts = counts(&nodes[0]).expect("a store"); assert!(writes.iter().all(|write| counts.3.contains(write))); } /// The version passing on the first version of tv, lass and makefu that only moves to a /// checkpoint at `heads`, proposed by tv, and backed by tv and lass: its spec, and the /// proposal and votes. fn checkpoint_at( (tv, lass, _): &(Party, Party, Party), first: &Spec, heads: &[Id], ) -> (Spec, [Vec; 3]) { let on = first.hash(); let mut next = first.after(on); next.checkpoint = Some(heads.to_vec()); let proposal = tv.propose(heads, &next.json()); let votes = [tv, lass].map(|party| party.vote(&[Id::of(&proposal)], on, next.hash(), true)); let [tv_yes, lass_yes] = votes; (next, [proposal, tv_yes, lass_yes]) } /// A node away while the others moved to a checkpoint and forgot what came before: c, /// back, builds the checkpoint from the document, which holds what came before, and /// reads what a and b read, what came after it too. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn store_attack_24() { let sim = Sim::with_identity(83); let parties = three(); let (tv, lass, makefu) = &parties; let (first, root, id) = federation(&[(tv, &["a"]), (lass, &["b"]), (makefu, &["c"])]); let nodes = started( &sim, &root, vec![ ("a", holding("tv")), ("b", holding("lass")), ("c", holding("makefu")), ], ) .await; sim.net.take_down("c"); let gone = tv.write(&[id], "home/tv/a", s("0")); let head = tv.write(&[Id::of(&gone)], "home/tv/a", s("1")); let (next, [proposal, tv_yes, lass_yes]) = checkpoint_at(&parties, &first, &[Id::of(&head)]); put(&nodes[0], &[&gone, &head, &proposal, &tv_yes]).await; put(&nodes[1], &[&lass_yes]).await; let (all, passed) = (&nodes, next.hash()); sim.wait_until( "a and b moved to the checkpoint", LONG, move || async move { all[..2] .iter() .all(|node| current(node) == Some(passed) && checkpoint(node) != Some(id)) }, ) .await; let after = tv.write(&[Id::of(&head)], "home/tv/b", s("2")); put(&nodes[0], &[&after]).await; let moved = checkpoint(&nodes[0]).expect("a store"); assert_eq!(checkpoint(&nodes[1]), Some(moved)); assert_eq!( read(&nodes[0], |store| store.holds(&Id::of(&gone))), Some(false) ); sim.net.bring_up("c"); let after = Id::of(&after); sim.wait_until( "c moved there, and holds what came after", LONG, move || async move { checkpoint(&all[2]) == Some(moved) && holds(&all[2], &[after]) && holds(&all[0], &[after]) }, ) .await; assert_eq!(counts(&nodes[2]), counts(&nodes[0])); assert!(events::ids(&nodes[2].doc.read().await.doc).contains(&Id::of(&gone))); } /// Keys voting twice part honest nodes: tv backs a proposal of its own with lass, and one /// of lass's with makefu; b, apart, takes the other version than a and c. Together, they /// stay parted, and b's operator has it drop what it took and take the other side's; or, /// where the versions named checkpoints, starts it again from the checkpoint the others /// moved to, pinned. No fork. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn federation_attack_49() { for (seed, checkpoints) in [(84, false), (85, true)] { parted(seed, checkpoints).await; } } async fn parted(seed: u64, checkpoints: bool) { let sim = Sim::with_identity(seed); let (tv, lass, makefu) = three(); let (first, root, id) = federation(&[(&tv, &["a"]), (&lass, &["b"]), (&makefu, &["c"])]); let nodes = started( &sim, &root, vec![ ("a", holding("tv")), ("b", holding("lass")), ("c", holding("makefu")), ], ) .await; sim.net.take_down("b"); let on = first.hash(); let (mut one, mut two) = (first.after(on), first.after(on)); two.rules.push(("names/x".to_owned(), s("tv"))); let heads = [ tv.write(&[id], "home/tv/a", s("1")), lass.write(&[id], "home/lass/b", s("2")), ]; if checkpoints { one.checkpoint = Some(vec![Id::of(&heads[0])]); two.checkpoint = Some(vec![Id::of(&heads[1])]); } let p1 = tv.propose(&[Id::of(&heads[0])], &one.json()); let p2 = lass.propose(&[Id::of(&heads[1])], &two.json()); let ones = [&tv, &lass].map(|party| party.vote(&[Id::of(&p1)], on, one.hash(), true)); let twos = [&tv, &makefu].map(|party| party.vote(&[Id::of(&p2)], on, two.hash(), true)); put(&nodes[0], &[&heads[0], &p1, &ones[0], &ones[1]]).await; put(&nodes[1], &[&heads[1], &p2, &twos[0], &twos[1]]).await; let (all, one_hash, two_hash) = (&nodes, one.hash(), two.hash()); sim.wait_until( "a and c take one version, b the other", LONG, move || async move { current(&all[0]) == Some(one_hash) && current(&all[2]) == Some(one_hash) && current(&all[1]) == Some(two_hash) }, ) .await; sim.net.bring_up("b"); let members: Vec<&AppState> = nodes.iter().collect(); sim.wait_converged(&members, LONG).await; // Parted: each keeps the version it took. assert_eq!(current(&nodes[0]), Some(one_hash)); assert_eq!(current(&nodes[1]), Some(two_hash)); assert_eq!(current(&nodes[2]), Some(one_hash)); let b = if checkpoints { let pin = checkpoint(&nodes[0]).expect("a store"); sim.restart_with("b", &["a"], Start::Join, move |config| { of_the_store(config, id, None, pinned("lass", pin)) }) } else { let took = written(&nodes[1], two_hash).expect("b wrote what it took"); let other = written(&nodes[0], one_hash).expect("a wrote what it took"); let dropped = nodes[1].holding.drop_taken(&[took]); assert!(matches!(dropped, Some(Ok(_))), "{dropped:?}"); assert!(nodes[1].holding.allow(&[other])); nodes[1].clone() }; let b = &b; sim.wait_until("b takes the others' version", LONG, move || async move { current(b) == Some(one_hash) }) .await; assert_eq!(checkpoint(b), checkpoint(&nodes[0])); } /// A party keeps a proposal alive that nobody else backs, to hold back automatic /// checkpoints: makefu proposes a version and backs it alone. a votes for tv by itself, /// proposing a checkpoint once its store took five events; b votes for lass, following /// tv. a proposes one beside makefu's, b backs it, and every node moves there. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn federation_attack_67() { let sim = Sim::with_identity(86); let (tv, lass, makefu) = three(); let (first, root, id) = federation(&[(&tv, &["a"]), (&lass, &["b"]), (&makefu, &["c"])]); let dir = crate::files::scratch_dir("sim-attack-67"); let proposing = Rules { propose_events: Some(5), ..Rules::default() }; let following = Rules { follow: vec!["tv".to_owned()], ..Rules::default() }; let nodes = started( &sim, &root, vec![ ("a", voting(&tv, proposing, &dir, &first)), ("b", voting(&lass, following, &dir, &first)), ("c", holding("makefu")), ], ) .await; let on = first.hash(); let mut other = first.after(on); other.rules.push(("names/x".to_owned(), s("makefu"))); let alive = makefu.propose(&[id], &other.json()); let backed = makefu.vote(&[Id::of(&alive)], on, other.hash(), true); put(&nodes[2], &[&alive, &backed]).await; let writes: Vec> = (1..=3) .map(|n| tv.write(&[id], &format!("home/tv/{n}"), s("1"))) .collect(); let writes: Vec<&[u8]> = writes.iter().map(Vec::as_slice).collect(); put(&nodes[0], &writes).await; let (all, kept_alive) = (&nodes, other.hash()); sim.wait_until( "every node moved to a checkpoint", LONG, move || async move { all.iter().all(|node| { let named = read(node, |store| { let current = store.state().current(); current.number == 1 && current.checkpoint.is_some() && current.hash != kept_alive }); named == Some(true) && checkpoint(node) != Some(id) }) }, ) .await; } /// A group naming many nodes, to be every node's neighbours: makefu lists five, tv two, /// and lass one, whose operator names tv's too (`--store-neighbour-parties`); each node /// keeps two neighbours by the graph among all members. a and b keep each other, and c /// one of them, whatever makefu's nodes are. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn store_attack_19() { let sim = Sim::with_identity(87); let (tv, lass, makefu) = three(); let many = ["m1", "m2", "m3", "m4", "m5"]; let (_, root, _) = federation(&[(&tv, &["a", "b"]), (&lass, &["c"]), (&makefu, &many)]); let mut named = vec![ ("a", holding("tv")), ("b", holding("tv")), ("c", holding("lass").near(vec!["tv".to_owned()])), ]; named.extend(many.iter().map(|name| (*name, holding("makefu")))); let nodes = started_with(&sim, &root, named, |config| config.neighbours_k = 2).await; let [a, b, c] = [&nodes[0], &nodes[1], &nodes[2]]; let (a_id, b_id) = (a.local_id().to_owned(), b.local_id().to_owned()); sim.wait_until( "a and b keep each other, and c one of them", LONG, move || { let (a_id, b_id) = (a_id.clone(), b_id.clone()); async move { let known = [a, b, c].iter().all(|node| topology::view_known(node)); known && topology::is_neighbour(a, &b_id) && topology::is_neighbour(b, &a_id) && (topology::is_neighbour(c, &a_id) || topology::is_neighbour(c, &b_id)) } }, ) .await; } /// A writer forked: x1 and x2, tv's, start apart from one document under one writer, /// and each puts an event: changes of one writer at one number. Together, the documents /// part, each refusing the other's changes; the events reach every node, put again from /// the changes refused, and every node reads the same. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn store_attack_10() { let sim = Sim::with_identity(88); let (tv, lass, _) = three(); let (_, root, id) = federation(&[(&tv, &["a", "x1", "x2"]), (&lass, &["b"])]); let mut nodes = started( &sim, &root, vec![("a", holding("tv")), ("b", holding("lass"))], ) .await; let one = events::started(Federation::NAME, &root, &[]).expect("a root"); for name in ["x1", "x2"] { sim.net.take_down(name); let doc = one.clone(); nodes.push(sim.start_with( name, &["a"], Start::FromDocument(Origin::Root), move |config| { of_the_store(config, id, None, holding("tv")); config.initial_doc = doc; }, )); } let twins = [&nodes[2], &nodes[3]]; sim.wait_ready(&twins, SOON).await; sim.wait_until("x1 and x2 hold the store", SOON, move || async move { twins.iter().all(|node| node.holding.tells_members()) }) .await; let given = [ tv.write(&[id], "home/tv/x1", s("1")), tv.write(&[id], "home/tv/x2", s("2")), ]; put(twins[0], &[&given[0]]).await; put(twins[1], &[&given[1]]).await; for name in ["x1", "x2"] { sim.net.bring_up(name); } let (all, ids) = (&nodes, given.each_ref().map(|event| Id::of(event))); sim.wait_until( "every node holds both events, and reads the same", Duration::from_secs(60), move || async move { let first = counts(&all[0]); all.iter() .all(|node| holds(node, &ids) && counts(node) == first) }, ) .await; // Each refuses the other's changes for good: their documents part. assert_ne!(super::heads(twins[0]).await, super::heads(twins[1]).await); } /// Two checkpoints proposed at once, by nodes voting by themselves: a for tv and b for /// lass, apart, each propose one once their stores took six events, and back it; c votes /// for makefu, following tv; each votes `false` on a checkpoint beside the one it backs. /// a and b, together, vote `false` on each other's: the votes split. With c, tv's passes /// and every node moves there; lass's is dead by the nays of tv and makefu, which their /// records hold. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn federation_attack_59() { let sim = Sim::with_identity(89); let (tv, lass, makefu) = three(); let (first, root, id) = federation(&[(&tv, &["a"]), (&lass, &["b"]), (&makefu, &["c"])]); let dir = crate::files::scratch_dir("sim-attack-59"); let proposing = Rules { propose_events: Some(6), nay_beside: true, ..Rules::default() }; let following = Rules { follow: vec!["tv".to_owned()], nay_beside: true, held_for: Duration::from_secs(3600), ..Rules::default() }; let nodes = started( &sim, &root, vec![ ("a", voting(&tv, proposing.clone(), &dir, &first)), ("b", voting(&lass, proposing, &dir, &first)), ("c", voting(&makefu, following, &dir, &first)), ], ) .await; for name in ["a", "b", "c"] { sim.net.take_down(name); } // Six writes of each, one on the other. for (node, party) in [(&nodes[0], &tv), (&nodes[1], &lass)] { let mut on = id; for n in 0..6 { let write = party.write(&[on], &format!("home/{}/{n}", party.name), s("1")); on = Id::of(&write); put(node, &[&write]).await; } } let all = &nodes; sim.wait_until( "a and b each back a checkpoint of their own", LONG, move || async move { [(0, "tv"), (1, "lass")].into_iter().all(|(i, party)| { ballot(&all[i], party).is_some_and(|(_, yes, _)| yes.contains(party)) }) }, ) .await; sim.net.bring_up("a"); sim.net.bring_up("b"); sim.wait_until( "a and b vote `false` on each other's", LONG, move || async move { all[..2].iter().all(|node| { ballot(node, "tv") .is_some_and(|(_, yes, no)| yes == set(&["tv"]) && no == set(&["lass"])) && ballot(node, "lass") .is_some_and(|(_, yes, no)| yes == set(&["lass"]) && no == set(&["tv"])) }) }, ) .await; let (passing, _, _) = ballot(&nodes[0], "tv").expect("tv's"); let (other, _, _) = ballot(&nodes[0], "lass").expect("lass's"); sim.net.bring_up("c"); sim.wait_until( "every node moves to tv's checkpoint", LONG, move || async move { all.iter().all(|node| current(node) == Some(passing)) }, ) .await; for party in ["tv", "makefu"] { let record = Record::load(&dir.join(format!("{party}.json"))).expect("its record"); let nay = record.votes.iter().any(|vote| { matches!( Event::read_taken(vote.as_bytes()), Some(Event::Signed(Signed { does: Does::Vote { proposal, yes: false, .. }, .. })) if proposal == other ) }); assert!(nay, "{party} voted no `false` on lass's checkpoint"); } } /// A fork whose root doesn't read as its heads do, leaving out tv's write, which counts /// there, beside the fork a node holding the history builds: b, holding it, builds the /// one and not the other. Each fork's founder starts its swarm from its root and the /// events it keeps, and a node of lass joins each by its ID, taking it on its operator's /// word, and reads what the root keeps: tv's write in the one, nothing in the other. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn federation_attack_33() { let sim = Sim::with_identity(90); let (tv, lass, _) = three(); let (first, root, id) = federation(&[(&tv, &["a"]), (&lass, &["b"])]); let nodes = started( &sim, &root, vec![("a", holding("tv")), ("b", holding("lass"))], ) .await; let write = tv.write(&[id], "home/tv/x", s("1")); let head = Id::of(&write); put(&nodes[0], &[&write]).await; let all = &nodes; sim.wait_until("a and b hold tv's write", SOON, move || async move { all.iter().all(|node| holds(node, &[head])) }) .await; // The forks' versions, after the first, naming their nodes anew. let forking = |named: [&str; 2]| { let mut spec = first.after(first.hash()); for (party, node) in spec.parties.iter_mut().zip(named) { party.nodes = vec![sim_key(node).id()]; party.nodes_since = 1; } spec }; let (honest, crafted) = (forking(["f", "j"]), forking(["g", "k"])); let file = read(&nodes[0], |store| { fork::make(store, &[head], &honest.json()) }) .expect("a store") .expect("a fork"); let (honest_root, kept) = fork::read_root_file(&file); assert_eq!(kept, vec![write.clone()]); let crafted_root = Checkpoint { at: vec![head], keeps: Vec::new(), version: Version::read(&crafted.json()).expect("a version"), } .bytes(); let builds = |root: &[u8]| { read(&nodes[1], |store| { fork::check(store, root).map(|check| check.builds) }) }; assert_eq!(builds(&honest_root), Some(Ok(true))); assert_eq!(builds(&crafted_root), Some(Ok(false))); let mut forks = Vec::new(); for ((founder, joiner), (root, kept)) in [("f", "j"), ("g", "k")].into_iter().zip([ (honest_root.clone(), kept), (crafted_root.clone(), Vec::new()), ]) { let forked = Id::of(&root); let started = sim.start_with( founder, &[], Start::FromDocument(Origin::Root), move |config| { of_the_store(config, forked, None, holding("tv")); config.initial_doc = events::started(Federation::NAME, &root, &kept).expect("a fork's root"); }, ); let joined = sim.start_with(joiner, &[founder], Start::Join, move |config| { of_the_store(config, forked, None, holding("lass")) }); forks.push((started, joined)); } let forks = &forks; sim.wait_until( "each joiner holds its fork's store", LONG, move || async move { forks .iter() .all(|(_, joined)| joined.holding.tells_members()) }, ) .await; let reads = |node: &AppState| { read(node, |store| { (store.id(), store.state().value("home/tv/x").cloned()) }) }; assert_eq!( reads(&forks[0].1), Some((Id::of(&honest_root), Some(s("1")))) ); assert_eq!(reads(&forks[1].1), Some((Id::of(&crafted_root), None))); } /// Every node of `nodes` on generation `generation` and ready, within `timeout`. async fn on_generation(sim: &Sim, nodes: &[AppState], generation: u64, timeout: Duration) { sim.wait_until( &format!("every node is ready on generation {generation}"), timeout, move || async move { for node in nodes { if node.local_epoch().generation != generation || !node.is_ready().await { return false; } } true }, ) .await; } /// The compaction `node` proposes: its ID. async fn compact(node: &AppState) -> String { let (status, report) = crate::remake::testing::start(node.clone(), serde_json::json!({ "kind": "compact" }), None) .await; assert!(status.is_success(), "proposal refused: {status} {report}"); report["id"].as_str().expect("a remake's ID").to_owned() } /// Events written beyond a base, on nodes that switch and on nodes that don't yet: put /// again in the new epoch, the same events, the same state; and no change refused. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn store_attack_12() { let sim = Sim::with_identity(91); let (tv, lass, makefu) = three(); let (_, root, id) = federation(&[(&tv, &["a"]), (&lass, &["b"]), (&makefu, &["c"])]); let nodes = started( &sim, &root, vec![ ("a", holding("tv")), ("b", holding("lass")), ("c", holding("makefu")), ], ) .await; let all = &nodes; let before = tv.write(&[id], "home/tv/before", s("1")); put(&nodes[0], &[&before]).await; let held = [Id::of(&before)]; sim.wait_until("every node holds tv's write", LONG, move || async move { all.iter().all(|node| holds(node, &held)) }) .await; let remake = compact(&nodes[0]).await; let remake = remake.as_str(); sim.wait_until("b and c hold it committed", LONG, move || async move { for node in &all[1..] { let committed = crate::remake::testing::report(node, remake) .await .is_some_and(|report| report["phase"] == "committed"); if !committed { return false; } } true }) .await; // Beyond its base, on nodes that haven't switched yet. let during = [ lass.write(&[Id::of(&before)], "home/lass/during", s("2")), makefu.write(&[id], "home/makefu/during", s("3")), ]; put(&nodes[1], &[&during[0]]).await; put(&nodes[2], &[&during[1]]).await; on_generation(&sim, &nodes, 1, Duration::from_secs(60)).await; let after = tv.write(&[Id::of(&during[0])], "home/tv/after", s("4")); put(&nodes[0], &[&after]).await; let every = [&before, &during[0], &during[1], &after].map(|event| Id::of(event)); sim.wait_until("every node holds every event", LONG, move || async move { all.iter().all(|node| holds(node, &every)) }) .await; let read: Vec> = nodes.iter().map(counts).collect(); assert!(read.iter().all(|counts| counts == &read[0]), "{read:?}"); for node in &nodes { let refused = crate::locks::MutexExt::lock_or_recover(&*node.screening) .refused() .count(); assert_eq!(refused, 0, "a change refused"); } }