//! The store a node of a swarm holding one holds (`docs/store.md`, What the store takes //! from the document): read from its document, each event taken as the store would take //! it from anywhere, and by the node's own: what it refused by what it took before, and //! what its operator had it drop, which it keeps in a file of its own (`--store-file`). //! It keeps nothing else of the store across a restart: whenever it starts, it reads the //! store anew from its document, from the root. //! //! A worker reads the events the document holds that it hasn't read yet, whenever the //! document changes (`run_worker`). Which change put an event, and how often, changes //! nothing; nor does another document (a drop, a rejoin), whose events it reads as it //! comes, the store going on. use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use tracing::{error, info, warn}; use super::fold::Federation; use super::store::Id; use super::store::memory::{Offered, Store}; use crate::document::{ReadDoc, events, meta, store}; use crate::files::{FileKind, Files}; use crate::locks::MutexExt; /// What a node keeps of its own of its store across restarts, as its file holds it: the /// events it refused by what it took before, and those its operator had it drop, by ID. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct Kept { #[serde(default)] refused: BTreeSet, #[serde(default)] dropped: BTreeSet, } impl Kept { fn of(own: &[Id], dropped: &[Id]) -> Self { Kept { refused: own.iter().map(Id::to_string).collect(), dropped: dropped.iter().map(Id::to_string).collect(), } } /// The IDs it holds: what was refused, and what dropped. fn ids(&self) -> Option<(Vec, Vec)> { let ids = |set: &BTreeSet| { set.iter() .map(|id| Id::from_hex(id)) .collect::>>() }; Some((ids(&self.refused)?, ids(&self.dropped)?)) } } /// The store a node holds, if its swarm holds one, and the file it keeps its own in. #[derive(Default)] pub(crate) struct Holding { file: Option<(PathBuf, Arc)>, // A std mutex: never held while waiting for the document. inner: Mutex, } #[derive(Default)] struct Inner { /// None until a document holding a store holds its root. store: Option>, /// The document read from (`Replica::epoch_seq`), and its events read. epoch_seq: Option, read: HashSet, /// What the file holds, or will once written. kept: Kept, /// The root that is none, logged once. none: Option, } /// What a document holds that its store wasn't offered from it. pub(crate) struct Unread { epoch_seq: u32, /// The root, where the store isn't started yet. root: Option>, events: BTreeMap>, } impl Holding { /// Keeping the node's own in the file at `path`, if any, which holds what it kept /// before: a missing file nothing. One that doesn't read is an error: the node would /// take what it refused. pub(crate) fn open(path: Option, files: Arc) -> Result { let Some(path) = path else { return Ok(Holding::default()); }; let kept = match files .read(FileKind::Store, &path) .with_context(|| format!("Reading the store file {}", path.display()))? { None => Kept::default(), Some(bytes) => match serde_json::from_slice::(&bytes) { Ok(kept) if kept.ids().is_some() => kept, _ => bail!( "The store file {} doesn't read: move it away only knowing the node may then take what it refused", path.display() ), }, }; Ok(Holding { file: Some((path, files)), inner: Mutex::new(Inner { kept, ..Inner::default() }), }) } /// What `doc` (the document `epoch_seq`) holds that the store wasn't offered from it, /// if it holds a store: everything, from another document. pub(crate) fn unread(&self, doc: &impl ReadDoc, epoch_seq: u32) -> Option { if !matches!( crate::access::policy::of(doc), crate::access::policy::Policy::Store(_) ) { return None; } let swarm = match store::read_plain(doc, &meta::path(&[meta::SWARM])) { Ok(Some(serde_json::Value::String(id))) => Id::from_hex(&id)?, _ => return None, }; let inner = self.inner.lock_or_recover(); let same = inner.epoch_seq == Some(epoch_seq); let events = events::ids(doc) .into_iter() .filter(|id| !same || !inner.read.contains(id)) .filter_map(|id| Some((id, events::bytes(doc, &id)?))) .collect(); let root = match &inner.store { None => Some(events::bytes(doc, &swarm)?), Some(_) => None, }; Some(Unread { epoch_seq, root, events, }) } /// Offers the store what was `unread`: started from the root first, by what the node /// kept of its own. What it refuses of its own from then on, it keeps. pub(crate) fn read(&self, unread: Unread) { let mut inner = self.inner.lock_or_recover(); let inner = &mut *inner; if inner.epoch_seq != Some(unread.epoch_seq) { inner.epoch_seq = Some(unread.epoch_seq); inner.read.clear(); } if inner.store.is_none() { let Some(root) = unread.root else { return; }; let id = Id::of(&root); let (own, dropped) = inner.kept.ids().unwrap_or_default(); match Store::new(root, Vec::new()) { Ok(store) => { info!(store = %id, refused = own.len(), dropped = dropped.len(), "Reading the store from the document"); inner.store = Some(store.refusing(&own, &dropped)); } Err(why) => { if inner.none.replace(id) != Some(id) { error!(store = %id, why = %why.0, "The document's root is no store's: not reading it"); } return; } } } let store = inner.store.as_mut().expect("started"); let (mut taken, mut refused) = (0, 0); for (id, bytes) in unread.events { inner.read.insert(id); match store.offer(bytes) { Offered::Taken(ids) => taken += ids.len(), Offered::Refused(why) => { refused += 1; info!(event = %id, why = ?why, "The store refuses an event the document holds"); } Offered::Held | Offered::Waiting => {} } } if taken > 0 || refused > 0 { info!(taken, refused, "Read events from the document"); } let (own, dropped) = store.own(); let kept = Kept::of(&own, &dropped); if kept != inner.kept { for id in kept.refused.difference(&inner.kept.refused) { warn!(event = %id, "Refused an event its history takes, by what this node took before: kept so"); } inner.kept = kept; self.keep(&inner.kept); } } /// Writes what the node keeps of its own to its file, if it has one. fn keep(&self, kept: &Kept) { let Some((path, files)) = &self.file else { return; }; let written = serde_json::to_vec_pretty(kept) .map_err(std::io::Error::from) .and_then(|mut bytes| { bytes.push(b'\n'); files.write_atomic(FileKind::Store, path, &bytes) }); if let Err(error) = written { error!(path = %path.display(), error = %error, "Couldn't write the store file: a restart may take what this node refused"); } } /// `f` of the store, where the node holds one. pub(crate) fn with(&self, f: impl FnOnce(&Store) -> T) -> Option { self.inner.lock_or_recover().store.as_ref().map(f) } } /// Reads the store from the document whenever it changes. pub(crate) async fn run_worker(state: crate::state::AppState) { let mut changed = state.change_tx.subscribe(); loop { let holding = Arc::clone(&state.holding); let doc = Arc::clone(&state.doc); let read = tokio::task::spawn_blocking(move || { // Read from the document under its lock; taken without it. let unread = { let replica = doc.blocking_read(); holding.unread(&replica.doc, replica.epoch_seq) }; if let Some(unread) = unread { holding.read(unread); } }); if let Err(error) = read.await { error!(error = %error, "Reading the store from the document failed"); } if changed.changed().await.is_err() { return; } } } #[cfg(test)] mod tests { use super::*; use crate::document::typed::{TimestampFormat, Val}; use crate::document::{Document, ScalarValue}; use crate::federation::event::Checkpoint; use crate::federation::harness::{self, Party, Spec}; use crate::federation::store::Fold; use crate::federation::version::Version; fn root_of(spec: &Spec) -> Vec { Checkpoint { at: Vec::new(), keeps: Vec::new(), version: Version::read(&spec.json()).expect("a version"), } .bytes() } fn put(doc: &mut Document, event: &[u8]) { crate::document::testing::change(doc, |doc| { store::write_val_path( doc, &events::event(&Id::of(event)), &Val::Scalar(ScalarValue::Bytes(event.to_vec())), store::Writing::sticky(TimestampFormat::default()), ) .unwrap(); }); } fn read(holding: &Holding, doc: &Document, epoch_seq: u32) { holding.read(holding.unread(doc, epoch_seq).unwrap()); } #[test] fn the_store_takes_the_events_the_document_holds() { let tv = Party::new("tv", 1); let first = Spec::first(&[&tv], &[]); let root = root_of(&first); let mut doc = events::started(Federation::NAME, &root).unwrap(); let holding = Holding::default(); assert!(holding.unread(&Document::new(), 0).is_none()); read(&holding, &doc, 0); assert_eq!(holding.with(|store| store.id()), Some(Id::of(&root))); let write = tv.write(&[Id::of(&root)], "home/tv/x", harness::str("1")); let on = tv.write(&[Id::of(&write)], "home/tv/x", harness::str("2")); // What it builds on comes later: it waits for it. put(&mut doc, &on); read(&holding, &doc, 0); assert_eq!( holding.with(|store| store.is_waiting(&Id::of(&on))), Some(true) ); put(&mut doc, &write); read(&holding, &doc, 0); assert_eq!( holding.with(|store| store.taken().to_vec()), Some(vec![Id::of(&write), Id::of(&on)]) ); // Another document holding them: nothing new. let other = doc.fork(); read(&holding, &other, 1); assert_eq!(holding.with(|store| store.taken().len()), Some(2)); } /// Two version events changing version 0, each backed by tv and one other. #[test] fn a_node_read_anew_refuses_what_it_refused_of_its_own() { let (tv, lass, makefu) = ( Party::new("tv", 1), Party::new("lass", 2), Party::new("makefu", 3), ); let first = Spec::first(&[&tv, &lass, &makefu], &[]); let root = root_of(&first); let on = first.hash(); let one = first.after(on); let mut two = first.after(on); two.rules.push(("names/x".to_owned(), harness::str("tv"))); let p1 = tv.propose(&[Id::of(&root)], &one.json()); let p2 = lass.propose(&[Id::of(&root)], &two.json()); let votes = [ tv.vote(&[Id::of(&p1)], on, one.hash(), true), lass.vote(&[Id::of(&p1)], on, one.hash(), true), tv.vote(&[Id::of(&p2)], on, two.hash(), true), makefu.vote(&[Id::of(&p2)], on, two.hash(), true), ]; let v1 = harness::version_event(&[Id::of(&votes[0]), Id::of(&votes[1])], one.hash()); let v2 = harness::version_event(&[Id::of(&votes[2]), Id::of(&votes[3])], two.hash()); let before: Vec<&Vec> = [&p1, &p2].into_iter().chain(&votes).collect(); let taken = |holding: &Holding| holding.with(|store| store.taken().to_vec()).unwrap(); // The one a store reading all of them at once takes, and the other, which a node // took as it came first. let mut all = events::started(Federation::NAME, &root).unwrap(); for event in before.iter().copied().chain([&v1, &v2]) { put(&mut all, event); } let forgetting = Holding::default(); read(&forgetting, &all, 0); let (later, first_taken) = if taken(&forgetting).contains(&Id::of(&v1)) { (v1, v2) } else { (v2, v1) }; let dir = crate::files::scratch_dir("held"); let path = dir.join("store.json"); let files = Arc::new(Files::default()); let holding = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap(); let mut doc = events::started(Federation::NAME, &root).unwrap(); for event in before.iter().copied().chain([&first_taken]) { put(&mut doc, event); } read(&holding, &doc, 0); put(&mut doc, &later); read(&holding, &doc, 0); assert!(taken(&holding).contains(&Id::of(&first_taken))); assert!(!taken(&holding).contains(&Id::of(&later))); // Restarted: the store read anew, all at once, refuses the later as before. let anew = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap(); read(&anew, &all, 0); assert!(taken(&anew).contains(&Id::of(&first_taken))); assert!(!taken(&anew).contains(&Id::of(&later))); std::fs::write(&path, b"{\"refused\": [\"not an ID\"]}").unwrap(); assert!(Holding::open(Some(path), files).is_err()); } }