From c9707297bd393f69461c781a38a5ddb017dced60 Mon Sep 17 00:00:00 2001 From: tv Date: Mon, 16 Nov 2026 19:54:35 +0000 Subject: federation: the store read from the document A node of a swarm holding a store holds the store, read from its document (`docs/store.md`, What the store takes from the document): `federation::held`. Whenever the document changes, a worker reads the events it holds that the store wasn't offered from it, under the document's lock, and offers them to the store without it, each taken as from anywhere: once its dependencies are held, judged at its history. Another document (a drop, a rejoin) is read as it comes, the store going on. The node reads its store anew from its document whenever it starts, from the root, by what is its own of it: the events it refused by what it took before, and those its operator had it drop, which it keeps in a file (`--store-file`), written whenever that changes, so that a store read anew in another order refuses what it refused before. A file that doesn't read stops the node from starting; without one, a restart may take what it refused. --- src/federation/held.rs | 379 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100644 src/federation/held.rs (limited to 'src/federation/held.rs') diff --git a/src/federation/held.rs b/src/federation/held.rs new file mode 100644 index 0000000..28f4ca5 --- /dev/null +++ b/src/federation/held.rs @@ -0,0 +1,379 @@ +//! The store a node of a swarm holding one holds (`docs/store.md`, What the store takes +//! from the document): read from its document, each event taken as the store would take +//! it from anywhere, and by the node's own: what it refused by what it took before, and +//! what its operator had it drop, which it keeps in a file of its own (`--store-file`). +//! It keeps nothing else of the store across a restart: whenever it starts, it reads the +//! store anew from its document, from the root. +//! +//! A worker reads the events the document holds that it hasn't read yet, whenever the +//! document changes (`run_worker`). Which change put an event, and how often, changes +//! nothing; nor does another document (a drop, a rejoin), whose events it reads as it +//! comes, the store going on. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use anyhow::{Context, Result, bail}; +use serde::{Deserialize, Serialize}; +use tracing::{error, info, warn}; + +use super::fold::Federation; +use super::store::Id; +use super::store::memory::{Offered, Store}; +use crate::document::{ReadDoc, events, meta, store}; +use crate::files::{FileKind, Files}; +use crate::locks::MutexExt; + +/// What a node keeps of its own of its store across restarts, as its file holds it: the +/// events it refused by what it took before, and those its operator had it drop, by ID. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Kept { + #[serde(default)] + refused: BTreeSet, + #[serde(default)] + dropped: BTreeSet, +} + +impl Kept { + fn of(own: &[Id], dropped: &[Id]) -> Self { + Kept { + refused: own.iter().map(Id::to_string).collect(), + dropped: dropped.iter().map(Id::to_string).collect(), + } + } + + /// The IDs it holds: what was refused, and what dropped. + fn ids(&self) -> Option<(Vec, Vec)> { + let ids = |set: &BTreeSet| { + set.iter() + .map(|id| Id::from_hex(id)) + .collect::>>() + }; + Some((ids(&self.refused)?, ids(&self.dropped)?)) + } +} + +/// The store a node holds, if its swarm holds one, and the file it keeps its own in. +#[derive(Default)] +pub(crate) struct Holding { + file: Option<(PathBuf, Arc)>, + // A std mutex: never held while waiting for the document. + inner: Mutex, +} + +#[derive(Default)] +struct Inner { + /// None until a document holding a store holds its root. + store: Option>, + /// The document read from (`Replica::epoch_seq`), and its events read. + epoch_seq: Option, + read: HashSet, + /// What the file holds, or will once written. + kept: Kept, + /// The root that is none, logged once. + none: Option, +} + +/// What a document holds that its store wasn't offered from it. +pub(crate) struct Unread { + epoch_seq: u32, + /// The root, where the store isn't started yet. + root: Option>, + events: BTreeMap>, +} + +impl Holding { + /// Keeping the node's own in the file at `path`, if any, which holds what it kept + /// before: a missing file nothing. One that doesn't read is an error: the node would + /// take what it refused. + pub(crate) fn open(path: Option, files: Arc) -> Result { + let Some(path) = path else { + return Ok(Holding::default()); + }; + let kept = match files + .read(FileKind::Store, &path) + .with_context(|| format!("Reading the store file {}", path.display()))? + { + None => Kept::default(), + Some(bytes) => match serde_json::from_slice::(&bytes) { + Ok(kept) if kept.ids().is_some() => kept, + _ => bail!( + "The store file {} doesn't read: move it away only knowing the node may then take what it refused", + path.display() + ), + }, + }; + Ok(Holding { + file: Some((path, files)), + inner: Mutex::new(Inner { + kept, + ..Inner::default() + }), + }) + } + + /// What `doc` (the document `epoch_seq`) holds that the store wasn't offered from it, + /// if it holds a store: everything, from another document. + pub(crate) fn unread(&self, doc: &impl ReadDoc, epoch_seq: u32) -> Option { + if !matches!( + crate::access::policy::of(doc), + crate::access::policy::Policy::Store(_) + ) { + return None; + } + let swarm = match store::read_plain(doc, &meta::path(&[meta::SWARM])) { + Ok(Some(serde_json::Value::String(id))) => Id::from_hex(&id)?, + _ => return None, + }; + let inner = self.inner.lock_or_recover(); + let same = inner.epoch_seq == Some(epoch_seq); + let events = events::ids(doc) + .into_iter() + .filter(|id| !same || !inner.read.contains(id)) + .filter_map(|id| Some((id, events::bytes(doc, &id)?))) + .collect(); + let root = match &inner.store { + None => Some(events::bytes(doc, &swarm)?), + Some(_) => None, + }; + Some(Unread { + epoch_seq, + root, + events, + }) + } + + /// Offers the store what was `unread`: started from the root first, by what the node + /// kept of its own. What it refuses of its own from then on, it keeps. + pub(crate) fn read(&self, unread: Unread) { + let mut inner = self.inner.lock_or_recover(); + let inner = &mut *inner; + if inner.epoch_seq != Some(unread.epoch_seq) { + inner.epoch_seq = Some(unread.epoch_seq); + inner.read.clear(); + } + if inner.store.is_none() { + let Some(root) = unread.root else { + return; + }; + let id = Id::of(&root); + let (own, dropped) = inner.kept.ids().unwrap_or_default(); + match Store::new(root, Vec::new()) { + Ok(store) => { + info!(store = %id, refused = own.len(), dropped = dropped.len(), "Reading the store from the document"); + inner.store = Some(store.refusing(&own, &dropped)); + } + Err(why) => { + if inner.none.replace(id) != Some(id) { + error!(store = %id, why = %why.0, "The document's root is no store's: not reading it"); + } + return; + } + } + } + let store = inner.store.as_mut().expect("started"); + let (mut taken, mut refused) = (0, 0); + for (id, bytes) in unread.events { + inner.read.insert(id); + match store.offer(bytes) { + Offered::Taken(ids) => taken += ids.len(), + Offered::Refused(why) => { + refused += 1; + info!(event = %id, why = ?why, "The store refuses an event the document holds"); + } + Offered::Held | Offered::Waiting => {} + } + } + if taken > 0 || refused > 0 { + info!(taken, refused, "Read events from the document"); + } + let (own, dropped) = store.own(); + let kept = Kept::of(&own, &dropped); + if kept != inner.kept { + for id in kept.refused.difference(&inner.kept.refused) { + warn!(event = %id, "Refused an event its history takes, by what this node took before: kept so"); + } + inner.kept = kept; + self.keep(&inner.kept); + } + } + + /// Writes what the node keeps of its own to its file, if it has one. + fn keep(&self, kept: &Kept) { + let Some((path, files)) = &self.file else { + return; + }; + let written = serde_json::to_vec_pretty(kept) + .map_err(std::io::Error::from) + .and_then(|mut bytes| { + bytes.push(b'\n'); + files.write_atomic(FileKind::Store, path, &bytes) + }); + if let Err(error) = written { + error!(path = %path.display(), error = %error, "Couldn't write the store file: a restart may take what this node refused"); + } + } + + /// `f` of the store, where the node holds one. + pub(crate) fn with(&self, f: impl FnOnce(&Store) -> T) -> Option { + self.inner.lock_or_recover().store.as_ref().map(f) + } +} + +/// Reads the store from the document whenever it changes. +pub(crate) async fn run_worker(state: crate::state::AppState) { + let mut changed = state.change_tx.subscribe(); + loop { + let holding = Arc::clone(&state.holding); + let doc = Arc::clone(&state.doc); + let read = tokio::task::spawn_blocking(move || { + // Read from the document under its lock; taken without it. + let unread = { + let replica = doc.blocking_read(); + holding.unread(&replica.doc, replica.epoch_seq) + }; + if let Some(unread) = unread { + holding.read(unread); + } + }); + if let Err(error) = read.await { + error!(error = %error, "Reading the store from the document failed"); + } + if changed.changed().await.is_err() { + return; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::document::typed::{TimestampFormat, Val}; + use crate::document::{Document, ScalarValue}; + use crate::federation::event::Checkpoint; + use crate::federation::harness::{self, Party, Spec}; + use crate::federation::store::Fold; + use crate::federation::version::Version; + + fn root_of(spec: &Spec) -> Vec { + Checkpoint { + at: Vec::new(), + keeps: Vec::new(), + version: Version::read(&spec.json()).expect("a version"), + } + .bytes() + } + + fn put(doc: &mut Document, event: &[u8]) { + crate::document::testing::change(doc, |doc| { + store::write_val_path( + doc, + &events::event(&Id::of(event)), + &Val::Scalar(ScalarValue::Bytes(event.to_vec())), + store::Writing::sticky(TimestampFormat::default()), + ) + .unwrap(); + }); + } + + fn read(holding: &Holding, doc: &Document, epoch_seq: u32) { + holding.read(holding.unread(doc, epoch_seq).unwrap()); + } + + #[test] + fn the_store_takes_the_events_the_document_holds() { + let tv = Party::new("tv", 1); + let first = Spec::first(&[&tv], &[]); + let root = root_of(&first); + let mut doc = events::started(Federation::NAME, &root).unwrap(); + let holding = Holding::default(); + assert!(holding.unread(&Document::new(), 0).is_none()); + read(&holding, &doc, 0); + assert_eq!(holding.with(|store| store.id()), Some(Id::of(&root))); + let write = tv.write(&[Id::of(&root)], "home/tv/x", harness::str("1")); + let on = tv.write(&[Id::of(&write)], "home/tv/x", harness::str("2")); + // What it builds on comes later: it waits for it. + put(&mut doc, &on); + read(&holding, &doc, 0); + assert_eq!( + holding.with(|store| store.is_waiting(&Id::of(&on))), + Some(true) + ); + put(&mut doc, &write); + read(&holding, &doc, 0); + assert_eq!( + holding.with(|store| store.taken().to_vec()), + Some(vec![Id::of(&write), Id::of(&on)]) + ); + // Another document holding them: nothing new. + let other = doc.fork(); + read(&holding, &other, 1); + assert_eq!(holding.with(|store| store.taken().len()), Some(2)); + } + + /// Two version events changing version 0, each backed by tv and one other. + #[test] + fn a_node_read_anew_refuses_what_it_refused_of_its_own() { + let (tv, lass, makefu) = ( + Party::new("tv", 1), + Party::new("lass", 2), + Party::new("makefu", 3), + ); + let first = Spec::first(&[&tv, &lass, &makefu], &[]); + let root = root_of(&first); + let on = first.hash(); + let one = first.after(on); + let mut two = first.after(on); + two.rules.push(("names/x".to_owned(), harness::str("tv"))); + let p1 = tv.propose(&[Id::of(&root)], &one.json()); + let p2 = lass.propose(&[Id::of(&root)], &two.json()); + let votes = [ + tv.vote(&[Id::of(&p1)], on, one.hash(), true), + lass.vote(&[Id::of(&p1)], on, one.hash(), true), + tv.vote(&[Id::of(&p2)], on, two.hash(), true), + makefu.vote(&[Id::of(&p2)], on, two.hash(), true), + ]; + let v1 = harness::version_event(&[Id::of(&votes[0]), Id::of(&votes[1])], one.hash()); + let v2 = harness::version_event(&[Id::of(&votes[2]), Id::of(&votes[3])], two.hash()); + let before: Vec<&Vec> = [&p1, &p2].into_iter().chain(&votes).collect(); + let taken = |holding: &Holding| holding.with(|store| store.taken().to_vec()).unwrap(); + // The one a store reading all of them at once takes, and the other, which a node + // took as it came first. + let mut all = events::started(Federation::NAME, &root).unwrap(); + for event in before.iter().copied().chain([&v1, &v2]) { + put(&mut all, event); + } + let forgetting = Holding::default(); + read(&forgetting, &all, 0); + let (later, first_taken) = if taken(&forgetting).contains(&Id::of(&v1)) { + (v1, v2) + } else { + (v2, v1) + }; + + let dir = crate::files::scratch_dir("held"); + let path = dir.join("store.json"); + let files = Arc::new(Files::default()); + let holding = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap(); + let mut doc = events::started(Federation::NAME, &root).unwrap(); + for event in before.iter().copied().chain([&first_taken]) { + put(&mut doc, event); + } + read(&holding, &doc, 0); + put(&mut doc, &later); + read(&holding, &doc, 0); + assert!(taken(&holding).contains(&Id::of(&first_taken))); + assert!(!taken(&holding).contains(&Id::of(&later))); + + // Restarted: the store read anew, all at once, refuses the later as before. + let anew = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap(); + read(&anew, &all, 0); + assert!(taken(&anew).contains(&Id::of(&first_taken))); + assert!(!taken(&anew).contains(&Id::of(&later))); + + std::fs::write(&path, b"{\"refused\": [\"not an ID\"]}").unwrap(); + assert!(Holding::open(Some(path), files).is_err()); + } +} -- cgit v1.3.1