diff options
| author | tv <tv@krebsco.de> | 2026-11-16 20:06:42 +0000 |
|---|---|---|
| committer | tv <tv@krebsco.de> | 2026-11-16 21:26:29 +0100 |
| commit | 9d18983e4b643eac04de49a42a6f29805570dba3 (patch) | |
| tree | 90bff68e2b2ec5e3bbbd3fee974c53ff44dcbe32 /src | |
| parent | d8cec1cd61c63659d583168244f8f51ec550b3be (diff) | |
federation: a pinned checkpoint, found in the document
Once a store moved to a checkpoint, a node reading it from the root could take what was
grown anew from the root since, by keys of versions long replaced: so its operator pins
the checkpoint (`--store-checkpoint`, by its ID), and the node reads its store from
there whenever it starts (`docs/store.md`, Pinned checkpoints). It finds it in its
document: at heads a proposal's version names, built from their history read from the
root, as any node builds the checkpoint it moves to (`Store::checkpoint_at`), the one
whose ID is pinned. Until the document holds it, the node holds no store, and says so
once.
Diffstat (limited to 'src')
| -rw-r--r-- | src/config/mod.rs | 20 | ||||
| -rw-r--r-- | src/federation/held.rs | 139 | ||||
| -rw-r--r-- | src/main.rs | 8 |
3 files changed, 154 insertions, 13 deletions
diff --git a/src/config/mod.rs b/src/config/mod.rs index afd9c4e..e2557cd 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -224,6 +224,14 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ None, "With store-root or store-id: where the node keeps what is its own of its store across restarts, written whenever it changes: the events it refused by what it took before (of two versions changing one, the second), and those its operator had it drop. It reads its store anew from its document whenever it starts: without the file, it may then take what it refused. A file that doesn't read stops the node from starting.", ), + #[cfg(feature = "federation")] + option( + "store-checkpoint", + Kind::Text, + "ID", + None, + "With store-root or store-id: a checkpoint of the store, by its ID (hex), from which the node reads its store whenever it starts, instead of from the root: once a store moved to a checkpoint, a node reading from the root could take what was grown anew from it since, by keys of versions long replaced (docs/federation.md, Checkpoints). The node finds it in its document, at heads a proposal names, built from their history as any node builds it; until then, it holds no store.", + ), option( "strict-keys", Kind::List, @@ -661,6 +669,8 @@ pub(crate) struct Settings { pub(crate) store_id: Option<String>, #[cfg(feature = "federation")] pub(crate) store_file: Option<PathBuf>, + #[cfg(feature = "federation")] + pub(crate) store_checkpoint: Option<String>, #[serde(default)] pub(crate) strict_keys: Vec<String>, @@ -1120,6 +1130,11 @@ impl Settings { { bail!("store-id: {id:?} isn't a store's ID (64 hex digits)"); } + if let Some(id) = &self.store_checkpoint + && crate::federation::store::Id::from_hex(id).is_none() + { + bail!("store-checkpoint: {id:?} isn't a checkpoint's ID (64 hex digits)"); + } let store = match (&self.store_root, &self.store_id) { (Some(_), Some(_)) => { bail!("store-root names the store's ID itself; unset store-id") @@ -1131,6 +1146,9 @@ impl Settings { "store-file keeps what is the node's own of a store: set store-root or store-id" ) } + (None, None) if self.store_checkpoint.is_some() => { + bail!("store-checkpoint pins a checkpoint of a store: set store-root or store-id") + } (None, None) => return Ok(()), }; let first_change = [ @@ -1186,6 +1204,8 @@ impl Settings { ("store-id", self.store_id.is_some()), #[cfg(feature = "federation")] ("store-file", self.store_file.is_some()), + #[cfg(feature = "federation")] + ("store-checkpoint", self.store_checkpoint.is_some()), ("alias", self.alias.is_some()), ("dashboard-url", self.dashboard_url.is_some()), ("debug-api", self.debug_api), diff --git a/src/federation/held.rs b/src/federation/held.rs index 5c0d8c8..8f48c65 100644 --- a/src/federation/held.rs +++ b/src/federation/held.rs @@ -3,7 +3,10 @@ //! it from anywhere, and by the node's own: what it refused by what it took before, and //! what its operator had it drop, which it keeps in a file of its own (`--store-file`). //! It keeps nothing else of the store across a restart: whenever it starts, it reads the -//! store anew from its document, from the root. +//! store anew from its document, from the root, or from a checkpoint its operator pins +//! (`--store-checkpoint`), found in the document (`found`): once its store moved, a node +//! reading from the root could take what was grown anew from it since, by keys of +//! versions long replaced. //! //! A worker reads the events the document holds that it hasn't read yet, whenever the //! document changes (`run_worker`). Which change put an event, and how often, changes @@ -26,9 +29,11 @@ use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use tracing::{error, info, warn}; +use super::event::{Does, Event, Signed}; use super::fold::Federation; -use super::store::Id; -use super::store::memory::{Offered, Store}; +use super::store::memory::{Built, Offered, Store}; +use super::store::{Fold, Id}; +use super::version::Version; use crate::document::{Change, ChangeHash, ReadDoc, ScalarValue, events, meta, store}; use crate::files::{FileKind, Files}; use crate::locks::MutexExt; @@ -72,10 +77,12 @@ impl Kept { } } -/// The store a node holds, if its swarm holds one, and the file it keeps its own in. +/// The store a node holds, if its swarm holds one, the file it keeps its own in, and the +/// checkpoint its operator pins. #[derive(Default)] pub(crate) struct Holding { file: Option<(PathBuf, Arc<Files>)>, + pin: Option<Id>, // A std mutex: never held while waiting for the document. inner: Mutex<Inner>, } @@ -91,6 +98,8 @@ struct Inner { kept: Kept, /// The root that is none, logged once. none: Option<Id>, + /// The pinned checkpoint not found in the document, logged once. + unfound: bool, /// The refused changes looked at, of those screening keeps (`look`). looked: HashSet<ChangeHash>, } @@ -106,10 +115,13 @@ pub(crate) struct Unread { impl Holding { /// Keeping the node's own in the file at `path`, if any, which holds what it kept /// before: a missing file nothing. One that doesn't read is an error: the node would - /// take what it refused. - pub(crate) fn open(path: Option<PathBuf>, files: Arc<Files>) -> Result<Self> { + /// take what it refused. Its store starts from the checkpoint `pin`, if any. + pub(crate) fn open(path: Option<PathBuf>, files: Arc<Files>, pin: Option<Id>) -> Result<Self> { let Some(path) = path else { - return Ok(Holding::default()); + return Ok(Holding { + pin, + ..Holding::default() + }); }; let kept = match files .read(FileKind::Store, &path) @@ -126,6 +138,7 @@ impl Holding { }; Ok(Holding { file: Some((path, files)), + pin, inner: Mutex::new(Inner { kept, ..Inner::default() @@ -179,7 +192,22 @@ impl Holding { }; let id = Id::of(&root); let (own, dropped) = inner.kept.ids().unwrap_or_default(); - match Store::new(root, Vec::new()) { + let started = match self.pin { + None => Store::new(root, Vec::new()), + Some(pin) => match found(&pin, &root, &unread.events) { + Some((checkpoint, kept)) => { + info!(store = %id, checkpoint = %pin, "Found the pinned checkpoint in the document"); + Store::pinned(id, checkpoint, kept.into_values().collect()) + } + None => { + if !std::mem::replace(&mut inner.unfound, true) { + warn!(store = %id, checkpoint = %pin, "The pinned checkpoint isn't in the document (yet): holding no store until it is"); + } + return; + } + }, + }; + match started { Ok(store) => { info!(store = %id, refused = own.len(), dropped = dropped.len(), "Reading the store from the document"); inner.store = Some(store.refusing(&own, &dropped)); @@ -285,6 +313,38 @@ impl Holding { } } +/// The checkpoint `pin` names, found in the document's `events`: at heads a proposal's +/// version names, built from their history, read from the root, as any node builds the +/// checkpoint it moves to (`Store::checkpoint_at`). +fn found(pin: &Id, root: &[u8], events: &BTreeMap<Id, Vec<u8>>) -> Option<Built> { + let named = events + .values() + .filter_map(|bytes| match Event::read_taken(bytes)? { + Event::Signed(Signed { + does: Does::Proposal { version, .. }, + .. + }) => Version::read(&version).ok()?.checkpoint, + _ => None, + }); + for heads in named { + let mut store = Store::<Federation>::new(root.to_vec(), Vec::new()).ok()?; + let mut history = heads.clone(); + let mut seen = HashSet::new(); + while let Some(id) = history.pop() { + if let Some(bytes) = events.get(&id).filter(|_| seen.insert(id)) { + history.extend(Federation::dependencies(bytes).unwrap_or_default()); + store.offer(bytes.clone()); + } + } + if let Some((checkpoint, kept)) = store.checkpoint_at(&heads) + && Id::of(&checkpoint) == *pin + { + return Some((checkpoint, kept)); + } + } + None +} + /// Reads the store from the document whenever it changes, and from the changes the node /// refused now and then; puts again what the document lacks, a random wait after it /// first lacked it. @@ -396,7 +456,6 @@ mod tests { use crate::document::{Document, ScalarValue}; use crate::federation::event::Checkpoint; use crate::federation::harness::{self, Party, Spec}; - use crate::federation::store::Fold; use crate::federation::version::Version; fn root_of(spec: &Spec) -> Vec<u8> { @@ -424,6 +483,62 @@ mod tests { holding.read(holding.unread(doc, epoch_seq).unwrap()); } + /// A checkpoint at a write of lass's built on two of tv's, a write of makefu's beside + /// it: a node pinning it starts there, found in the document. + #[test] + fn a_node_starts_from_the_checkpoint_its_operator_pins() { + let (tv, lass, makefu) = ( + Party::new("tv", 1), + Party::new("lass", 2), + Party::new("makefu", 3), + ); + let first = Spec::first(&[&tv, &lass, &makefu], &[]); + let root = root_of(&first); + let on = first.hash(); + let gone = tv.write(&[Id::of(&root)], "home/tv/a", harness::str("1")); + let kept = tv.write(&[Id::of(&gone)], "home/tv/a", harness::str("2")); + let head = lass.write(&[Id::of(&kept)], "home/lass/b", harness::str("3")); + let beside = makefu.write(&[Id::of(&root)], "home/makefu/c", harness::str("4")); + let mut next = first.after(on); + next.checkpoint = Some(vec![Id::of(&head)]); + let proposal = tv.propose(&[Id::of(&head)], &next.json()); + let votes = [ + tv.vote(&[Id::of(&proposal)], on, next.hash(), true), + lass.vote(&[Id::of(&proposal)], on, next.hash(), true), + ]; + let written = harness::version_event(&[Id::of(&votes[0]), Id::of(&votes[1])], next.hash()); + let mut doc = events::started(Federation::NAME, &root).unwrap(); + for event in [&gone, &kept, &head, &beside, &proposal] + .into_iter() + .chain(&votes) + .chain([&written]) + { + put(&mut doc, event); + } + let from_root = Holding::default(); + read(&from_root, &doc, 0); + let pin = from_root.with(|store| Id::of(store.checkpoint())).unwrap(); + assert_ne!(pin, Id::of(&root)); + + let files = Arc::new(Files::default()); + let pinned = Holding::open(None, Arc::clone(&files), Some(pin)).unwrap(); + read(&pinned, &doc, 0); + assert_eq!(pinned.with(|store| Id::of(store.checkpoint())), Some(pin)); + let taken = |holding: &Holding| { + let mut taken = holding.with(|store| store.taken().to_vec()).unwrap(); + taken.sort_unstable(); + taken + }; + assert_eq!(taken(&pinned), taken(&from_root)); + assert_eq!( + pinned.with(|store| store.holds(&Id::of(&beside))), + Some(false) + ); + let elsewhere = Holding::open(None, files, Some(Id::of(b"elsewhere"))).unwrap(); + read(&elsewhere, &doc, 0); + assert!(elsewhere.with(|_| ()).is_none()); + } + #[test] fn a_node_puts_again_what_its_document_lacks() { let tv = Party::new("tv", 1); @@ -523,7 +638,7 @@ mod tests { let dir = crate::files::scratch_dir("held"); let path = dir.join("store.json"); let files = Arc::new(Files::default()); - let holding = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap(); + let holding = Holding::open(Some(path.clone()), Arc::clone(&files), None).unwrap(); let mut doc = events::started(Federation::NAME, &root).unwrap(); for event in before.iter().copied().chain([&first_taken]) { put(&mut doc, event); @@ -535,12 +650,12 @@ mod tests { assert!(!taken(&holding).contains(&Id::of(&later))); // Restarted: the store read anew, all at once, refuses the later as before. - let anew = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap(); + let anew = Holding::open(Some(path.clone()), Arc::clone(&files), None).unwrap(); read(&anew, &all, 0); assert!(taken(&anew).contains(&Id::of(&first_taken))); assert!(!taken(&anew).contains(&Id::of(&later))); std::fs::write(&path, b"{\"refused\": [\"not an ID\"]}").unwrap(); - assert!(Holding::open(Some(path), files).is_err()); + assert!(Holding::open(Some(path), files, None).is_err()); } } diff --git a/src/main.rs b/src/main.rs index 1e4af50..16afb7b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -312,7 +312,13 @@ async fn main() -> Result<()> { #[cfg(feature = "federation")] store: store.as_ref().map(|store| store.id), #[cfg(feature = "federation")] - holding: federation::held::Holding::open(args.store_file.clone(), Arc::clone(&files))?, + holding: federation::held::Holding::open( + args.store_file.clone(), + Arc::clone(&files), + args.store_checkpoint + .as_deref() + .and_then(federation::store::Id::from_hex), + )?, strict_keys: args .strict_keys .iter() |
