aboutsummaryrefslogtreecommitdiffstats
path: root/src/federation
diff options
context:
space:
mode:
Diffstat (limited to 'src/federation')
-rw-r--r--src/federation/held.rs379
-rw-r--r--src/federation/mod.rs1
2 files changed, 380 insertions, 0 deletions
diff --git a/src/federation/held.rs b/src/federation/held.rs
new file mode 100644
index 0000000..28f4ca5
--- /dev/null
+++ b/src/federation/held.rs
@@ -0,0 +1,379 @@
+//! The store a node of a swarm holding one holds (`docs/store.md`, What the store takes
+//! from the document): read from its document, each event taken as the store would take
+//! it from anywhere, and by the node's own: what it refused by what it took before, and
+//! what its operator had it drop, which it keeps in a file of its own (`--store-file`).
+//! It keeps nothing else of the store across a restart: whenever it starts, it reads the
+//! store anew from its document, from the root.
+//!
+//! A worker reads the events the document holds that it hasn't read yet, whenever the
+//! document changes (`run_worker`). Which change put an event, and how often, changes
+//! nothing; nor does another document (a drop, a rejoin), whose events it reads as it
+//! comes, the store going on.
+
+use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex};
+
+use anyhow::{Context, Result, bail};
+use serde::{Deserialize, Serialize};
+use tracing::{error, info, warn};
+
+use super::fold::Federation;
+use super::store::Id;
+use super::store::memory::{Offered, Store};
+use crate::document::{ReadDoc, events, meta, store};
+use crate::files::{FileKind, Files};
+use crate::locks::MutexExt;
+
+/// What a node keeps of its own of its store across restarts, as its file holds it: the
+/// events it refused by what it took before, and those its operator had it drop, by ID.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct Kept {
+ #[serde(default)]
+ refused: BTreeSet<String>,
+ #[serde(default)]
+ dropped: BTreeSet<String>,
+}
+
+impl Kept {
+ fn of(own: &[Id], dropped: &[Id]) -> Self {
+ Kept {
+ refused: own.iter().map(Id::to_string).collect(),
+ dropped: dropped.iter().map(Id::to_string).collect(),
+ }
+ }
+
+ /// The IDs it holds: what was refused, and what dropped.
+ fn ids(&self) -> Option<(Vec<Id>, Vec<Id>)> {
+ let ids = |set: &BTreeSet<String>| {
+ set.iter()
+ .map(|id| Id::from_hex(id))
+ .collect::<Option<Vec<Id>>>()
+ };
+ Some((ids(&self.refused)?, ids(&self.dropped)?))
+ }
+}
+
+/// The store a node holds, if its swarm holds one, and the file it keeps its own in.
+#[derive(Default)]
+pub(crate) struct Holding {
+ file: Option<(PathBuf, Arc<Files>)>,
+ // A std mutex: never held while waiting for the document.
+ inner: Mutex<Inner>,
+}
+
+#[derive(Default)]
+struct Inner {
+ /// None until a document holding a store holds its root.
+ store: Option<Store<Federation>>,
+ /// The document read from (`Replica::epoch_seq`), and its events read.
+ epoch_seq: Option<u32>,
+ read: HashSet<Id>,
+ /// What the file holds, or will once written.
+ kept: Kept,
+ /// The root that is none, logged once.
+ none: Option<Id>,
+}
+
+/// What a document holds that its store wasn't offered from it.
+pub(crate) struct Unread {
+ epoch_seq: u32,
+ /// The root, where the store isn't started yet.
+ root: Option<Vec<u8>>,
+ events: BTreeMap<Id, Vec<u8>>,
+}
+
+impl Holding {
+ /// Keeping the node's own in the file at `path`, if any, which holds what it kept
+ /// before: a missing file nothing. One that doesn't read is an error: the node would
+ /// take what it refused.
+ pub(crate) fn open(path: Option<PathBuf>, files: Arc<Files>) -> Result<Self> {
+ let Some(path) = path else {
+ return Ok(Holding::default());
+ };
+ let kept = match files
+ .read(FileKind::Store, &path)
+ .with_context(|| format!("Reading the store file {}", path.display()))?
+ {
+ None => Kept::default(),
+ Some(bytes) => match serde_json::from_slice::<Kept>(&bytes) {
+ Ok(kept) if kept.ids().is_some() => kept,
+ _ => bail!(
+ "The store file {} doesn't read: move it away only knowing the node may then take what it refused",
+ path.display()
+ ),
+ },
+ };
+ Ok(Holding {
+ file: Some((path, files)),
+ inner: Mutex::new(Inner {
+ kept,
+ ..Inner::default()
+ }),
+ })
+ }
+
+ /// What `doc` (the document `epoch_seq`) holds that the store wasn't offered from it,
+ /// if it holds a store: everything, from another document.
+ pub(crate) fn unread(&self, doc: &impl ReadDoc, epoch_seq: u32) -> Option<Unread> {
+ if !matches!(
+ crate::access::policy::of(doc),
+ crate::access::policy::Policy::Store(_)
+ ) {
+ return None;
+ }
+ let swarm = match store::read_plain(doc, &meta::path(&[meta::SWARM])) {
+ Ok(Some(serde_json::Value::String(id))) => Id::from_hex(&id)?,
+ _ => return None,
+ };
+ let inner = self.inner.lock_or_recover();
+ let same = inner.epoch_seq == Some(epoch_seq);
+ let events = events::ids(doc)
+ .into_iter()
+ .filter(|id| !same || !inner.read.contains(id))
+ .filter_map(|id| Some((id, events::bytes(doc, &id)?)))
+ .collect();
+ let root = match &inner.store {
+ None => Some(events::bytes(doc, &swarm)?),
+ Some(_) => None,
+ };
+ Some(Unread {
+ epoch_seq,
+ root,
+ events,
+ })
+ }
+
+ /// Offers the store what was `unread`: started from the root first, by what the node
+ /// kept of its own. What it refuses of its own from then on, it keeps.
+ pub(crate) fn read(&self, unread: Unread) {
+ let mut inner = self.inner.lock_or_recover();
+ let inner = &mut *inner;
+ if inner.epoch_seq != Some(unread.epoch_seq) {
+ inner.epoch_seq = Some(unread.epoch_seq);
+ inner.read.clear();
+ }
+ if inner.store.is_none() {
+ let Some(root) = unread.root else {
+ return;
+ };
+ let id = Id::of(&root);
+ let (own, dropped) = inner.kept.ids().unwrap_or_default();
+ match Store::new(root, Vec::new()) {
+ Ok(store) => {
+ info!(store = %id, refused = own.len(), dropped = dropped.len(), "Reading the store from the document");
+ inner.store = Some(store.refusing(&own, &dropped));
+ }
+ Err(why) => {
+ if inner.none.replace(id) != Some(id) {
+ error!(store = %id, why = %why.0, "The document's root is no store's: not reading it");
+ }
+ return;
+ }
+ }
+ }
+ let store = inner.store.as_mut().expect("started");
+ let (mut taken, mut refused) = (0, 0);
+ for (id, bytes) in unread.events {
+ inner.read.insert(id);
+ match store.offer(bytes) {
+ Offered::Taken(ids) => taken += ids.len(),
+ Offered::Refused(why) => {
+ refused += 1;
+ info!(event = %id, why = ?why, "The store refuses an event the document holds");
+ }
+ Offered::Held | Offered::Waiting => {}
+ }
+ }
+ if taken > 0 || refused > 0 {
+ info!(taken, refused, "Read events from the document");
+ }
+ let (own, dropped) = store.own();
+ let kept = Kept::of(&own, &dropped);
+ if kept != inner.kept {
+ for id in kept.refused.difference(&inner.kept.refused) {
+ warn!(event = %id, "Refused an event its history takes, by what this node took before: kept so");
+ }
+ inner.kept = kept;
+ self.keep(&inner.kept);
+ }
+ }
+
+ /// Writes what the node keeps of its own to its file, if it has one.
+ fn keep(&self, kept: &Kept) {
+ let Some((path, files)) = &self.file else {
+ return;
+ };
+ let written = serde_json::to_vec_pretty(kept)
+ .map_err(std::io::Error::from)
+ .and_then(|mut bytes| {
+ bytes.push(b'\n');
+ files.write_atomic(FileKind::Store, path, &bytes)
+ });
+ if let Err(error) = written {
+ error!(path = %path.display(), error = %error, "Couldn't write the store file: a restart may take what this node refused");
+ }
+ }
+
+ /// `f` of the store, where the node holds one.
+ pub(crate) fn with<T>(&self, f: impl FnOnce(&Store<Federation>) -> T) -> Option<T> {
+ self.inner.lock_or_recover().store.as_ref().map(f)
+ }
+}
+
+/// Reads the store from the document whenever it changes.
+pub(crate) async fn run_worker(state: crate::state::AppState) {
+ let mut changed = state.change_tx.subscribe();
+ loop {
+ let holding = Arc::clone(&state.holding);
+ let doc = Arc::clone(&state.doc);
+ let read = tokio::task::spawn_blocking(move || {
+ // Read from the document under its lock; taken without it.
+ let unread = {
+ let replica = doc.blocking_read();
+ holding.unread(&replica.doc, replica.epoch_seq)
+ };
+ if let Some(unread) = unread {
+ holding.read(unread);
+ }
+ });
+ if let Err(error) = read.await {
+ error!(error = %error, "Reading the store from the document failed");
+ }
+ if changed.changed().await.is_err() {
+ return;
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::document::typed::{TimestampFormat, Val};
+ use crate::document::{Document, ScalarValue};
+ use crate::federation::event::Checkpoint;
+ use crate::federation::harness::{self, Party, Spec};
+ use crate::federation::store::Fold;
+ use crate::federation::version::Version;
+
+ fn root_of(spec: &Spec) -> Vec<u8> {
+ Checkpoint {
+ at: Vec::new(),
+ keeps: Vec::new(),
+ version: Version::read(&spec.json()).expect("a version"),
+ }
+ .bytes()
+ }
+
+ fn put(doc: &mut Document, event: &[u8]) {
+ crate::document::testing::change(doc, |doc| {
+ store::write_val_path(
+ doc,
+ &events::event(&Id::of(event)),
+ &Val::Scalar(ScalarValue::Bytes(event.to_vec())),
+ store::Writing::sticky(TimestampFormat::default()),
+ )
+ .unwrap();
+ });
+ }
+
+ fn read(holding: &Holding, doc: &Document, epoch_seq: u32) {
+ holding.read(holding.unread(doc, epoch_seq).unwrap());
+ }
+
+ #[test]
+ fn the_store_takes_the_events_the_document_holds() {
+ let tv = Party::new("tv", 1);
+ let first = Spec::first(&[&tv], &[]);
+ let root = root_of(&first);
+ let mut doc = events::started(Federation::NAME, &root).unwrap();
+ let holding = Holding::default();
+ assert!(holding.unread(&Document::new(), 0).is_none());
+ read(&holding, &doc, 0);
+ assert_eq!(holding.with(|store| store.id()), Some(Id::of(&root)));
+ let write = tv.write(&[Id::of(&root)], "home/tv/x", harness::str("1"));
+ let on = tv.write(&[Id::of(&write)], "home/tv/x", harness::str("2"));
+ // What it builds on comes later: it waits for it.
+ put(&mut doc, &on);
+ read(&holding, &doc, 0);
+ assert_eq!(
+ holding.with(|store| store.is_waiting(&Id::of(&on))),
+ Some(true)
+ );
+ put(&mut doc, &write);
+ read(&holding, &doc, 0);
+ assert_eq!(
+ holding.with(|store| store.taken().to_vec()),
+ Some(vec![Id::of(&write), Id::of(&on)])
+ );
+ // Another document holding them: nothing new.
+ let other = doc.fork();
+ read(&holding, &other, 1);
+ assert_eq!(holding.with(|store| store.taken().len()), Some(2));
+ }
+
+ /// Two version events changing version 0, each backed by tv and one other.
+ #[test]
+ fn a_node_read_anew_refuses_what_it_refused_of_its_own() {
+ let (tv, lass, makefu) = (
+ Party::new("tv", 1),
+ Party::new("lass", 2),
+ Party::new("makefu", 3),
+ );
+ let first = Spec::first(&[&tv, &lass, &makefu], &[]);
+ let root = root_of(&first);
+ let on = first.hash();
+ let one = first.after(on);
+ let mut two = first.after(on);
+ two.rules.push(("names/x".to_owned(), harness::str("tv")));
+ let p1 = tv.propose(&[Id::of(&root)], &one.json());
+ let p2 = lass.propose(&[Id::of(&root)], &two.json());
+ let votes = [
+ tv.vote(&[Id::of(&p1)], on, one.hash(), true),
+ lass.vote(&[Id::of(&p1)], on, one.hash(), true),
+ tv.vote(&[Id::of(&p2)], on, two.hash(), true),
+ makefu.vote(&[Id::of(&p2)], on, two.hash(), true),
+ ];
+ let v1 = harness::version_event(&[Id::of(&votes[0]), Id::of(&votes[1])], one.hash());
+ let v2 = harness::version_event(&[Id::of(&votes[2]), Id::of(&votes[3])], two.hash());
+ let before: Vec<&Vec<u8>> = [&p1, &p2].into_iter().chain(&votes).collect();
+ let taken = |holding: &Holding| holding.with(|store| store.taken().to_vec()).unwrap();
+ // The one a store reading all of them at once takes, and the other, which a node
+ // took as it came first.
+ let mut all = events::started(Federation::NAME, &root).unwrap();
+ for event in before.iter().copied().chain([&v1, &v2]) {
+ put(&mut all, event);
+ }
+ let forgetting = Holding::default();
+ read(&forgetting, &all, 0);
+ let (later, first_taken) = if taken(&forgetting).contains(&Id::of(&v1)) {
+ (v1, v2)
+ } else {
+ (v2, v1)
+ };
+
+ let dir = crate::files::scratch_dir("held");
+ let path = dir.join("store.json");
+ let files = Arc::new(Files::default());
+ let holding = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap();
+ let mut doc = events::started(Federation::NAME, &root).unwrap();
+ for event in before.iter().copied().chain([&first_taken]) {
+ put(&mut doc, event);
+ }
+ read(&holding, &doc, 0);
+ put(&mut doc, &later);
+ read(&holding, &doc, 0);
+ assert!(taken(&holding).contains(&Id::of(&first_taken)));
+ assert!(!taken(&holding).contains(&Id::of(&later)));
+
+ // Restarted: the store read anew, all at once, refuses the later as before.
+ let anew = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap();
+ read(&anew, &all, 0);
+ assert!(taken(&anew).contains(&Id::of(&first_taken)));
+ assert!(!taken(&anew).contains(&Id::of(&later)));
+
+ std::fs::write(&path, b"{\"refused\": [\"not an ID\"]}").unwrap();
+ assert!(Holding::open(Some(path), files).is_err());
+ }
+}
diff --git a/src/federation/mod.rs b/src/federation/mod.rs
index c3e4431..f7b3d01 100644
--- a/src/federation/mod.rs
+++ b/src/federation/mod.rs
@@ -11,6 +11,7 @@ pub(crate) mod event;
pub(crate) mod fold;
#[cfg(test)]
pub(crate) mod harness;
+pub(crate) mod held;
pub(crate) mod jcs;
pub(crate) mod keys;
#[cfg(test)]