aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--docs/example.toml6
-rw-r--r--docs/openapi.yaml4
-rw-r--r--src/config/mod.rs17
-rw-r--r--src/document/events.rs1
-rw-r--r--src/federation/held.rs379
-rw-r--r--src/federation/mod.rs1
-rw-r--r--src/files.rs3
-rw-r--r--src/main.rs2
-rw-r--r--src/node.rs11
-rw-r--r--src/simtest/mod.rs2
-rw-r--r--src/state/mod.rs3
11 files changed, 425 insertions, 4 deletions
diff --git a/docs/example.toml b/docs/example.toml
index d7f8d08..c21158f 100644
--- a/docs/example.toml
+++ b/docs/example.toml
@@ -126,6 +126,12 @@ htpasswd = "/etc/continuum/htpasswd"
# store-root = "/var/lib/continuum/root.json"
# store-id = "…"
+# Behind the `federation` feature, with store-root or store-id: where the node keeps
+# what is its own of its store across restarts (the events it refused by what it took
+# before, and those its operator had it drop). Without, a restart, reading the store
+# anew from the document, may take what it refused.
+# store-file = "/var/lib/continuum/store.json"
+
# SSH keys, as authorized_keys lines (ssh-ed25519 <base64> [comment]), whose signature
# on a strict remake's manifest this node takes: the values kept when the swarm turns
# strict (docs/access.md). Without, it votes for no strict remake; a majority of the
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index bfb5ba3..0f1af33 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -2802,8 +2802,8 @@ components:
description: |
This node's own files, by kind (`state`: the state file, `document`: the
document file, `backup`: a backup recovered from, `remake`: the input of a
- genesis remake, and behind the `federation` feature `store`: a store's root),
- since it started.
+ genesis remake, and behind the `federation` feature `store`: a store's root,
+ and what the node keeps of its own of its store), since it started.
additionalProperties:
type: object
required: [reads, readBytes, writes, writtenBytes, appends, appendedBytes, errors]
diff --git a/src/config/mod.rs b/src/config/mod.rs
index ae6070f..afd9c4e 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -216,6 +216,14 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[
None,
"Join a swarm holding a store (docs/store.md) through seeds, by its ID: its root's SHA-256, in hex. Joining with an empty document, the node takes as its root only that store's genesis change, built again from the root it holds. Can't be combined with store-root, genesis, or what makes a new swarm's first change.",
),
+ #[cfg(feature = "federation")]
+ option(
+ "store-file",
+ Kind::Text,
+ "PATH",
+ None,
+ "With store-root or store-id: where the node keeps what is its own of its store across restarts, written whenever it changes: the events it refused by what it took before (of two versions changing one, the second), and those its operator had it drop. It reads its store anew from its document whenever it starts: without the file, it may then take what it refused. A file that doesn't read stops the node from starting.",
+ ),
option(
"strict-keys",
Kind::List,
@@ -651,6 +659,8 @@ pub(crate) struct Settings {
pub(crate) store_root: Option<PathBuf>,
#[cfg(feature = "federation")]
pub(crate) store_id: Option<String>,
+ #[cfg(feature = "federation")]
+ pub(crate) store_file: Option<PathBuf>,
#[serde(default)]
pub(crate) strict_keys: Vec<String>,
@@ -1116,6 +1126,11 @@ impl Settings {
}
(Some(_), None) => "store-root",
(None, Some(_)) => "store-id",
+ (None, None) if self.store_file.is_some() => {
+ bail!(
+ "store-file keeps what is the node's own of a store: set store-root or store-id"
+ )
+ }
(None, None) => return Ok(()),
};
let first_change = [
@@ -1169,6 +1184,8 @@ impl Settings {
("store-root", self.store_root.is_some()),
#[cfg(feature = "federation")]
("store-id", self.store_id.is_some()),
+ #[cfg(feature = "federation")]
+ ("store-file", self.store_file.is_some()),
("alias", self.alias.is_some()),
("dashboard-url", self.dashboard_url.is_some()),
("debug-api", self.debug_api),
diff --git a/src/document/events.rs b/src/document/events.rs
index 2b3a7e0..ee16de7 100644
--- a/src/document/events.rs
+++ b/src/document/events.rs
@@ -7,7 +7,6 @@
//! and the root at its ID, as bytes; each making the maps on its way
//! (`store::write_path`). Its writer is the first 16 bytes of the root's ID, its time 0,
//! and it has no message.
-#![allow(dead_code)]
use automerge::legacy::OpType;
use automerge::transaction::CommitOptions;
diff --git a/src/federation/held.rs b/src/federation/held.rs
new file mode 100644
index 0000000..28f4ca5
--- /dev/null
+++ b/src/federation/held.rs
@@ -0,0 +1,379 @@
+//! The store a node of a swarm holding one holds (`docs/store.md`, What the store takes
+//! from the document): read from its document, each event taken as the store would take
+//! it from anywhere, and by the node's own: what it refused by what it took before, and
+//! what its operator had it drop, which it keeps in a file of its own (`--store-file`).
+//! It keeps nothing else of the store across a restart: whenever it starts, it reads the
+//! store anew from its document, from the root.
+//!
+//! A worker reads the events the document holds that it hasn't read yet, whenever the
+//! document changes (`run_worker`). Which change put an event, and how often, changes
+//! nothing; nor does another document (a drop, a rejoin), whose events it reads as it
+//! comes, the store going on.
+
+use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex};
+
+use anyhow::{Context, Result, bail};
+use serde::{Deserialize, Serialize};
+use tracing::{error, info, warn};
+
+use super::fold::Federation;
+use super::store::Id;
+use super::store::memory::{Offered, Store};
+use crate::document::{ReadDoc, events, meta, store};
+use crate::files::{FileKind, Files};
+use crate::locks::MutexExt;
+
+/// What a node keeps of its own of its store across restarts, as its file holds it: the
+/// events it refused by what it took before, and those its operator had it drop, by ID.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct Kept {
+ #[serde(default)]
+ refused: BTreeSet<String>,
+ #[serde(default)]
+ dropped: BTreeSet<String>,
+}
+
+impl Kept {
+ fn of(own: &[Id], dropped: &[Id]) -> Self {
+ Kept {
+ refused: own.iter().map(Id::to_string).collect(),
+ dropped: dropped.iter().map(Id::to_string).collect(),
+ }
+ }
+
+ /// The IDs it holds: what was refused, and what dropped.
+ fn ids(&self) -> Option<(Vec<Id>, Vec<Id>)> {
+ let ids = |set: &BTreeSet<String>| {
+ set.iter()
+ .map(|id| Id::from_hex(id))
+ .collect::<Option<Vec<Id>>>()
+ };
+ Some((ids(&self.refused)?, ids(&self.dropped)?))
+ }
+}
+
+/// The store a node holds, if its swarm holds one, and the file it keeps its own in.
+#[derive(Default)]
+pub(crate) struct Holding {
+ file: Option<(PathBuf, Arc<Files>)>,
+ // A std mutex: never held while waiting for the document.
+ inner: Mutex<Inner>,
+}
+
+#[derive(Default)]
+struct Inner {
+ /// None until a document holding a store holds its root.
+ store: Option<Store<Federation>>,
+ /// The document read from (`Replica::epoch_seq`), and its events read.
+ epoch_seq: Option<u32>,
+ read: HashSet<Id>,
+ /// What the file holds, or will once written.
+ kept: Kept,
+ /// The root that is none, logged once.
+ none: Option<Id>,
+}
+
+/// What a document holds that its store wasn't offered from it.
+pub(crate) struct Unread {
+ epoch_seq: u32,
+ /// The root, where the store isn't started yet.
+ root: Option<Vec<u8>>,
+ events: BTreeMap<Id, Vec<u8>>,
+}
+
+impl Holding {
+ /// Keeping the node's own in the file at `path`, if any, which holds what it kept
+ /// before: a missing file nothing. One that doesn't read is an error: the node would
+ /// take what it refused.
+ pub(crate) fn open(path: Option<PathBuf>, files: Arc<Files>) -> Result<Self> {
+ let Some(path) = path else {
+ return Ok(Holding::default());
+ };
+ let kept = match files
+ .read(FileKind::Store, &path)
+ .with_context(|| format!("Reading the store file {}", path.display()))?
+ {
+ None => Kept::default(),
+ Some(bytes) => match serde_json::from_slice::<Kept>(&bytes) {
+ Ok(kept) if kept.ids().is_some() => kept,
+ _ => bail!(
+ "The store file {} doesn't read: move it away only knowing the node may then take what it refused",
+ path.display()
+ ),
+ },
+ };
+ Ok(Holding {
+ file: Some((path, files)),
+ inner: Mutex::new(Inner {
+ kept,
+ ..Inner::default()
+ }),
+ })
+ }
+
+ /// What `doc` (the document `epoch_seq`) holds that the store wasn't offered from it,
+ /// if it holds a store: everything, from another document.
+ pub(crate) fn unread(&self, doc: &impl ReadDoc, epoch_seq: u32) -> Option<Unread> {
+ if !matches!(
+ crate::access::policy::of(doc),
+ crate::access::policy::Policy::Store(_)
+ ) {
+ return None;
+ }
+ let swarm = match store::read_plain(doc, &meta::path(&[meta::SWARM])) {
+ Ok(Some(serde_json::Value::String(id))) => Id::from_hex(&id)?,
+ _ => return None,
+ };
+ let inner = self.inner.lock_or_recover();
+ let same = inner.epoch_seq == Some(epoch_seq);
+ let events = events::ids(doc)
+ .into_iter()
+ .filter(|id| !same || !inner.read.contains(id))
+ .filter_map(|id| Some((id, events::bytes(doc, &id)?)))
+ .collect();
+ let root = match &inner.store {
+ None => Some(events::bytes(doc, &swarm)?),
+ Some(_) => None,
+ };
+ Some(Unread {
+ epoch_seq,
+ root,
+ events,
+ })
+ }
+
+ /// Offers the store what was `unread`: started from the root first, by what the node
+ /// kept of its own. What it refuses of its own from then on, it keeps.
+ pub(crate) fn read(&self, unread: Unread) {
+ let mut inner = self.inner.lock_or_recover();
+ let inner = &mut *inner;
+ if inner.epoch_seq != Some(unread.epoch_seq) {
+ inner.epoch_seq = Some(unread.epoch_seq);
+ inner.read.clear();
+ }
+ if inner.store.is_none() {
+ let Some(root) = unread.root else {
+ return;
+ };
+ let id = Id::of(&root);
+ let (own, dropped) = inner.kept.ids().unwrap_or_default();
+ match Store::new(root, Vec::new()) {
+ Ok(store) => {
+ info!(store = %id, refused = own.len(), dropped = dropped.len(), "Reading the store from the document");
+ inner.store = Some(store.refusing(&own, &dropped));
+ }
+ Err(why) => {
+ if inner.none.replace(id) != Some(id) {
+ error!(store = %id, why = %why.0, "The document's root is no store's: not reading it");
+ }
+ return;
+ }
+ }
+ }
+ let store = inner.store.as_mut().expect("started");
+ let (mut taken, mut refused) = (0, 0);
+ for (id, bytes) in unread.events {
+ inner.read.insert(id);
+ match store.offer(bytes) {
+ Offered::Taken(ids) => taken += ids.len(),
+ Offered::Refused(why) => {
+ refused += 1;
+ info!(event = %id, why = ?why, "The store refuses an event the document holds");
+ }
+ Offered::Held | Offered::Waiting => {}
+ }
+ }
+ if taken > 0 || refused > 0 {
+ info!(taken, refused, "Read events from the document");
+ }
+ let (own, dropped) = store.own();
+ let kept = Kept::of(&own, &dropped);
+ if kept != inner.kept {
+ for id in kept.refused.difference(&inner.kept.refused) {
+ warn!(event = %id, "Refused an event its history takes, by what this node took before: kept so");
+ }
+ inner.kept = kept;
+ self.keep(&inner.kept);
+ }
+ }
+
+ /// Writes what the node keeps of its own to its file, if it has one.
+ fn keep(&self, kept: &Kept) {
+ let Some((path, files)) = &self.file else {
+ return;
+ };
+ let written = serde_json::to_vec_pretty(kept)
+ .map_err(std::io::Error::from)
+ .and_then(|mut bytes| {
+ bytes.push(b'\n');
+ files.write_atomic(FileKind::Store, path, &bytes)
+ });
+ if let Err(error) = written {
+ error!(path = %path.display(), error = %error, "Couldn't write the store file: a restart may take what this node refused");
+ }
+ }
+
+ /// `f` of the store, where the node holds one.
+ pub(crate) fn with<T>(&self, f: impl FnOnce(&Store<Federation>) -> T) -> Option<T> {
+ self.inner.lock_or_recover().store.as_ref().map(f)
+ }
+}
+
+/// Reads the store from the document whenever it changes.
+pub(crate) async fn run_worker(state: crate::state::AppState) {
+ let mut changed = state.change_tx.subscribe();
+ loop {
+ let holding = Arc::clone(&state.holding);
+ let doc = Arc::clone(&state.doc);
+ let read = tokio::task::spawn_blocking(move || {
+ // Read from the document under its lock; taken without it.
+ let unread = {
+ let replica = doc.blocking_read();
+ holding.unread(&replica.doc, replica.epoch_seq)
+ };
+ if let Some(unread) = unread {
+ holding.read(unread);
+ }
+ });
+ if let Err(error) = read.await {
+ error!(error = %error, "Reading the store from the document failed");
+ }
+ if changed.changed().await.is_err() {
+ return;
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::document::typed::{TimestampFormat, Val};
+ use crate::document::{Document, ScalarValue};
+ use crate::federation::event::Checkpoint;
+ use crate::federation::harness::{self, Party, Spec};
+ use crate::federation::store::Fold;
+ use crate::federation::version::Version;
+
+ fn root_of(spec: &Spec) -> Vec<u8> {
+ Checkpoint {
+ at: Vec::new(),
+ keeps: Vec::new(),
+ version: Version::read(&spec.json()).expect("a version"),
+ }
+ .bytes()
+ }
+
+ fn put(doc: &mut Document, event: &[u8]) {
+ crate::document::testing::change(doc, |doc| {
+ store::write_val_path(
+ doc,
+ &events::event(&Id::of(event)),
+ &Val::Scalar(ScalarValue::Bytes(event.to_vec())),
+ store::Writing::sticky(TimestampFormat::default()),
+ )
+ .unwrap();
+ });
+ }
+
+ fn read(holding: &Holding, doc: &Document, epoch_seq: u32) {
+ holding.read(holding.unread(doc, epoch_seq).unwrap());
+ }
+
+ #[test]
+ fn the_store_takes_the_events_the_document_holds() {
+ let tv = Party::new("tv", 1);
+ let first = Spec::first(&[&tv], &[]);
+ let root = root_of(&first);
+ let mut doc = events::started(Federation::NAME, &root).unwrap();
+ let holding = Holding::default();
+ assert!(holding.unread(&Document::new(), 0).is_none());
+ read(&holding, &doc, 0);
+ assert_eq!(holding.with(|store| store.id()), Some(Id::of(&root)));
+ let write = tv.write(&[Id::of(&root)], "home/tv/x", harness::str("1"));
+ let on = tv.write(&[Id::of(&write)], "home/tv/x", harness::str("2"));
+ // What it builds on comes later: it waits for it.
+ put(&mut doc, &on);
+ read(&holding, &doc, 0);
+ assert_eq!(
+ holding.with(|store| store.is_waiting(&Id::of(&on))),
+ Some(true)
+ );
+ put(&mut doc, &write);
+ read(&holding, &doc, 0);
+ assert_eq!(
+ holding.with(|store| store.taken().to_vec()),
+ Some(vec![Id::of(&write), Id::of(&on)])
+ );
+ // Another document holding them: nothing new.
+ let other = doc.fork();
+ read(&holding, &other, 1);
+ assert_eq!(holding.with(|store| store.taken().len()), Some(2));
+ }
+
+ /// Two version events changing version 0, each backed by tv and one other.
+ #[test]
+ fn a_node_read_anew_refuses_what_it_refused_of_its_own() {
+ let (tv, lass, makefu) = (
+ Party::new("tv", 1),
+ Party::new("lass", 2),
+ Party::new("makefu", 3),
+ );
+ let first = Spec::first(&[&tv, &lass, &makefu], &[]);
+ let root = root_of(&first);
+ let on = first.hash();
+ let one = first.after(on);
+ let mut two = first.after(on);
+ two.rules.push(("names/x".to_owned(), harness::str("tv")));
+ let p1 = tv.propose(&[Id::of(&root)], &one.json());
+ let p2 = lass.propose(&[Id::of(&root)], &two.json());
+ let votes = [
+ tv.vote(&[Id::of(&p1)], on, one.hash(), true),
+ lass.vote(&[Id::of(&p1)], on, one.hash(), true),
+ tv.vote(&[Id::of(&p2)], on, two.hash(), true),
+ makefu.vote(&[Id::of(&p2)], on, two.hash(), true),
+ ];
+ let v1 = harness::version_event(&[Id::of(&votes[0]), Id::of(&votes[1])], one.hash());
+ let v2 = harness::version_event(&[Id::of(&votes[2]), Id::of(&votes[3])], two.hash());
+ let before: Vec<&Vec<u8>> = [&p1, &p2].into_iter().chain(&votes).collect();
+ let taken = |holding: &Holding| holding.with(|store| store.taken().to_vec()).unwrap();
+ // The one a store reading all of them at once takes, and the other, which a node
+ // took as it came first.
+ let mut all = events::started(Federation::NAME, &root).unwrap();
+ for event in before.iter().copied().chain([&v1, &v2]) {
+ put(&mut all, event);
+ }
+ let forgetting = Holding::default();
+ read(&forgetting, &all, 0);
+ let (later, first_taken) = if taken(&forgetting).contains(&Id::of(&v1)) {
+ (v1, v2)
+ } else {
+ (v2, v1)
+ };
+
+ let dir = crate::files::scratch_dir("held");
+ let path = dir.join("store.json");
+ let files = Arc::new(Files::default());
+ let holding = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap();
+ let mut doc = events::started(Federation::NAME, &root).unwrap();
+ for event in before.iter().copied().chain([&first_taken]) {
+ put(&mut doc, event);
+ }
+ read(&holding, &doc, 0);
+ put(&mut doc, &later);
+ read(&holding, &doc, 0);
+ assert!(taken(&holding).contains(&Id::of(&first_taken)));
+ assert!(!taken(&holding).contains(&Id::of(&later)));
+
+ // Restarted: the store read anew, all at once, refuses the later as before.
+ let anew = Holding::open(Some(path.clone()), Arc::clone(&files)).unwrap();
+ read(&anew, &all, 0);
+ assert!(taken(&anew).contains(&Id::of(&first_taken)));
+ assert!(!taken(&anew).contains(&Id::of(&later)));
+
+ std::fs::write(&path, b"{\"refused\": [\"not an ID\"]}").unwrap();
+ assert!(Holding::open(Some(path), files).is_err());
+ }
+}
diff --git a/src/federation/mod.rs b/src/federation/mod.rs
index c3e4431..f7b3d01 100644
--- a/src/federation/mod.rs
+++ b/src/federation/mod.rs
@@ -11,6 +11,7 @@ pub(crate) mod event;
pub(crate) mod fold;
#[cfg(test)]
pub(crate) mod harness;
+pub(crate) mod held;
pub(crate) mod jcs;
pub(crate) mod keys;
#[cfg(test)]
diff --git a/src/files.rs b/src/files.rs
index 7c9d808..3e27066 100644
--- a/src/files.rs
+++ b/src/files.rs
@@ -21,7 +21,8 @@ pub(crate) enum FileKind {
Backup,
/// The input of a genesis remake (`--genesis --remake`).
Remake,
- /// A store's root (`--store-root`).
+ /// A store's root (`--store-root`), and what a node keeps of its own of its store
+ /// (`--store-file`).
#[cfg(feature = "federation")]
Store,
}
diff --git a/src/main.rs b/src/main.rs
index d074475..1e4af50 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -311,6 +311,8 @@ async fn main() -> Result<()> {
genesis_root: args.genesis_root.as_deref().and_then(swarm::roots::parse),
#[cfg(feature = "federation")]
store: store.as_ref().map(|store| store.id),
+ #[cfg(feature = "federation")]
+ holding: federation::held::Holding::open(args.store_file.clone(), Arc::clone(&files))?,
strict_keys: args
.strict_keys
.iter()
diff --git a/src/node.rs b/src/node.rs
index f722112..4e19da5 100644
--- a/src/node.rs
+++ b/src/node.rs
@@ -89,6 +89,9 @@ pub(crate) struct NodeConfig {
/// `--store-id`).
#[cfg(feature = "federation")]
pub(crate) store: Option<crate::federation::store::Id>,
+ /// The store it holds, and the file it keeps its own of it in (`--store-file`).
+ #[cfg(feature = "federation")]
+ pub(crate) holding: crate::federation::held::Holding,
/// `--strict-keys`.
pub(crate) strict_keys: Vec<crate::access::sshsig::PublicKey>,
/// `--cors-origins`.
@@ -206,6 +209,8 @@ impl Node {
)),
roots: Default::default(),
genesis_root: config.genesis_root,
+ #[cfg(feature = "federation")]
+ holding: Arc::new(config.holding),
challenges: Default::default(),
ended_sessions: Default::default(),
session_ended: Default::default(),
@@ -296,6 +301,12 @@ impl Node {
tokio::spawn(swarm::topology::run_worker(state.clone()));
trace!("Topology worker task spawned");
+ #[cfg(feature = "federation")]
+ {
+ tokio::spawn(crate::federation::held::run_worker(state.clone()));
+ trace!("Store worker task spawned");
+ }
+
state
}
}
diff --git a/src/simtest/mod.rs b/src/simtest/mod.rs
index 476658f..4da8821 100644
--- a/src/simtest/mod.rs
+++ b/src/simtest/mod.rs
@@ -227,6 +227,8 @@ impl Sim {
genesis_root: None,
#[cfg(feature = "federation")]
store: None,
+ #[cfg(feature = "federation")]
+ holding: Default::default(),
strict_keys: Vec::new(),
cors_origins: Vec::new(),
files: Arc::new(Files::default()),
diff --git a/src/state/mod.rs b/src/state/mod.rs
index 822f214..05eef1a 100644
--- a/src/state/mod.rs
+++ b/src/state/mod.rs
@@ -180,6 +180,9 @@ pub(crate) struct AppState {
/// the one it was given (`--genesis-root`).
pub(crate) roots: Arc<std::sync::Mutex<crate::swarm::roots::Roots>>,
pub(crate) genesis_root: Option<crate::document::ChangeHash>,
+ /// The store it holds, where its swarm holds one (`federation::held`).
+ #[cfg(feature = "federation")]
+ pub(crate) holding: Arc<crate::federation::held::Holding>,
/// Whose signature on a strict remake's manifest this node takes (`access::strict`).
pub(crate) strict_keys: Arc<Vec<crate::access::sshsig::PublicKey>>,
/// Origins whose pages may call this node's open endpoints (`api::cors`).