diff options
| -rw-r--r-- | docs/example.toml | 14 | ||||
| -rw-r--r-- | src/config/mod.rs | 147 | ||||
| -rw-r--r-- | src/federation/held.rs | 214 | ||||
| -rw-r--r-- | src/lib.rs | 26 |
4 files changed, 394 insertions, 7 deletions
diff --git a/docs/example.toml b/docs/example.toml index d101964..36a4ae2 100644 --- a/docs/example.toml +++ b/docs/example.toml @@ -148,6 +148,20 @@ htpasswd = "/etc/continuum/htpasswd" # party listing many nodes can't be all of them. # store-neighbour-parties = ["lass"] +# Behind the `federation` feature: this node votes and proposes for its party by itself +# (docs/federation.md, Votes a node casts by itself), with a key its party's version +# names, kept here, and its party's one record of its votes; its party then votes by +# hand only through it (POST /admin/events/vote). It follows the parties named (by +# default only on versions that only move to a checkpoint), and proposes a checkpoint +# once it holds so many events beyond its checkpoint, or so long after it moved there. +# store-vote-key = "/var/lib/continuum/party.key" +# store-vote-record = "/var/lib/continuum/party-record.json" +# store-follow = ["lass"] +# store-follow-any = false +# store-propose-events = 10000 +# store-propose-every = "7d" +# store-held-for = "10m" + # SSH keys, as authorized_keys lines (ssh-ed25519 <base64> [comment]), whose signature # on a strict remake's manifest this node takes: the values kept when the swarm turns # strict (docs/access.md). Without, it votes for no strict remake; a majority of the diff --git a/src/config/mod.rs b/src/config/mod.rs index 762e9a4..7889d7c 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -248,6 +248,62 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ None, "With store-root or store-id: parties among whose members, with those of its own party, this node keeps some of its neighbours, by the graph among those alone (docs/store.md, Members): so that a party listing many nodes can't be all of them.", ), + #[cfg(feature = "federation")] + option( + "store-vote-key", + Kind::Text, + "PATH", + None, + "With store-root or store-id: this node votes and proposes for its party by itself, by the rules below (docs/federation.md, Votes a node casts by itself), signing with this key, an OpenSSH private key file, unencrypted, one its party's version names, kept on this node: one stolen here votes for the party. Its party then votes by hand only through this node (POST /admin/events/vote), which keeps its one record (store-vote-record).", + ), + #[cfg(feature = "federation")] + option( + "store-vote-record", + Kind::Text, + "PATH", + None, + "With store-vote-key, which needs it: its party's one record of its votes, which the node votes by, written before each vote is sent; begun, where there is none, on the version current then, on which it casts nothing.", + ), + #[cfg(feature = "federation")] + option( + "store-follow", + Kind::List, + "NAME,[NAME...]", + None, + "With store-vote-key: parties it follows, voting `true` on a proposal where one of them did, `false` where one did; by default only on a version that only moves to a checkpoint.", + ), + #[cfg(feature = "federation")] + option( + "store-follow-any", + Kind::Flag, + "BOOL", + Some("false"), + "With store-follow: follow on any version, not only on one that only moves to a checkpoint. It hands those parties the node's vote, its `false` among it.", + ), + #[cfg(feature = "federation")] + option( + "store-propose-events", + Kind::Integer, + "COUNT", + None, + "With store-vote-key: propose a version that only moves to a checkpoint once it holds this many events beyond its checkpoint, at the heads of what it held for a while (store-held-for), where nothing that can still pass is proposed on the current version; it backs what it proposes.", + ), + #[cfg(feature = "federation")] + option( + "store-propose-every", + Kind::Duration, + "DURATION", + None, + "With store-vote-key: propose a checkpoint, as store-propose-events, once this long passed by its own clock since its store moved to its checkpoint, or started there.", + ), + #[cfg(feature = "federation")] + option( + "store-held-for", + Kind::Duration, + "DURATION", + Some("10m"), + "With store-vote-key: how long it holds an event before it holds it for a while: it proposes a checkpoint at the heads of such, and backs none whose heads leave out one it held for a while before it was proposed.", + ), option( "strict-keys", Kind::List, @@ -692,6 +748,31 @@ pub(crate) struct Settings { #[cfg(feature = "federation")] #[serde(default)] pub(crate) store_neighbour_parties: Vec<String>, + #[cfg(feature = "federation")] + pub(crate) store_vote_key: Option<PathBuf>, + #[cfg(feature = "federation")] + pub(crate) store_vote_record: Option<PathBuf>, + #[cfg(feature = "federation")] + #[serde(default)] + pub(crate) store_follow: Vec<String>, + #[cfg(feature = "federation")] + #[serde(default)] + pub(crate) store_follow_any: bool, + #[cfg(feature = "federation")] + pub(crate) store_propose_events: Option<usize>, + #[cfg(feature = "federation")] + #[serde( + default, + deserialize_with = "crate::units::optional_duration", + serialize_with = "crate::units::serialize_optional_duration" + )] + pub(crate) store_propose_every: Option<Duration>, + #[cfg(feature = "federation")] + #[serde( + default = "default_store_held_for", + with = "crate::units::duration_option" + )] + pub(crate) store_held_for: Duration, #[serde(default)] pub(crate) strict_keys: Vec<String>, @@ -883,6 +964,10 @@ fn default_token_lifetime() -> Duration { fn default_session_lifetime() -> Duration { Duration::from_secs(7 * 24 * 3600) } +#[cfg(feature = "federation")] +fn default_store_held_for() -> Duration { + Duration::from_secs(10 * 60) +} fn default_challenge_lifetime() -> Duration { Duration::from_secs(5 * 60) } @@ -1171,6 +1256,7 @@ impl Settings { "store-neighbour-parties", !self.store_neighbour_parties.is_empty(), ), + ("store-vote-key", self.store_vote_key.is_some()), ]; let set: Vec<&str> = of_a_store .iter() @@ -1203,6 +1289,44 @@ impl Settings { { bail!("store-neighbour-parties: {party:?} isn't a party's name"); } + // Voting by itself: its key, its party's record, and rules only with them. + let voting = [ + ("store-vote-record", self.store_vote_record.is_some()), + ("store-follow", !self.store_follow.is_empty()), + ("store-follow-any", self.store_follow_any), + ("store-propose-events", self.store_propose_events.is_some()), + ("store-propose-every", self.store_propose_every.is_some()), + ]; + match (&self.store_vote_key, &self.store_vote_record) { + (Some(_), None) => { + bail!("store-vote-key: set store-vote-record, its party's one record of its votes") + } + (None, _) => { + let set: Vec<&str> = voting + .iter() + .filter(|(_, set)| *set) + .map(|(name, _)| *name) + .collect(); + if !set.is_empty() { + bail!( + "{} {} of a node voting by itself: set store-vote-key", + set.join(", "), + if set.len() == 1 { "is" } else { "are" } + ); + } + } + (Some(_), Some(_)) => {} + } + if self.store_follow_any && self.store_follow.is_empty() { + bail!("store-follow-any follows on any version: set store-follow, whom"); + } + if let Some(party) = self + .store_follow + .iter() + .find(|party| !crate::federation::version::is_party(party)) + { + bail!("store-follow: {party:?} isn't a party's name"); + } let first_change = [ ("genesis", self.genesis), ("bootstrap-users", !self.bootstrap_users.is_empty()), @@ -1282,6 +1406,8 @@ impl Settings { "store-neighbour-parties", !self.store_neighbour_parties.is_empty(), ), + #[cfg(feature = "federation")] + ("store-vote-key", self.store_vote_key.is_some()), ("alias", self.alias.is_some()), ("dashboard-url", self.dashboard_url.is_some()), ("debug-api", self.debug_api), @@ -2390,6 +2516,27 @@ mod tests { .is_err() ); assert!(checked(json!({ "store-neighbour-parties": ["lass"] })).is_err()); + let voting = |more: JsonValue| { + let mut values = json!({ "store-id": id, "store-party": "tv" }); + for (name, value) in more.as_object().unwrap() { + values[name] = value.clone(); + } + checked(values) + }; + assert!( + voting( + json!({ "store-vote-key": "k", "store-vote-record": "r", "store-follow": ["lass"] }) + ) + .is_ok() + ); + assert!(voting(json!({ "store-vote-key": "k" })).is_err()); + assert!(voting(json!({ "store-follow": ["lass"] })).is_err()); + assert!( + voting( + json!({ "store-vote-key": "k", "store-vote-record": "r", "store-follow-any": true }) + ) + .is_err() + ); assert!( checked(json!({ "store-id": id, "store-root": "root.json", "store-party": "tv" })) .is_err() diff --git a/src/federation/held.rs b/src/federation/held.rs index 07660e0..f48778d 100644 --- a/src/federation/held.rs +++ b/src/federation/held.rs @@ -20,20 +20,22 @@ //! its own has passed, random, so that not every node puts the same, and none another //! node put meanwhile (`put_again`). -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; -use std::time::Duration; +use std::time::{Duration, Instant}; use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use tracing::{error, info, warn}; -use super::event::{Does, Event, Signed}; +use super::client::Record; +use super::event::{self, Does, Event, Signed}; use super::fold::Federation; use super::store::memory::{Built, Offered, Store}; use super::store::{Fold, Id, Refused}; use super::version::Version; +use super::vote::{self, Act, Rules, Seen}; use crate::document::{Change, ChangeHash, ReadDoc, ScalarValue, events, meta, store}; use crate::files::{FileKind, Files}; use crate::locks::{MutexExt, RwLockExt}; @@ -90,10 +92,21 @@ pub(crate) struct Holding { /// The members the store names, each with its party, as it reads now: apart from the /// store, so that whoever asks doesn't wait while the store takes events. members: RwLock<Option<BTreeSet<(String, String)>>>, + /// What it votes for its party with, if it votes by itself. + voter: Option<Voter>, // A std mutex: never held while waiting for the document. inner: Mutex<Inner>, } +/// What a node voting by itself for its party signs with, keeps and goes by +/// (`federation::vote`): a key its party's version names, kept on the node; its party's +/// one record of its votes, in a file; and its operator's rules. +pub(crate) struct Voter { + pub(crate) key: ed25519_dalek::SigningKey, + pub(crate) record: PathBuf, + pub(crate) rules: Rules, +} + #[derive(Default)] struct Inner { /// None until a document holding a store holds its root. @@ -109,6 +122,13 @@ struct Inner { unfound: bool, /// The refused changes looked at, of those screening keeps (`look`). looked: HashSet<ChangeHash>, + /// When, by its own clock, the node took each event its store took after its + /// checkpoint, and when the store moved to its checkpoint, or started there. + taken_at: HashMap<Id, Instant>, + at_checkpoint: Option<(Id, Instant)>, + /// Its party's record, voting by itself, once read; and why it isn't, said once. + record: Option<Record>, + unrecorded: Option<String>, } /// What a document holds that its store wasn't offered from it. @@ -156,6 +176,7 @@ impl Holding { party, near: Vec::new(), members: RwLock::default(), + voter: None, inner: Mutex::new(Inner { kept, ..Inner::default() @@ -262,6 +283,16 @@ impl Holding { let Some(store) = &inner.store else { return; }; + let now = Instant::now(); + let checkpoint = Id::of(store.checkpoint()); + if inner.at_checkpoint.map(|(at, _)| at) != Some(checkpoint) { + inner.at_checkpoint = Some((checkpoint, now)); + } + let taken: HashSet<&Id> = store.taken().iter().collect(); + inner.taken_at.retain(|id, _| taken.contains(id)); + for id in store.taken() { + inner.taken_at.entry(*id).or_insert(now); + } let (own, dropped) = store.own(); let kept = Kept::of(&own, &dropped); if kept != inner.kept { @@ -401,6 +432,140 @@ impl Holding { self.members.read_or_recover().clone() } + /// Itself, voting by itself for its party as `voter` says, if it does. + pub(crate) fn voting(self, voter: Option<Voter>) -> Self { + Holding { voter, ..self } + } + + /// Its party's record, read from its file, or begun on the current version where there + /// is none: one of another store or party, or that doesn't read, it votes by none. + fn record<'a>(&self, voter: &Voter, inner: &'a mut Inner) -> Option<&'a mut Record> { + let store = inner.store.as_ref()?; + let party = self.party.as_deref()?; + if inner.record.is_none() { + let read = match Record::load(&voter.record) { + Ok(record) if record.store == store.id().to_string() && record.party == party => { + Ok(record) + } + Ok(_) => Err(format!( + "{}: the record of another store or party", + voter.record.display() + )), + Err(_) if !voter.record.exists() => { + let record = Record::begin(store.id(), party, store.state().current()); + record.save(&voter.record).map(|()| record) + } + Err(why) => Err(why), + }; + match read { + Ok(record) => inner.record = Some(record), + Err(why) => { + if inner.unrecorded.as_ref() != Some(&why) { + error!(why = %why, "Voting by itself, without its party's record: votes none"); + inner.unrecorded = Some(why); + } + return None; + } + } + } + inner.record.as_mut() + } + + /// What it signs now, voting by itself by its operator's rules (`vote::decide`), or by + /// hand (`hand`, `vote::by_hand`): recorded, the record written, and offered to the + /// store; the events, to put into the document. By hand, why not where it isn't. + fn sign(&self, hand: Option<(Id, bool)>) -> Result<Vec<Vec<u8>>, String> { + let (Some(voter), Some(party)) = (&self.voter, &self.party) else { + return Err("this node doesn't vote for its party".to_owned()); + }; + let mut inner = self.inner.lock_or_recover(); + let inner = &mut *inner; + self.record(voter, inner) + .ok_or("this node holds no store, or no record of its party")?; + let (Some(store), Some(record)) = (&inner.store, &inner.record) else { + return Err("this node holds no store".to_owned()); + }; + let now = Instant::now(); + let held = |id: &Id| inner.taken_at.get(id).map(|at| now.duration_since(*at)); + let seen = Seen { + store, + held: &held, + since_move: inner + .at_checkpoint + .map_or(Duration::ZERO, |(_, at)| now.duration_since(at)), + }; + let acts = match hand { + Some((proposal, yes)) => { + vec![vote::by_hand(&seen, &voter.rules, record, proposal, yes)?] + } + None => vote::decide(&seen, &voter.rules, record), + }; + if acts.is_empty() { + return Ok(Vec::new()); + } + let current = store.state().current().clone(); + let heads = store.heads(); + let line = crate::access::sshsig::PublicKey(voter.key.verifying_key().to_bytes()).line(); + let mut record = record.clone(); + let mut signed = Vec::new(); + for act in acts { + match act { + Act::Vote { + proposal, + yes, + nays, + } => { + let cast = event::signed( + &voter.key, + party, + &line, + &heads, + event::vote(current.hash, proposal, yes), + ); + record.cast(¤t, &cast, nays); + info!(proposal = %proposal, yes, by_hand = hand.is_some(), "Voting for its party"); + signed.push(cast); + } + Act::Propose(version) => { + let proposing = + event::signed(&voter.key, party, &line, &heads, event::proposal(&version)); + record.signed(&proposing); + info!(proposal = %Id::of(&version.bytes()), "Proposing a checkpoint for its party"); + signed.push(proposing); + } + } + } + // Written before anything is sent. + record.save(&voter.record)?; + inner.record = Some(record); + let store = inner.store.as_mut().expect("held above"); + for bytes in &signed { + store.offer(bytes.clone()); + } + self.settle(inner); + Ok(signed) + } + + /// What it signs now, voting by itself by its operator's rules: to put into the + /// document. + pub(crate) fn act(&self) -> Vec<Vec<u8>> { + if self.voter.is_none() { + return Vec::new(); + } + self.sign(None).unwrap_or_else(|why| { + warn!(why = %why, "Couldn't vote by itself"); + Vec::new() + }) + } + + /// A vote by hand through this node, which votes for its party and keeps its one + /// record: the event, to put into the document, or why not. + pub(crate) fn vote_by_hand(&self, proposal: Id, yes: bool) -> Result<Vec<u8>, String> { + self.sign(Some((proposal, yes)))? + .pop() + .ok_or_else(|| "nothing to cast".to_owned()) + } + /// The party its operator says the node is of (`--store-party`). pub(crate) fn party(&self) -> Option<&str> { self.party.as_deref() @@ -497,12 +662,21 @@ pub(crate) async fn run_worker(state: crate::state::AppState) { if let Some(unread) = unread { holding.read(unread); } - !holding.lacking(&doc.blocking_read().doc).is_empty() + let signed = holding.act(); + ( + !holding.lacking(&doc.blocking_read().doc).is_empty(), + signed, + ) }); - let lacking = read.await.unwrap_or_else(|error| { + let (lacking, signed) = read.await.unwrap_or_else(|error| { error!(error = %error, "Reading the store from the document failed"); - false + (false, Vec::new()) }); + for bytes in signed { + if let Err(error) = write(&state, None, bytes).await { + warn!(error = %error, "Couldn't put what it signed for its party: put again later"); + } + } let member = state .holding .member(state.local_id(), state.holding.party()); @@ -556,6 +730,34 @@ fn wait() -> Duration { /// Puts again, in a change of its own, the events the store holds that the document /// lacks still, each after its dependencies, up to `PUT_BYTES` of them. +/// Puts `bytes`, an event, into the document at its ID unless it holds it: for `user`, who +/// put it through the API, or for the node itself. +pub(crate) async fn write( + state: &crate::state::AppState, + user: Option<String>, + bytes: Vec<u8>, +) -> Result<(), store::StoreError> { + let id = Id::of(&bytes); + state + .mutate_doc_for( + user, + crate::document::signatures::Sign::Protocol, + move |doc| { + if events::bytes(doc, &id).is_none() { + store::write_val_path( + doc, + &events::event(&id), + &crate::document::typed::Val::Scalar(ScalarValue::Bytes(bytes)), + store::Writing::sticky(Default::default()), + )?; + } + Ok::<(), store::StoreError>(()) + }, + ) + .await + .map(|_| ()) +} + async fn put_again(state: &crate::state::AppState) { let holding = Arc::clone(&state.holding); let put = state @@ -50,6 +50,29 @@ pub async fn party() -> Result<()> { federation::party::main().await } +/// What a node voting by itself for its party goes by, where it does (`--store-vote-key`). +#[cfg(feature = "federation")] +fn voter(args: &config::Settings) -> Result<Option<federation::held::Voter>> { + let (Some(key), Some(record)) = (&args.store_vote_key, &args.store_vote_record) else { + return Ok(None); + }; + let text = std::fs::read_to_string(key) + .with_context(|| format!("Reading the party's key {}", key.display()))?; + let key = federation::client::signing_key(&text) + .map_err(|why| anyhow::anyhow!("The party's key {}: {why}", key.display()))?; + Ok(Some(federation::held::Voter { + key, + record: record.clone(), + rules: federation::vote::Rules { + follow: args.store_follow.clone(), + follow_any: args.store_follow_any, + propose_events: args.store_propose_events, + propose_every: args.store_propose_every, + held_for: args.store_held_for, + }, + })) +} + /// Runs a node, as its arguments and configuration say, until it stops. pub async fn run() -> Result<()> { logging::init(); @@ -328,7 +351,8 @@ pub async fn run() -> Result<()> { .and_then(federation::store::Id::from_hex), args.store_party.clone(), )? - .near(args.store_neighbour_parties.clone()), + .near(args.store_neighbour_parties.clone()) + .voting(voter(args)?), strict_keys: args .strict_keys .iter() |
