From c77f4a88dfd0f4ed423951da6535244aea4a0fa1 Mon Sep 17 00:00:00 2001 From: tv Date: Mon, 16 Nov 2026 22:25:31 +0000 Subject: node: votes it casts by itself, for its party A node may vote and propose for its party by itself, by its operator's rules (`docs/federation.md`, Votes a node casts by itself; `federation::vote`): * `--store-vote-key`: the party's key it signs with, an OpenSSH private key file, one the party's version names, kept on the node; * `--store-vote-record`: its party's one record of its votes, which it votes by, written before each vote is sent, begun where there is none on the version current then; * `--store-follow`, `--store-follow-any`: the parties it follows, by default only on a version that only moves to a checkpoint; * `--store-propose-events`, `--store-propose-every`: when it proposes a checkpoint, once it holds so many events beyond its checkpoint, or so long after its store moved there; * `--store-held-for` (10 minutes): how long it holds an event before it holds it for a while, for the heads it proposes and the voters' rule. Its store keeps when, by the node's own clock, it took each event, and when it moved to its checkpoint. Whenever the worker reads the store, the node decides what to sign (`Holding::act`), records it, writes the record, offers it to its store, and puts it into the document (`held::write`). A vote by hand through the node, by the record and the voters' rule, is `Holding::vote_by_hand`. --- docs/example.toml | 14 ++++ src/config/mod.rs | 147 +++++++++++++++++++++++++++++++++ src/federation/held.rs | 214 +++++++++++++++++++++++++++++++++++++++++++++++-- src/lib.rs | 26 +++++- 4 files changed, 394 insertions(+), 7 deletions(-) diff --git a/docs/example.toml b/docs/example.toml index d101964..36a4ae2 100644 --- a/docs/example.toml +++ b/docs/example.toml @@ -148,6 +148,20 @@ htpasswd = "/etc/continuum/htpasswd" # party listing many nodes can't be all of them. # store-neighbour-parties = ["lass"] +# Behind the `federation` feature: this node votes and proposes for its party by itself +# (docs/federation.md, Votes a node casts by itself), with a key its party's version +# names, kept here, and its party's one record of its votes; its party then votes by +# hand only through it (POST /admin/events/vote). It follows the parties named (by +# default only on versions that only move to a checkpoint), and proposes a checkpoint +# once it holds so many events beyond its checkpoint, or so long after it moved there. +# store-vote-key = "/var/lib/continuum/party.key" +# store-vote-record = "/var/lib/continuum/party-record.json" +# store-follow = ["lass"] +# store-follow-any = false +# store-propose-events = 10000 +# store-propose-every = "7d" +# store-held-for = "10m" + # SSH keys, as authorized_keys lines (ssh-ed25519 [comment]), whose signature # on a strict remake's manifest this node takes: the values kept when the swarm turns # strict (docs/access.md). Without, it votes for no strict remake; a majority of the diff --git a/src/config/mod.rs b/src/config/mod.rs index 762e9a4..7889d7c 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -248,6 +248,62 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ None, "With store-root or store-id: parties among whose members, with those of its own party, this node keeps some of its neighbours, by the graph among those alone (docs/store.md, Members): so that a party listing many nodes can't be all of them.", ), + #[cfg(feature = "federation")] + option( + "store-vote-key", + Kind::Text, + "PATH", + None, + "With store-root or store-id: this node votes and proposes for its party by itself, by the rules below (docs/federation.md, Votes a node casts by itself), signing with this key, an OpenSSH private key file, unencrypted, one its party's version names, kept on this node: one stolen here votes for the party. Its party then votes by hand only through this node (POST /admin/events/vote), which keeps its one record (store-vote-record).", + ), + #[cfg(feature = "federation")] + option( + "store-vote-record", + Kind::Text, + "PATH", + None, + "With store-vote-key, which needs it: its party's one record of its votes, which the node votes by, written before each vote is sent; begun, where there is none, on the version current then, on which it casts nothing.", + ), + #[cfg(feature = "federation")] + option( + "store-follow", + Kind::List, + "NAME,[NAME...]", + None, + "With store-vote-key: parties it follows, voting `true` on a proposal where one of them did, `false` where one did; by default only on a version that only moves to a checkpoint.", + ), + #[cfg(feature = "federation")] + option( + "store-follow-any", + Kind::Flag, + "BOOL", + Some("false"), + "With store-follow: follow on any version, not only on one that only moves to a checkpoint. It hands those parties the node's vote, its `false` among it.", + ), + #[cfg(feature = "federation")] + option( + "store-propose-events", + Kind::Integer, + "COUNT", + None, + "With store-vote-key: propose a version that only moves to a checkpoint once it holds this many events beyond its checkpoint, at the heads of what it held for a while (store-held-for), where nothing that can still pass is proposed on the current version; it backs what it proposes.", + ), + #[cfg(feature = "federation")] + option( + "store-propose-every", + Kind::Duration, + "DURATION", + None, + "With store-vote-key: propose a checkpoint, as store-propose-events, once this long passed by its own clock since its store moved to its checkpoint, or started there.", + ), + #[cfg(feature = "federation")] + option( + "store-held-for", + Kind::Duration, + "DURATION", + Some("10m"), + "With store-vote-key: how long it holds an event before it holds it for a while: it proposes a checkpoint at the heads of such, and backs none whose heads leave out one it held for a while before it was proposed.", + ), option( "strict-keys", Kind::List, @@ -692,6 +748,31 @@ pub(crate) struct Settings { #[cfg(feature = "federation")] #[serde(default)] pub(crate) store_neighbour_parties: Vec, + #[cfg(feature = "federation")] + pub(crate) store_vote_key: Option, + #[cfg(feature = "federation")] + pub(crate) store_vote_record: Option, + #[cfg(feature = "federation")] + #[serde(default)] + pub(crate) store_follow: Vec, + #[cfg(feature = "federation")] + #[serde(default)] + pub(crate) store_follow_any: bool, + #[cfg(feature = "federation")] + pub(crate) store_propose_events: Option, + #[cfg(feature = "federation")] + #[serde( + default, + deserialize_with = "crate::units::optional_duration", + serialize_with = "crate::units::serialize_optional_duration" + )] + pub(crate) store_propose_every: Option, + #[cfg(feature = "federation")] + #[serde( + default = "default_store_held_for", + with = "crate::units::duration_option" + )] + pub(crate) store_held_for: Duration, #[serde(default)] pub(crate) strict_keys: Vec, @@ -883,6 +964,10 @@ fn default_token_lifetime() -> Duration { fn default_session_lifetime() -> Duration { Duration::from_secs(7 * 24 * 3600) } +#[cfg(feature = "federation")] +fn default_store_held_for() -> Duration { + Duration::from_secs(10 * 60) +} fn default_challenge_lifetime() -> Duration { Duration::from_secs(5 * 60) } @@ -1171,6 +1256,7 @@ impl Settings { "store-neighbour-parties", !self.store_neighbour_parties.is_empty(), ), + ("store-vote-key", self.store_vote_key.is_some()), ]; let set: Vec<&str> = of_a_store .iter() @@ -1203,6 +1289,44 @@ impl Settings { { bail!("store-neighbour-parties: {party:?} isn't a party's name"); } + // Voting by itself: its key, its party's record, and rules only with them. + let voting = [ + ("store-vote-record", self.store_vote_record.is_some()), + ("store-follow", !self.store_follow.is_empty()), + ("store-follow-any", self.store_follow_any), + ("store-propose-events", self.store_propose_events.is_some()), + ("store-propose-every", self.store_propose_every.is_some()), + ]; + match (&self.store_vote_key, &self.store_vote_record) { + (Some(_), None) => { + bail!("store-vote-key: set store-vote-record, its party's one record of its votes") + } + (None, _) => { + let set: Vec<&str> = voting + .iter() + .filter(|(_, set)| *set) + .map(|(name, _)| *name) + .collect(); + if !set.is_empty() { + bail!( + "{} {} of a node voting by itself: set store-vote-key", + set.join(", "), + if set.len() == 1 { "is" } else { "are" } + ); + } + } + (Some(_), Some(_)) => {} + } + if self.store_follow_any && self.store_follow.is_empty() { + bail!("store-follow-any follows on any version: set store-follow, whom"); + } + if let Some(party) = self + .store_follow + .iter() + .find(|party| !crate::federation::version::is_party(party)) + { + bail!("store-follow: {party:?} isn't a party's name"); + } let first_change = [ ("genesis", self.genesis), ("bootstrap-users", !self.bootstrap_users.is_empty()), @@ -1282,6 +1406,8 @@ impl Settings { "store-neighbour-parties", !self.store_neighbour_parties.is_empty(), ), + #[cfg(feature = "federation")] + ("store-vote-key", self.store_vote_key.is_some()), ("alias", self.alias.is_some()), ("dashboard-url", self.dashboard_url.is_some()), ("debug-api", self.debug_api), @@ -2390,6 +2516,27 @@ mod tests { .is_err() ); assert!(checked(json!({ "store-neighbour-parties": ["lass"] })).is_err()); + let voting = |more: JsonValue| { + let mut values = json!({ "store-id": id, "store-party": "tv" }); + for (name, value) in more.as_object().unwrap() { + values[name] = value.clone(); + } + checked(values) + }; + assert!( + voting( + json!({ "store-vote-key": "k", "store-vote-record": "r", "store-follow": ["lass"] }) + ) + .is_ok() + ); + assert!(voting(json!({ "store-vote-key": "k" })).is_err()); + assert!(voting(json!({ "store-follow": ["lass"] })).is_err()); + assert!( + voting( + json!({ "store-vote-key": "k", "store-vote-record": "r", "store-follow-any": true }) + ) + .is_err() + ); assert!( checked(json!({ "store-id": id, "store-root": "root.json", "store-party": "tv" })) .is_err() diff --git a/src/federation/held.rs b/src/federation/held.rs index 07660e0..f48778d 100644 --- a/src/federation/held.rs +++ b/src/federation/held.rs @@ -20,20 +20,22 @@ //! its own has passed, random, so that not every node puts the same, and none another //! node put meanwhile (`put_again`). -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; -use std::time::Duration; +use std::time::{Duration, Instant}; use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use tracing::{error, info, warn}; -use super::event::{Does, Event, Signed}; +use super::client::Record; +use super::event::{self, Does, Event, Signed}; use super::fold::Federation; use super::store::memory::{Built, Offered, Store}; use super::store::{Fold, Id, Refused}; use super::version::Version; +use super::vote::{self, Act, Rules, Seen}; use crate::document::{Change, ChangeHash, ReadDoc, ScalarValue, events, meta, store}; use crate::files::{FileKind, Files}; use crate::locks::{MutexExt, RwLockExt}; @@ -90,10 +92,21 @@ pub(crate) struct Holding { /// The members the store names, each with its party, as it reads now: apart from the /// store, so that whoever asks doesn't wait while the store takes events. members: RwLock>>, + /// What it votes for its party with, if it votes by itself. + voter: Option, // A std mutex: never held while waiting for the document. inner: Mutex, } +/// What a node voting by itself for its party signs with, keeps and goes by +/// (`federation::vote`): a key its party's version names, kept on the node; its party's +/// one record of its votes, in a file; and its operator's rules. +pub(crate) struct Voter { + pub(crate) key: ed25519_dalek::SigningKey, + pub(crate) record: PathBuf, + pub(crate) rules: Rules, +} + #[derive(Default)] struct Inner { /// None until a document holding a store holds its root. @@ -109,6 +122,13 @@ struct Inner { unfound: bool, /// The refused changes looked at, of those screening keeps (`look`). looked: HashSet, + /// When, by its own clock, the node took each event its store took after its + /// checkpoint, and when the store moved to its checkpoint, or started there. + taken_at: HashMap, + at_checkpoint: Option<(Id, Instant)>, + /// Its party's record, voting by itself, once read; and why it isn't, said once. + record: Option, + unrecorded: Option, } /// What a document holds that its store wasn't offered from it. @@ -156,6 +176,7 @@ impl Holding { party, near: Vec::new(), members: RwLock::default(), + voter: None, inner: Mutex::new(Inner { kept, ..Inner::default() @@ -262,6 +283,16 @@ impl Holding { let Some(store) = &inner.store else { return; }; + let now = Instant::now(); + let checkpoint = Id::of(store.checkpoint()); + if inner.at_checkpoint.map(|(at, _)| at) != Some(checkpoint) { + inner.at_checkpoint = Some((checkpoint, now)); + } + let taken: HashSet<&Id> = store.taken().iter().collect(); + inner.taken_at.retain(|id, _| taken.contains(id)); + for id in store.taken() { + inner.taken_at.entry(*id).or_insert(now); + } let (own, dropped) = store.own(); let kept = Kept::of(&own, &dropped); if kept != inner.kept { @@ -401,6 +432,140 @@ impl Holding { self.members.read_or_recover().clone() } + /// Itself, voting by itself for its party as `voter` says, if it does. + pub(crate) fn voting(self, voter: Option) -> Self { + Holding { voter, ..self } + } + + /// Its party's record, read from its file, or begun on the current version where there + /// is none: one of another store or party, or that doesn't read, it votes by none. + fn record<'a>(&self, voter: &Voter, inner: &'a mut Inner) -> Option<&'a mut Record> { + let store = inner.store.as_ref()?; + let party = self.party.as_deref()?; + if inner.record.is_none() { + let read = match Record::load(&voter.record) { + Ok(record) if record.store == store.id().to_string() && record.party == party => { + Ok(record) + } + Ok(_) => Err(format!( + "{}: the record of another store or party", + voter.record.display() + )), + Err(_) if !voter.record.exists() => { + let record = Record::begin(store.id(), party, store.state().current()); + record.save(&voter.record).map(|()| record) + } + Err(why) => Err(why), + }; + match read { + Ok(record) => inner.record = Some(record), + Err(why) => { + if inner.unrecorded.as_ref() != Some(&why) { + error!(why = %why, "Voting by itself, without its party's record: votes none"); + inner.unrecorded = Some(why); + } + return None; + } + } + } + inner.record.as_mut() + } + + /// What it signs now, voting by itself by its operator's rules (`vote::decide`), or by + /// hand (`hand`, `vote::by_hand`): recorded, the record written, and offered to the + /// store; the events, to put into the document. By hand, why not where it isn't. + fn sign(&self, hand: Option<(Id, bool)>) -> Result>, String> { + let (Some(voter), Some(party)) = (&self.voter, &self.party) else { + return Err("this node doesn't vote for its party".to_owned()); + }; + let mut inner = self.inner.lock_or_recover(); + let inner = &mut *inner; + self.record(voter, inner) + .ok_or("this node holds no store, or no record of its party")?; + let (Some(store), Some(record)) = (&inner.store, &inner.record) else { + return Err("this node holds no store".to_owned()); + }; + let now = Instant::now(); + let held = |id: &Id| inner.taken_at.get(id).map(|at| now.duration_since(*at)); + let seen = Seen { + store, + held: &held, + since_move: inner + .at_checkpoint + .map_or(Duration::ZERO, |(_, at)| now.duration_since(at)), + }; + let acts = match hand { + Some((proposal, yes)) => { + vec![vote::by_hand(&seen, &voter.rules, record, proposal, yes)?] + } + None => vote::decide(&seen, &voter.rules, record), + }; + if acts.is_empty() { + return Ok(Vec::new()); + } + let current = store.state().current().clone(); + let heads = store.heads(); + let line = crate::access::sshsig::PublicKey(voter.key.verifying_key().to_bytes()).line(); + let mut record = record.clone(); + let mut signed = Vec::new(); + for act in acts { + match act { + Act::Vote { + proposal, + yes, + nays, + } => { + let cast = event::signed( + &voter.key, + party, + &line, + &heads, + event::vote(current.hash, proposal, yes), + ); + record.cast(¤t, &cast, nays); + info!(proposal = %proposal, yes, by_hand = hand.is_some(), "Voting for its party"); + signed.push(cast); + } + Act::Propose(version) => { + let proposing = + event::signed(&voter.key, party, &line, &heads, event::proposal(&version)); + record.signed(&proposing); + info!(proposal = %Id::of(&version.bytes()), "Proposing a checkpoint for its party"); + signed.push(proposing); + } + } + } + // Written before anything is sent. + record.save(&voter.record)?; + inner.record = Some(record); + let store = inner.store.as_mut().expect("held above"); + for bytes in &signed { + store.offer(bytes.clone()); + } + self.settle(inner); + Ok(signed) + } + + /// What it signs now, voting by itself by its operator's rules: to put into the + /// document. + pub(crate) fn act(&self) -> Vec> { + if self.voter.is_none() { + return Vec::new(); + } + self.sign(None).unwrap_or_else(|why| { + warn!(why = %why, "Couldn't vote by itself"); + Vec::new() + }) + } + + /// A vote by hand through this node, which votes for its party and keeps its one + /// record: the event, to put into the document, or why not. + pub(crate) fn vote_by_hand(&self, proposal: Id, yes: bool) -> Result, String> { + self.sign(Some((proposal, yes)))? + .pop() + .ok_or_else(|| "nothing to cast".to_owned()) + } + /// The party its operator says the node is of (`--store-party`). pub(crate) fn party(&self) -> Option<&str> { self.party.as_deref() @@ -497,12 +662,21 @@ pub(crate) async fn run_worker(state: crate::state::AppState) { if let Some(unread) = unread { holding.read(unread); } - !holding.lacking(&doc.blocking_read().doc).is_empty() + let signed = holding.act(); + ( + !holding.lacking(&doc.blocking_read().doc).is_empty(), + signed, + ) }); - let lacking = read.await.unwrap_or_else(|error| { + let (lacking, signed) = read.await.unwrap_or_else(|error| { error!(error = %error, "Reading the store from the document failed"); - false + (false, Vec::new()) }); + for bytes in signed { + if let Err(error) = write(&state, None, bytes).await { + warn!(error = %error, "Couldn't put what it signed for its party: put again later"); + } + } let member = state .holding .member(state.local_id(), state.holding.party()); @@ -556,6 +730,34 @@ fn wait() -> Duration { /// Puts again, in a change of its own, the events the store holds that the document /// lacks still, each after its dependencies, up to `PUT_BYTES` of them. +/// Puts `bytes`, an event, into the document at its ID unless it holds it: for `user`, who +/// put it through the API, or for the node itself. +pub(crate) async fn write( + state: &crate::state::AppState, + user: Option, + bytes: Vec, +) -> Result<(), store::StoreError> { + let id = Id::of(&bytes); + state + .mutate_doc_for( + user, + crate::document::signatures::Sign::Protocol, + move |doc| { + if events::bytes(doc, &id).is_none() { + store::write_val_path( + doc, + &events::event(&id), + &crate::document::typed::Val::Scalar(ScalarValue::Bytes(bytes)), + store::Writing::sticky(Default::default()), + )?; + } + Ok::<(), store::StoreError>(()) + }, + ) + .await + .map(|_| ()) +} + async fn put_again(state: &crate::state::AppState) { let holding = Arc::clone(&state.holding); let put = state diff --git a/src/lib.rs b/src/lib.rs index c3bfac9..4ae6657 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,6 +50,29 @@ pub async fn party() -> Result<()> { federation::party::main().await } +/// What a node voting by itself for its party goes by, where it does (`--store-vote-key`). +#[cfg(feature = "federation")] +fn voter(args: &config::Settings) -> Result> { + let (Some(key), Some(record)) = (&args.store_vote_key, &args.store_vote_record) else { + return Ok(None); + }; + let text = std::fs::read_to_string(key) + .with_context(|| format!("Reading the party's key {}", key.display()))?; + let key = federation::client::signing_key(&text) + .map_err(|why| anyhow::anyhow!("The party's key {}: {why}", key.display()))?; + Ok(Some(federation::held::Voter { + key, + record: record.clone(), + rules: federation::vote::Rules { + follow: args.store_follow.clone(), + follow_any: args.store_follow_any, + propose_events: args.store_propose_events, + propose_every: args.store_propose_every, + held_for: args.store_held_for, + }, + })) +} + /// Runs a node, as its arguments and configuration say, until it stops. pub async fn run() -> Result<()> { logging::init(); @@ -328,7 +351,8 @@ pub async fn run() -> Result<()> { .and_then(federation::store::Id::from_hex), args.store_party.clone(), )? - .near(args.store_neighbour_parties.clone()), + .near(args.store_neighbour_parties.clone()) + .voting(voter(args)?), strict_keys: args .strict_keys .iter() -- cgit v1.3.1