aboutsummaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/config/mod.rs147
-rw-r--r--src/federation/held.rs214
-rw-r--r--src/lib.rs26
3 files changed, 380 insertions, 7 deletions
diff --git a/src/config/mod.rs b/src/config/mod.rs
index 762e9a4..7889d7c 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -248,6 +248,62 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[
None,
"With store-root or store-id: parties among whose members, with those of its own party, this node keeps some of its neighbours, by the graph among those alone (docs/store.md, Members): so that a party listing many nodes can't be all of them.",
),
+ #[cfg(feature = "federation")]
+ option(
+ "store-vote-key",
+ Kind::Text,
+ "PATH",
+ None,
+ "With store-root or store-id: this node votes and proposes for its party by itself, by the rules below (docs/federation.md, Votes a node casts by itself), signing with this key, an OpenSSH private key file, unencrypted, one its party's version names, kept on this node: one stolen here votes for the party. Its party then votes by hand only through this node (POST /admin/events/vote), which keeps its one record (store-vote-record).",
+ ),
+ #[cfg(feature = "federation")]
+ option(
+ "store-vote-record",
+ Kind::Text,
+ "PATH",
+ None,
+ "With store-vote-key, which needs it: its party's one record of its votes, which the node votes by, written before each vote is sent; begun, where there is none, on the version current then, on which it casts nothing.",
+ ),
+ #[cfg(feature = "federation")]
+ option(
+ "store-follow",
+ Kind::List,
+ "NAME,[NAME...]",
+ None,
+ "With store-vote-key: parties it follows, voting `true` on a proposal where one of them did, `false` where one did; by default only on a version that only moves to a checkpoint.",
+ ),
+ #[cfg(feature = "federation")]
+ option(
+ "store-follow-any",
+ Kind::Flag,
+ "BOOL",
+ Some("false"),
+ "With store-follow: follow on any version, not only on one that only moves to a checkpoint. It hands those parties the node's vote, its `false` among it.",
+ ),
+ #[cfg(feature = "federation")]
+ option(
+ "store-propose-events",
+ Kind::Integer,
+ "COUNT",
+ None,
+ "With store-vote-key: propose a version that only moves to a checkpoint once it holds this many events beyond its checkpoint, at the heads of what it held for a while (store-held-for), where nothing that can still pass is proposed on the current version; it backs what it proposes.",
+ ),
+ #[cfg(feature = "federation")]
+ option(
+ "store-propose-every",
+ Kind::Duration,
+ "DURATION",
+ None,
+ "With store-vote-key: propose a checkpoint, as store-propose-events, once this long passed by its own clock since its store moved to its checkpoint, or started there.",
+ ),
+ #[cfg(feature = "federation")]
+ option(
+ "store-held-for",
+ Kind::Duration,
+ "DURATION",
+ Some("10m"),
+ "With store-vote-key: how long it holds an event before it holds it for a while: it proposes a checkpoint at the heads of such, and backs none whose heads leave out one it held for a while before it was proposed.",
+ ),
option(
"strict-keys",
Kind::List,
@@ -692,6 +748,31 @@ pub(crate) struct Settings {
#[cfg(feature = "federation")]
#[serde(default)]
pub(crate) store_neighbour_parties: Vec<String>,
+ #[cfg(feature = "federation")]
+ pub(crate) store_vote_key: Option<PathBuf>,
+ #[cfg(feature = "federation")]
+ pub(crate) store_vote_record: Option<PathBuf>,
+ #[cfg(feature = "federation")]
+ #[serde(default)]
+ pub(crate) store_follow: Vec<String>,
+ #[cfg(feature = "federation")]
+ #[serde(default)]
+ pub(crate) store_follow_any: bool,
+ #[cfg(feature = "federation")]
+ pub(crate) store_propose_events: Option<usize>,
+ #[cfg(feature = "federation")]
+ #[serde(
+ default,
+ deserialize_with = "crate::units::optional_duration",
+ serialize_with = "crate::units::serialize_optional_duration"
+ )]
+ pub(crate) store_propose_every: Option<Duration>,
+ #[cfg(feature = "federation")]
+ #[serde(
+ default = "default_store_held_for",
+ with = "crate::units::duration_option"
+ )]
+ pub(crate) store_held_for: Duration,
#[serde(default)]
pub(crate) strict_keys: Vec<String>,
@@ -883,6 +964,10 @@ fn default_token_lifetime() -> Duration {
fn default_session_lifetime() -> Duration {
Duration::from_secs(7 * 24 * 3600)
}
+#[cfg(feature = "federation")]
+fn default_store_held_for() -> Duration {
+ Duration::from_secs(10 * 60)
+}
fn default_challenge_lifetime() -> Duration {
Duration::from_secs(5 * 60)
}
@@ -1171,6 +1256,7 @@ impl Settings {
"store-neighbour-parties",
!self.store_neighbour_parties.is_empty(),
),
+ ("store-vote-key", self.store_vote_key.is_some()),
];
let set: Vec<&str> = of_a_store
.iter()
@@ -1203,6 +1289,44 @@ impl Settings {
{
bail!("store-neighbour-parties: {party:?} isn't a party's name");
}
+ // Voting by itself: its key, its party's record, and rules only with them.
+ let voting = [
+ ("store-vote-record", self.store_vote_record.is_some()),
+ ("store-follow", !self.store_follow.is_empty()),
+ ("store-follow-any", self.store_follow_any),
+ ("store-propose-events", self.store_propose_events.is_some()),
+ ("store-propose-every", self.store_propose_every.is_some()),
+ ];
+ match (&self.store_vote_key, &self.store_vote_record) {
+ (Some(_), None) => {
+ bail!("store-vote-key: set store-vote-record, its party's one record of its votes")
+ }
+ (None, _) => {
+ let set: Vec<&str> = voting
+ .iter()
+ .filter(|(_, set)| *set)
+ .map(|(name, _)| *name)
+ .collect();
+ if !set.is_empty() {
+ bail!(
+ "{} {} of a node voting by itself: set store-vote-key",
+ set.join(", "),
+ if set.len() == 1 { "is" } else { "are" }
+ );
+ }
+ }
+ (Some(_), Some(_)) => {}
+ }
+ if self.store_follow_any && self.store_follow.is_empty() {
+ bail!("store-follow-any follows on any version: set store-follow, whom");
+ }
+ if let Some(party) = self
+ .store_follow
+ .iter()
+ .find(|party| !crate::federation::version::is_party(party))
+ {
+ bail!("store-follow: {party:?} isn't a party's name");
+ }
let first_change = [
("genesis", self.genesis),
("bootstrap-users", !self.bootstrap_users.is_empty()),
@@ -1282,6 +1406,8 @@ impl Settings {
"store-neighbour-parties",
!self.store_neighbour_parties.is_empty(),
),
+ #[cfg(feature = "federation")]
+ ("store-vote-key", self.store_vote_key.is_some()),
("alias", self.alias.is_some()),
("dashboard-url", self.dashboard_url.is_some()),
("debug-api", self.debug_api),
@@ -2390,6 +2516,27 @@ mod tests {
.is_err()
);
assert!(checked(json!({ "store-neighbour-parties": ["lass"] })).is_err());
+ let voting = |more: JsonValue| {
+ let mut values = json!({ "store-id": id, "store-party": "tv" });
+ for (name, value) in more.as_object().unwrap() {
+ values[name] = value.clone();
+ }
+ checked(values)
+ };
+ assert!(
+ voting(
+ json!({ "store-vote-key": "k", "store-vote-record": "r", "store-follow": ["lass"] })
+ )
+ .is_ok()
+ );
+ assert!(voting(json!({ "store-vote-key": "k" })).is_err());
+ assert!(voting(json!({ "store-follow": ["lass"] })).is_err());
+ assert!(
+ voting(
+ json!({ "store-vote-key": "k", "store-vote-record": "r", "store-follow-any": true })
+ )
+ .is_err()
+ );
assert!(
checked(json!({ "store-id": id, "store-root": "root.json", "store-party": "tv" }))
.is_err()
diff --git a/src/federation/held.rs b/src/federation/held.rs
index 07660e0..f48778d 100644
--- a/src/federation/held.rs
+++ b/src/federation/held.rs
@@ -20,20 +20,22 @@
//! its own has passed, random, so that not every node puts the same, and none another
//! node put meanwhile (`put_again`).
-use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::path::PathBuf;
use std::sync::{Arc, Mutex, RwLock};
-use std::time::Duration;
+use std::time::{Duration, Instant};
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use tracing::{error, info, warn};
-use super::event::{Does, Event, Signed};
+use super::client::Record;
+use super::event::{self, Does, Event, Signed};
use super::fold::Federation;
use super::store::memory::{Built, Offered, Store};
use super::store::{Fold, Id, Refused};
use super::version::Version;
+use super::vote::{self, Act, Rules, Seen};
use crate::document::{Change, ChangeHash, ReadDoc, ScalarValue, events, meta, store};
use crate::files::{FileKind, Files};
use crate::locks::{MutexExt, RwLockExt};
@@ -90,10 +92,21 @@ pub(crate) struct Holding {
/// The members the store names, each with its party, as it reads now: apart from the
/// store, so that whoever asks doesn't wait while the store takes events.
members: RwLock<Option<BTreeSet<(String, String)>>>,
+ /// What it votes for its party with, if it votes by itself.
+ voter: Option<Voter>,
// A std mutex: never held while waiting for the document.
inner: Mutex<Inner>,
}
+/// What a node voting by itself for its party signs with, keeps and goes by
+/// (`federation::vote`): a key its party's version names, kept on the node; its party's
+/// one record of its votes, in a file; and its operator's rules.
+pub(crate) struct Voter {
+ pub(crate) key: ed25519_dalek::SigningKey,
+ pub(crate) record: PathBuf,
+ pub(crate) rules: Rules,
+}
+
#[derive(Default)]
struct Inner {
/// None until a document holding a store holds its root.
@@ -109,6 +122,13 @@ struct Inner {
unfound: bool,
/// The refused changes looked at, of those screening keeps (`look`).
looked: HashSet<ChangeHash>,
+ /// When, by its own clock, the node took each event its store took after its
+ /// checkpoint, and when the store moved to its checkpoint, or started there.
+ taken_at: HashMap<Id, Instant>,
+ at_checkpoint: Option<(Id, Instant)>,
+ /// Its party's record, voting by itself, once read; and why it isn't, said once.
+ record: Option<Record>,
+ unrecorded: Option<String>,
}
/// What a document holds that its store wasn't offered from it.
@@ -156,6 +176,7 @@ impl Holding {
party,
near: Vec::new(),
members: RwLock::default(),
+ voter: None,
inner: Mutex::new(Inner {
kept,
..Inner::default()
@@ -262,6 +283,16 @@ impl Holding {
let Some(store) = &inner.store else {
return;
};
+ let now = Instant::now();
+ let checkpoint = Id::of(store.checkpoint());
+ if inner.at_checkpoint.map(|(at, _)| at) != Some(checkpoint) {
+ inner.at_checkpoint = Some((checkpoint, now));
+ }
+ let taken: HashSet<&Id> = store.taken().iter().collect();
+ inner.taken_at.retain(|id, _| taken.contains(id));
+ for id in store.taken() {
+ inner.taken_at.entry(*id).or_insert(now);
+ }
let (own, dropped) = store.own();
let kept = Kept::of(&own, &dropped);
if kept != inner.kept {
@@ -401,6 +432,140 @@ impl Holding {
self.members.read_or_recover().clone()
}
+ /// Itself, voting by itself for its party as `voter` says, if it does.
+ pub(crate) fn voting(self, voter: Option<Voter>) -> Self {
+ Holding { voter, ..self }
+ }
+
+ /// Its party's record, read from its file, or begun on the current version where there
+ /// is none: one of another store or party, or that doesn't read, it votes by none.
+ fn record<'a>(&self, voter: &Voter, inner: &'a mut Inner) -> Option<&'a mut Record> {
+ let store = inner.store.as_ref()?;
+ let party = self.party.as_deref()?;
+ if inner.record.is_none() {
+ let read = match Record::load(&voter.record) {
+ Ok(record) if record.store == store.id().to_string() && record.party == party => {
+ Ok(record)
+ }
+ Ok(_) => Err(format!(
+ "{}: the record of another store or party",
+ voter.record.display()
+ )),
+ Err(_) if !voter.record.exists() => {
+ let record = Record::begin(store.id(), party, store.state().current());
+ record.save(&voter.record).map(|()| record)
+ }
+ Err(why) => Err(why),
+ };
+ match read {
+ Ok(record) => inner.record = Some(record),
+ Err(why) => {
+ if inner.unrecorded.as_ref() != Some(&why) {
+ error!(why = %why, "Voting by itself, without its party's record: votes none");
+ inner.unrecorded = Some(why);
+ }
+ return None;
+ }
+ }
+ }
+ inner.record.as_mut()
+ }
+
+ /// What it signs now, voting by itself by its operator's rules (`vote::decide`), or by
+ /// hand (`hand`, `vote::by_hand`): recorded, the record written, and offered to the
+ /// store; the events, to put into the document. By hand, why not where it isn't.
+ fn sign(&self, hand: Option<(Id, bool)>) -> Result<Vec<Vec<u8>>, String> {
+ let (Some(voter), Some(party)) = (&self.voter, &self.party) else {
+ return Err("this node doesn't vote for its party".to_owned());
+ };
+ let mut inner = self.inner.lock_or_recover();
+ let inner = &mut *inner;
+ self.record(voter, inner)
+ .ok_or("this node holds no store, or no record of its party")?;
+ let (Some(store), Some(record)) = (&inner.store, &inner.record) else {
+ return Err("this node holds no store".to_owned());
+ };
+ let now = Instant::now();
+ let held = |id: &Id| inner.taken_at.get(id).map(|at| now.duration_since(*at));
+ let seen = Seen {
+ store,
+ held: &held,
+ since_move: inner
+ .at_checkpoint
+ .map_or(Duration::ZERO, |(_, at)| now.duration_since(at)),
+ };
+ let acts = match hand {
+ Some((proposal, yes)) => {
+ vec![vote::by_hand(&seen, &voter.rules, record, proposal, yes)?]
+ }
+ None => vote::decide(&seen, &voter.rules, record),
+ };
+ if acts.is_empty() {
+ return Ok(Vec::new());
+ }
+ let current = store.state().current().clone();
+ let heads = store.heads();
+ let line = crate::access::sshsig::PublicKey(voter.key.verifying_key().to_bytes()).line();
+ let mut record = record.clone();
+ let mut signed = Vec::new();
+ for act in acts {
+ match act {
+ Act::Vote {
+ proposal,
+ yes,
+ nays,
+ } => {
+ let cast = event::signed(
+ &voter.key,
+ party,
+ &line,
+ &heads,
+ event::vote(current.hash, proposal, yes),
+ );
+ record.cast(&current, &cast, nays);
+ info!(proposal = %proposal, yes, by_hand = hand.is_some(), "Voting for its party");
+ signed.push(cast);
+ }
+ Act::Propose(version) => {
+ let proposing =
+ event::signed(&voter.key, party, &line, &heads, event::proposal(&version));
+ record.signed(&proposing);
+ info!(proposal = %Id::of(&version.bytes()), "Proposing a checkpoint for its party");
+ signed.push(proposing);
+ }
+ }
+ }
+ // Written before anything is sent.
+ record.save(&voter.record)?;
+ inner.record = Some(record);
+ let store = inner.store.as_mut().expect("held above");
+ for bytes in &signed {
+ store.offer(bytes.clone());
+ }
+ self.settle(inner);
+ Ok(signed)
+ }
+
+ /// What it signs now, voting by itself by its operator's rules: to put into the
+ /// document.
+ pub(crate) fn act(&self) -> Vec<Vec<u8>> {
+ if self.voter.is_none() {
+ return Vec::new();
+ }
+ self.sign(None).unwrap_or_else(|why| {
+ warn!(why = %why, "Couldn't vote by itself");
+ Vec::new()
+ })
+ }
+
+ /// A vote by hand through this node, which votes for its party and keeps its one
+ /// record: the event, to put into the document, or why not.
+ pub(crate) fn vote_by_hand(&self, proposal: Id, yes: bool) -> Result<Vec<u8>, String> {
+ self.sign(Some((proposal, yes)))?
+ .pop()
+ .ok_or_else(|| "nothing to cast".to_owned())
+ }
+
/// The party its operator says the node is of (`--store-party`).
pub(crate) fn party(&self) -> Option<&str> {
self.party.as_deref()
@@ -497,12 +662,21 @@ pub(crate) async fn run_worker(state: crate::state::AppState) {
if let Some(unread) = unread {
holding.read(unread);
}
- !holding.lacking(&doc.blocking_read().doc).is_empty()
+ let signed = holding.act();
+ (
+ !holding.lacking(&doc.blocking_read().doc).is_empty(),
+ signed,
+ )
});
- let lacking = read.await.unwrap_or_else(|error| {
+ let (lacking, signed) = read.await.unwrap_or_else(|error| {
error!(error = %error, "Reading the store from the document failed");
- false
+ (false, Vec::new())
});
+ for bytes in signed {
+ if let Err(error) = write(&state, None, bytes).await {
+ warn!(error = %error, "Couldn't put what it signed for its party: put again later");
+ }
+ }
let member = state
.holding
.member(state.local_id(), state.holding.party());
@@ -556,6 +730,34 @@ fn wait() -> Duration {
/// Puts again, in a change of its own, the events the store holds that the document
/// lacks still, each after its dependencies, up to `PUT_BYTES` of them.
+/// Puts `bytes`, an event, into the document at its ID unless it holds it: for `user`, who
+/// put it through the API, or for the node itself.
+pub(crate) async fn write(
+ state: &crate::state::AppState,
+ user: Option<String>,
+ bytes: Vec<u8>,
+) -> Result<(), store::StoreError> {
+ let id = Id::of(&bytes);
+ state
+ .mutate_doc_for(
+ user,
+ crate::document::signatures::Sign::Protocol,
+ move |doc| {
+ if events::bytes(doc, &id).is_none() {
+ store::write_val_path(
+ doc,
+ &events::event(&id),
+ &crate::document::typed::Val::Scalar(ScalarValue::Bytes(bytes)),
+ store::Writing::sticky(Default::default()),
+ )?;
+ }
+ Ok::<(), store::StoreError>(())
+ },
+ )
+ .await
+ .map(|_| ())
+}
+
async fn put_again(state: &crate::state::AppState) {
let holding = Arc::clone(&state.holding);
let put = state
diff --git a/src/lib.rs b/src/lib.rs
index c3bfac9..4ae6657 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -50,6 +50,29 @@ pub async fn party() -> Result<()> {
federation::party::main().await
}
+/// What a node voting by itself for its party goes by, where it does (`--store-vote-key`).
+#[cfg(feature = "federation")]
+fn voter(args: &config::Settings) -> Result<Option<federation::held::Voter>> {
+ let (Some(key), Some(record)) = (&args.store_vote_key, &args.store_vote_record) else {
+ return Ok(None);
+ };
+ let text = std::fs::read_to_string(key)
+ .with_context(|| format!("Reading the party's key {}", key.display()))?;
+ let key = federation::client::signing_key(&text)
+ .map_err(|why| anyhow::anyhow!("The party's key {}: {why}", key.display()))?;
+ Ok(Some(federation::held::Voter {
+ key,
+ record: record.clone(),
+ rules: federation::vote::Rules {
+ follow: args.store_follow.clone(),
+ follow_any: args.store_follow_any,
+ propose_events: args.store_propose_events,
+ propose_every: args.store_propose_every,
+ held_for: args.store_held_for,
+ },
+ }))
+}
+
/// Runs a node, as its arguments and configuration say, until it stops.
pub async fn run() -> Result<()> {
logging::init();
@@ -328,7 +351,8 @@ pub async fn run() -> Result<()> {
.and_then(federation::store::Id::from_hex),
args.store_party.clone(),
)?
- .near(args.store_neighbour_parties.clone()),
+ .near(args.store_neighbour_parties.clone())
+ .voting(voter(args)?),
strict_keys: args
.strict_keys
.iter()