diff options
Diffstat (limited to 'src/federation')
| -rw-r--r-- | src/federation/held.rs | 214 |
1 files changed, 208 insertions, 6 deletions
diff --git a/src/federation/held.rs b/src/federation/held.rs index 07660e0..f48778d 100644 --- a/src/federation/held.rs +++ b/src/federation/held.rs @@ -20,20 +20,22 @@ //! its own has passed, random, so that not every node puts the same, and none another //! node put meanwhile (`put_again`). -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; -use std::time::Duration; +use std::time::{Duration, Instant}; use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use tracing::{error, info, warn}; -use super::event::{Does, Event, Signed}; +use super::client::Record; +use super::event::{self, Does, Event, Signed}; use super::fold::Federation; use super::store::memory::{Built, Offered, Store}; use super::store::{Fold, Id, Refused}; use super::version::Version; +use super::vote::{self, Act, Rules, Seen}; use crate::document::{Change, ChangeHash, ReadDoc, ScalarValue, events, meta, store}; use crate::files::{FileKind, Files}; use crate::locks::{MutexExt, RwLockExt}; @@ -90,10 +92,21 @@ pub(crate) struct Holding { /// The members the store names, each with its party, as it reads now: apart from the /// store, so that whoever asks doesn't wait while the store takes events. members: RwLock<Option<BTreeSet<(String, String)>>>, + /// What it votes for its party with, if it votes by itself. + voter: Option<Voter>, // A std mutex: never held while waiting for the document. inner: Mutex<Inner>, } +/// What a node voting by itself for its party signs with, keeps and goes by +/// (`federation::vote`): a key its party's version names, kept on the node; its party's +/// one record of its votes, in a file; and its operator's rules. +pub(crate) struct Voter { + pub(crate) key: ed25519_dalek::SigningKey, + pub(crate) record: PathBuf, + pub(crate) rules: Rules, +} + #[derive(Default)] struct Inner { /// None until a document holding a store holds its root. @@ -109,6 +122,13 @@ struct Inner { unfound: bool, /// The refused changes looked at, of those screening keeps (`look`). looked: HashSet<ChangeHash>, + /// When, by its own clock, the node took each event its store took after its + /// checkpoint, and when the store moved to its checkpoint, or started there. + taken_at: HashMap<Id, Instant>, + at_checkpoint: Option<(Id, Instant)>, + /// Its party's record, voting by itself, once read; and why it isn't, said once. + record: Option<Record>, + unrecorded: Option<String>, } /// What a document holds that its store wasn't offered from it. @@ -156,6 +176,7 @@ impl Holding { party, near: Vec::new(), members: RwLock::default(), + voter: None, inner: Mutex::new(Inner { kept, ..Inner::default() @@ -262,6 +283,16 @@ impl Holding { let Some(store) = &inner.store else { return; }; + let now = Instant::now(); + let checkpoint = Id::of(store.checkpoint()); + if inner.at_checkpoint.map(|(at, _)| at) != Some(checkpoint) { + inner.at_checkpoint = Some((checkpoint, now)); + } + let taken: HashSet<&Id> = store.taken().iter().collect(); + inner.taken_at.retain(|id, _| taken.contains(id)); + for id in store.taken() { + inner.taken_at.entry(*id).or_insert(now); + } let (own, dropped) = store.own(); let kept = Kept::of(&own, &dropped); if kept != inner.kept { @@ -401,6 +432,140 @@ impl Holding { self.members.read_or_recover().clone() } + /// Itself, voting by itself for its party as `voter` says, if it does. + pub(crate) fn voting(self, voter: Option<Voter>) -> Self { + Holding { voter, ..self } + } + + /// Its party's record, read from its file, or begun on the current version where there + /// is none: one of another store or party, or that doesn't read, it votes by none. + fn record<'a>(&self, voter: &Voter, inner: &'a mut Inner) -> Option<&'a mut Record> { + let store = inner.store.as_ref()?; + let party = self.party.as_deref()?; + if inner.record.is_none() { + let read = match Record::load(&voter.record) { + Ok(record) if record.store == store.id().to_string() && record.party == party => { + Ok(record) + } + Ok(_) => Err(format!( + "{}: the record of another store or party", + voter.record.display() + )), + Err(_) if !voter.record.exists() => { + let record = Record::begin(store.id(), party, store.state().current()); + record.save(&voter.record).map(|()| record) + } + Err(why) => Err(why), + }; + match read { + Ok(record) => inner.record = Some(record), + Err(why) => { + if inner.unrecorded.as_ref() != Some(&why) { + error!(why = %why, "Voting by itself, without its party's record: votes none"); + inner.unrecorded = Some(why); + } + return None; + } + } + } + inner.record.as_mut() + } + + /// What it signs now, voting by itself by its operator's rules (`vote::decide`), or by + /// hand (`hand`, `vote::by_hand`): recorded, the record written, and offered to the + /// store; the events, to put into the document. By hand, why not where it isn't. + fn sign(&self, hand: Option<(Id, bool)>) -> Result<Vec<Vec<u8>>, String> { + let (Some(voter), Some(party)) = (&self.voter, &self.party) else { + return Err("this node doesn't vote for its party".to_owned()); + }; + let mut inner = self.inner.lock_or_recover(); + let inner = &mut *inner; + self.record(voter, inner) + .ok_or("this node holds no store, or no record of its party")?; + let (Some(store), Some(record)) = (&inner.store, &inner.record) else { + return Err("this node holds no store".to_owned()); + }; + let now = Instant::now(); + let held = |id: &Id| inner.taken_at.get(id).map(|at| now.duration_since(*at)); + let seen = Seen { + store, + held: &held, + since_move: inner + .at_checkpoint + .map_or(Duration::ZERO, |(_, at)| now.duration_since(at)), + }; + let acts = match hand { + Some((proposal, yes)) => { + vec![vote::by_hand(&seen, &voter.rules, record, proposal, yes)?] + } + None => vote::decide(&seen, &voter.rules, record), + }; + if acts.is_empty() { + return Ok(Vec::new()); + } + let current = store.state().current().clone(); + let heads = store.heads(); + let line = crate::access::sshsig::PublicKey(voter.key.verifying_key().to_bytes()).line(); + let mut record = record.clone(); + let mut signed = Vec::new(); + for act in acts { + match act { + Act::Vote { + proposal, + yes, + nays, + } => { + let cast = event::signed( + &voter.key, + party, + &line, + &heads, + event::vote(current.hash, proposal, yes), + ); + record.cast(¤t, &cast, nays); + info!(proposal = %proposal, yes, by_hand = hand.is_some(), "Voting for its party"); + signed.push(cast); + } + Act::Propose(version) => { + let proposing = + event::signed(&voter.key, party, &line, &heads, event::proposal(&version)); + record.signed(&proposing); + info!(proposal = %Id::of(&version.bytes()), "Proposing a checkpoint for its party"); + signed.push(proposing); + } + } + } + // Written before anything is sent. + record.save(&voter.record)?; + inner.record = Some(record); + let store = inner.store.as_mut().expect("held above"); + for bytes in &signed { + store.offer(bytes.clone()); + } + self.settle(inner); + Ok(signed) + } + + /// What it signs now, voting by itself by its operator's rules: to put into the + /// document. + pub(crate) fn act(&self) -> Vec<Vec<u8>> { + if self.voter.is_none() { + return Vec::new(); + } + self.sign(None).unwrap_or_else(|why| { + warn!(why = %why, "Couldn't vote by itself"); + Vec::new() + }) + } + + /// A vote by hand through this node, which votes for its party and keeps its one + /// record: the event, to put into the document, or why not. + pub(crate) fn vote_by_hand(&self, proposal: Id, yes: bool) -> Result<Vec<u8>, String> { + self.sign(Some((proposal, yes)))? + .pop() + .ok_or_else(|| "nothing to cast".to_owned()) + } + /// The party its operator says the node is of (`--store-party`). pub(crate) fn party(&self) -> Option<&str> { self.party.as_deref() @@ -497,12 +662,21 @@ pub(crate) async fn run_worker(state: crate::state::AppState) { if let Some(unread) = unread { holding.read(unread); } - !holding.lacking(&doc.blocking_read().doc).is_empty() + let signed = holding.act(); + ( + !holding.lacking(&doc.blocking_read().doc).is_empty(), + signed, + ) }); - let lacking = read.await.unwrap_or_else(|error| { + let (lacking, signed) = read.await.unwrap_or_else(|error| { error!(error = %error, "Reading the store from the document failed"); - false + (false, Vec::new()) }); + for bytes in signed { + if let Err(error) = write(&state, None, bytes).await { + warn!(error = %error, "Couldn't put what it signed for its party: put again later"); + } + } let member = state .holding .member(state.local_id(), state.holding.party()); @@ -556,6 +730,34 @@ fn wait() -> Duration { /// Puts again, in a change of its own, the events the store holds that the document /// lacks still, each after its dependencies, up to `PUT_BYTES` of them. +/// Puts `bytes`, an event, into the document at its ID unless it holds it: for `user`, who +/// put it through the API, or for the node itself. +pub(crate) async fn write( + state: &crate::state::AppState, + user: Option<String>, + bytes: Vec<u8>, +) -> Result<(), store::StoreError> { + let id = Id::of(&bytes); + state + .mutate_doc_for( + user, + crate::document::signatures::Sign::Protocol, + move |doc| { + if events::bytes(doc, &id).is_none() { + store::write_val_path( + doc, + &events::event(&id), + &crate::document::typed::Val::Scalar(ScalarValue::Bytes(bytes)), + store::Writing::sticky(Default::default()), + )?; + } + Ok::<(), store::StoreError>(()) + }, + ) + .await + .map(|_| ()) +} + async fn put_again(state: &crate::state::AppState) { let holding = Arc::clone(&state.holding); let put = state |
