aboutsummaryrefslogtreecommitdiffstats
path: root/src/federation
diff options
context:
space:
mode:
Diffstat (limited to 'src/federation')
-rw-r--r--src/federation/held.rs214
1 files changed, 208 insertions, 6 deletions
diff --git a/src/federation/held.rs b/src/federation/held.rs
index 07660e0..f48778d 100644
--- a/src/federation/held.rs
+++ b/src/federation/held.rs
@@ -20,20 +20,22 @@
//! its own has passed, random, so that not every node puts the same, and none another
//! node put meanwhile (`put_again`).
-use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::path::PathBuf;
use std::sync::{Arc, Mutex, RwLock};
-use std::time::Duration;
+use std::time::{Duration, Instant};
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use tracing::{error, info, warn};
-use super::event::{Does, Event, Signed};
+use super::client::Record;
+use super::event::{self, Does, Event, Signed};
use super::fold::Federation;
use super::store::memory::{Built, Offered, Store};
use super::store::{Fold, Id, Refused};
use super::version::Version;
+use super::vote::{self, Act, Rules, Seen};
use crate::document::{Change, ChangeHash, ReadDoc, ScalarValue, events, meta, store};
use crate::files::{FileKind, Files};
use crate::locks::{MutexExt, RwLockExt};
@@ -90,10 +92,21 @@ pub(crate) struct Holding {
/// The members the store names, each with its party, as it reads now: apart from the
/// store, so that whoever asks doesn't wait while the store takes events.
members: RwLock<Option<BTreeSet<(String, String)>>>,
+ /// What it votes for its party with, if it votes by itself.
+ voter: Option<Voter>,
// A std mutex: never held while waiting for the document.
inner: Mutex<Inner>,
}
+/// What a node voting by itself for its party signs with, keeps and goes by
+/// (`federation::vote`): a key its party's version names, kept on the node; its party's
+/// one record of its votes, in a file; and its operator's rules.
+pub(crate) struct Voter {
+ pub(crate) key: ed25519_dalek::SigningKey,
+ pub(crate) record: PathBuf,
+ pub(crate) rules: Rules,
+}
+
#[derive(Default)]
struct Inner {
/// None until a document holding a store holds its root.
@@ -109,6 +122,13 @@ struct Inner {
unfound: bool,
/// The refused changes looked at, of those screening keeps (`look`).
looked: HashSet<ChangeHash>,
+ /// When, by its own clock, the node took each event its store took after its
+ /// checkpoint, and when the store moved to its checkpoint, or started there.
+ taken_at: HashMap<Id, Instant>,
+ at_checkpoint: Option<(Id, Instant)>,
+ /// Its party's record, voting by itself, once read; and why it isn't, said once.
+ record: Option<Record>,
+ unrecorded: Option<String>,
}
/// What a document holds that its store wasn't offered from it.
@@ -156,6 +176,7 @@ impl Holding {
party,
near: Vec::new(),
members: RwLock::default(),
+ voter: None,
inner: Mutex::new(Inner {
kept,
..Inner::default()
@@ -262,6 +283,16 @@ impl Holding {
let Some(store) = &inner.store else {
return;
};
+ let now = Instant::now();
+ let checkpoint = Id::of(store.checkpoint());
+ if inner.at_checkpoint.map(|(at, _)| at) != Some(checkpoint) {
+ inner.at_checkpoint = Some((checkpoint, now));
+ }
+ let taken: HashSet<&Id> = store.taken().iter().collect();
+ inner.taken_at.retain(|id, _| taken.contains(id));
+ for id in store.taken() {
+ inner.taken_at.entry(*id).or_insert(now);
+ }
let (own, dropped) = store.own();
let kept = Kept::of(&own, &dropped);
if kept != inner.kept {
@@ -401,6 +432,140 @@ impl Holding {
self.members.read_or_recover().clone()
}
+ /// Itself, voting by itself for its party as `voter` says, if it does.
+ pub(crate) fn voting(self, voter: Option<Voter>) -> Self {
+ Holding { voter, ..self }
+ }
+
+ /// Its party's record, read from its file, or begun on the current version where there
+ /// is none: one of another store or party, or that doesn't read, it votes by none.
+ fn record<'a>(&self, voter: &Voter, inner: &'a mut Inner) -> Option<&'a mut Record> {
+ let store = inner.store.as_ref()?;
+ let party = self.party.as_deref()?;
+ if inner.record.is_none() {
+ let read = match Record::load(&voter.record) {
+ Ok(record) if record.store == store.id().to_string() && record.party == party => {
+ Ok(record)
+ }
+ Ok(_) => Err(format!(
+ "{}: the record of another store or party",
+ voter.record.display()
+ )),
+ Err(_) if !voter.record.exists() => {
+ let record = Record::begin(store.id(), party, store.state().current());
+ record.save(&voter.record).map(|()| record)
+ }
+ Err(why) => Err(why),
+ };
+ match read {
+ Ok(record) => inner.record = Some(record),
+ Err(why) => {
+ if inner.unrecorded.as_ref() != Some(&why) {
+ error!(why = %why, "Voting by itself, without its party's record: votes none");
+ inner.unrecorded = Some(why);
+ }
+ return None;
+ }
+ }
+ }
+ inner.record.as_mut()
+ }
+
+ /// What it signs now, voting by itself by its operator's rules (`vote::decide`), or by
+ /// hand (`hand`, `vote::by_hand`): recorded, the record written, and offered to the
+ /// store; the events, to put into the document. By hand, why not where it isn't.
+ fn sign(&self, hand: Option<(Id, bool)>) -> Result<Vec<Vec<u8>>, String> {
+ let (Some(voter), Some(party)) = (&self.voter, &self.party) else {
+ return Err("this node doesn't vote for its party".to_owned());
+ };
+ let mut inner = self.inner.lock_or_recover();
+ let inner = &mut *inner;
+ self.record(voter, inner)
+ .ok_or("this node holds no store, or no record of its party")?;
+ let (Some(store), Some(record)) = (&inner.store, &inner.record) else {
+ return Err("this node holds no store".to_owned());
+ };
+ let now = Instant::now();
+ let held = |id: &Id| inner.taken_at.get(id).map(|at| now.duration_since(*at));
+ let seen = Seen {
+ store,
+ held: &held,
+ since_move: inner
+ .at_checkpoint
+ .map_or(Duration::ZERO, |(_, at)| now.duration_since(at)),
+ };
+ let acts = match hand {
+ Some((proposal, yes)) => {
+ vec![vote::by_hand(&seen, &voter.rules, record, proposal, yes)?]
+ }
+ None => vote::decide(&seen, &voter.rules, record),
+ };
+ if acts.is_empty() {
+ return Ok(Vec::new());
+ }
+ let current = store.state().current().clone();
+ let heads = store.heads();
+ let line = crate::access::sshsig::PublicKey(voter.key.verifying_key().to_bytes()).line();
+ let mut record = record.clone();
+ let mut signed = Vec::new();
+ for act in acts {
+ match act {
+ Act::Vote {
+ proposal,
+ yes,
+ nays,
+ } => {
+ let cast = event::signed(
+ &voter.key,
+ party,
+ &line,
+ &heads,
+ event::vote(current.hash, proposal, yes),
+ );
+ record.cast(&current, &cast, nays);
+ info!(proposal = %proposal, yes, by_hand = hand.is_some(), "Voting for its party");
+ signed.push(cast);
+ }
+ Act::Propose(version) => {
+ let proposing =
+ event::signed(&voter.key, party, &line, &heads, event::proposal(&version));
+ record.signed(&proposing);
+ info!(proposal = %Id::of(&version.bytes()), "Proposing a checkpoint for its party");
+ signed.push(proposing);
+ }
+ }
+ }
+ // Written before anything is sent.
+ record.save(&voter.record)?;
+ inner.record = Some(record);
+ let store = inner.store.as_mut().expect("held above");
+ for bytes in &signed {
+ store.offer(bytes.clone());
+ }
+ self.settle(inner);
+ Ok(signed)
+ }
+
+ /// What it signs now, voting by itself by its operator's rules: to put into the
+ /// document.
+ pub(crate) fn act(&self) -> Vec<Vec<u8>> {
+ if self.voter.is_none() {
+ return Vec::new();
+ }
+ self.sign(None).unwrap_or_else(|why| {
+ warn!(why = %why, "Couldn't vote by itself");
+ Vec::new()
+ })
+ }
+
+ /// A vote by hand through this node, which votes for its party and keeps its one
+ /// record: the event, to put into the document, or why not.
+ pub(crate) fn vote_by_hand(&self, proposal: Id, yes: bool) -> Result<Vec<u8>, String> {
+ self.sign(Some((proposal, yes)))?
+ .pop()
+ .ok_or_else(|| "nothing to cast".to_owned())
+ }
+
/// The party its operator says the node is of (`--store-party`).
pub(crate) fn party(&self) -> Option<&str> {
self.party.as_deref()
@@ -497,12 +662,21 @@ pub(crate) async fn run_worker(state: crate::state::AppState) {
if let Some(unread) = unread {
holding.read(unread);
}
- !holding.lacking(&doc.blocking_read().doc).is_empty()
+ let signed = holding.act();
+ (
+ !holding.lacking(&doc.blocking_read().doc).is_empty(),
+ signed,
+ )
});
- let lacking = read.await.unwrap_or_else(|error| {
+ let (lacking, signed) = read.await.unwrap_or_else(|error| {
error!(error = %error, "Reading the store from the document failed");
- false
+ (false, Vec::new())
});
+ for bytes in signed {
+ if let Err(error) = write(&state, None, bytes).await {
+ warn!(error = %error, "Couldn't put what it signed for its party: put again later");
+ }
+ }
let member = state
.holding
.member(state.local_id(), state.holding.party());
@@ -556,6 +730,34 @@ fn wait() -> Duration {
/// Puts again, in a change of its own, the events the store holds that the document
/// lacks still, each after its dependencies, up to `PUT_BYTES` of them.
+/// Puts `bytes`, an event, into the document at its ID unless it holds it: for `user`, who
+/// put it through the API, or for the node itself.
+pub(crate) async fn write(
+ state: &crate::state::AppState,
+ user: Option<String>,
+ bytes: Vec<u8>,
+) -> Result<(), store::StoreError> {
+ let id = Id::of(&bytes);
+ state
+ .mutate_doc_for(
+ user,
+ crate::document::signatures::Sign::Protocol,
+ move |doc| {
+ if events::bytes(doc, &id).is_none() {
+ store::write_val_path(
+ doc,
+ &events::event(&id),
+ &crate::document::typed::Val::Scalar(ScalarValue::Bytes(bytes)),
+ store::Writing::sticky(Default::default()),
+ )?;
+ }
+ Ok::<(), store::StoreError>(())
+ },
+ )
+ .await
+ .map(|_| ())
+}
+
async fn put_again(state: &crate::state::AppState) {
let holding = Arc::clone(&state.holding);
let put = state