From 7a5aeb02024ddff71ba60f40913555cd21a633ad Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 19 Nov 2026 13:33:51 +0000 Subject: transport: SPEAKS, which wire versions a node speaks and what it knows Before each HELLO and each HELLO_REPLY, a node sends SPEAKS: the oldest and the newest wire version it speaks, and the ID of the newest revision of the federation it knows. It stands outside the wire versions, its first byte 0, which no version is, laid out as no build changes: that byte, the oldest, the newest, then 32 bytes of the revision's ID where there is one; a reader takes those and leaves whatever follows, which a later build may add. So a peer on another build reads it too, and nodes on different builds can say what they run, which a rollout needs. A build from before it takes it for a message of another wire version and logs that once of the peer (version=0), reading the rest of the peer's messages as before: SPEAKS rolls out node by node, with no change of VERSION. The node keeps what each peer said last (`swarm::speaking`, at most 4096 peers). It counts as `hello` traffic. `OLDEST`, the oldest version this build reads, is `VERSION` for now: a range for a later build to widen. operations.md also says what is so of the other-version warning now: every transport logs it once per peer at warn, not Mycelium alone. --- docs/architecture.md | 4 +-- docs/measurements.md | 2 +- docs/operations.md | 16 ++++++---- docs/peers.md | 8 +++++ src/node.rs | 1 + src/state/mod.rs | 2 ++ src/swarm/mod.rs | 1 + src/swarm/peer/handshake.rs | 11 +++++++ src/swarm/peer/inbound.rs | 8 ++++- src/swarm/speaking.rs | 46 +++++++++++++++++++++++++++++ src/transport/mod.rs | 19 +++++++++++- src/transport/traffic.rs | 6 ++-- src/transport/wire.rs | 71 ++++++++++++++++++++++++++++++++++++++++++++- 13 files changed, 182 insertions(+), 13 deletions(-) create mode 100644 src/swarm/speaking.rs diff --git a/docs/architecture.md b/docs/architecture.md index 8fc152f..202495d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -78,10 +78,10 @@ it switched to the right document without trusting anyone. ## Modules **[`transport/`](../src/transport/)** — [`mod.rs`](../src/transport/mod.rs) defines the message types (HELLO, HELLO_REPLY, sync -envelope, STATUS/STATUS_REPLY, PING/PONG, blob messages, PROBE/PROBE_REPLY, REFUSALS) and the `Transport` trait; +envelope, STATUS/STATUS_REPLY, PING/PONG, blob messages, PROBE/PROBE_REPLY, REFUSALS, SPEAKS) and the `Transport` trait; [`wire.rs`](../src/transport/wire.rs) is the wire format (a version byte, bumped with every change to the format, and a kind byte, then the message in MessagePack, derived from the types in [`mod.rs`](../src/transport/mod.rs) -with codecs for hashes and IDs); [`mycelium/`](../src/transport/mycelium/) (with the `mycelium` feature) is the REST +with codecs for hashes and IDs; SPEAKS, outside the versions, is laid out as no build changes); [`mycelium/`](../src/transport/mycelium/) (with the `mycelium` feature) is the REST client (polling inbox, outbound POST, one topic, `continuum`, for every message), with [`mycelium/daemon.rs`](../src/transport/mycelium/daemon.rs) running the daemon (an external one by its URL, or one it starts and supervises); [`libp2p/`](../src/transport/libp2p/) (with the `libp2p` feature) is libp2p as a transport, diff --git a/docs/measurements.md b/docs/measurements.md index d54f07b..2adc9ed 100644 --- a/docs/measurements.md +++ b/docs/measurements.md @@ -185,7 +185,7 @@ clocks"); setting them is the operating system's job (NTP). ## Traffic Messages and payload bytes by direction (`sent`, `received`, `refused`) and kind -(`hello`, `helloReply`, `sync`, `status`, `statusReply`, `blob`, `identity` for WHO and +(`hello` for HELLO and SPEAKS, `helloReply`, `sync`, `status`, `statusReply`, `blob`, `identity` for WHO and ME, `paths` for PING and PONG, `probe` for PROBE and PROBE_REPLY, `plumtree` for PRUNE and GRAFT, `hyparview` for NEIGHBOUR, NEIGHBOUR_REPLY and DISCONNECT, `refusals` for REFUSALS). The layers below the node count WHO and ME, and the diff --git a/docs/operations.md b/docs/operations.md index e1b8e79..4b42992 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -98,10 +98,15 @@ One node at a time, restarting each with its files. What to know: * **The wire format.** Each change to it bumps a version byte, and nodes ignore messages of another version. Across such a change, upgraded nodes and the others don't talk: the swarm is split until every node is upgraded, both sides go on - writing, and they merge then. Mycelium logs it (`Peer speaks another wire version - (another build); ignoring its messages`, warn, once per peer); libp2p and Tor at debug - only (`Dropping a message that doesn't decode`). Until the first release, assume - every upgrade is such a change, and upgrade all nodes close together. + writing, and they merge then. Each logs it (`Peer speaks another wire version + (another build); ignoring its messages`, warn, once per peer). With each handshake, a + node says which versions it speaks, and the newest revision of the federation it knows + (SPEAKS, outside the versions; [`peers.md`](peers.md), + [Handshake and join](peers.md#handshake-and-join)); a build from before SPEAKS logs it + as that warning, `version=0`, once of the peer, and reads the peer's other messages + where they are of its version. + Until the first release, assume every upgrade is such a change, and upgrade all nodes + close together. * **Automerge.** A remake needs every node to build the same snapshot, byte for byte, so the same Automerge version everywhere; a node with another one rejoins instead of switching ([`remakes.md`](remakes.md), [Limits](remakes.md#limits)). Don't remake while @@ -246,7 +251,8 @@ And from the log, at warn or error: read-only disk; * `The … file is readable by others than its owner; chmod 600 it`; * `Peer speaks another wire version (another build); ignoring its messages`: a node on - another build. + another build; with `version=0`, only a newer build's SPEAKS, which this build doesn't + read. ## What a node uses diff --git a/docs/peers.md b/docs/peers.md index e71d726..c63bebe 100644 --- a/docs/peers.md +++ b/docs/peers.md @@ -95,6 +95,14 @@ only valid identities (node IDs), and, once it knows the member list, only its own neighbours; their addresses are hints for its transport, and each gets a HELLO. +Before each HELLO and each HELLO_REPLY, a node sends SPEAKS: the wire versions it speaks +and the newest revision of the federation it knows, as its build has them. It stands +outside the wire versions, laid out as no build changes, so that a peer on another build +reads it too, once that peer reads the node's WHO and ME +([Node IDs and addresses](#node-ids-and-addresses)). A build from before SPEAKS takes it +for a message of another wire version, and logs that once of the peer +([`operations.md`](operations.md), [Upgrading](operations.md#upgrading)). + Every HELLO carries a random id of the sender's current run, and the HELLO_REPLY echoes it. A reply with another id answers a HELLO from an earlier run (delivered late, or queued in Mycelium while the node was down) and is ignored. diff --git a/src/node.rs b/src/node.rs index 4e19da5..22aa0df 100644 --- a/src/node.rs +++ b/src/node.rs @@ -191,6 +191,7 @@ impl Node { clock_ahead_ms: Default::default(), clock_skew: Default::default(), blobs: Arc::new(swarm::blob::Blobs::new(config.blob_chunk_bytes)), + speaking: Arc::default(), protocol_writers: Arc::new(config.protocol_writers), first_users: Arc::new(config.first_users), first_policy: config.first_policy, diff --git a/src/state/mod.rs b/src/state/mod.rs index 05eef1a..cbdc509 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -140,6 +140,8 @@ pub(crate) struct AppState { pub(crate) clock_skew: Arc>, /// Chunked transfers served and fetched (`swarm::blob`). pub(crate) blobs: Arc, + /// What the peers' builds speak and know, as each said (`swarm::speaking`). + pub(crate) speaking: Arc, /// HTTP API users who may write protocol data through the store API /// (`--protocol-writers`). pub(crate) protocol_writers: Arc>, diff --git a/src/swarm/mod.rs b/src/swarm/mod.rs index ad7f83e..6c0a242 100644 --- a/src/swarm/mod.rs +++ b/src/swarm/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod roots; pub(crate) mod rtt; pub(crate) mod skew; pub(crate) mod source; +pub(crate) mod speaking; pub(crate) mod stability; pub(crate) mod state_file; pub(crate) mod stats; diff --git a/src/swarm/peer/handshake.rs b/src/swarm/peer/handshake.rs index 4cc0cf4..ef7d379 100644 --- a/src/swarm/peer/handshake.rs +++ b/src/swarm/peer/handshake.rs @@ -127,6 +127,7 @@ async fn transmit_hello(state: &AppState, peer: &str) -> Result<()> { None => 0, }; state.lock_rtt().hello_sent(peer, Instant::now()); + say_what_we_speak(state, peer).await; let result = state .transport .send_message( @@ -169,6 +170,15 @@ async fn transmit_hello(state: &AppState, peer: &str) -> Result<()> { } } +/// Tells `peer` what this build speaks and knows, with each handshake (SPEAKS, +/// `swarm::speaking`): a build from before SPEAKS ignores it. +async fn say_what_we_speak(state: &AppState, peer: &str) { + let speaks = ParsedMessage::Speaks(crate::swarm::speaking::own()); + if let Err(error) = state.transport.send_message(peer, &speaks).await { + debug!(peer = %peer, error = %format!("{error:#}"), "SPEAKS not sent"); + } +} + pub(super) async fn ensure_handshake(state: &AppState, peer: &str) { if let Err(error) = send_hello_once(state, peer).await { warn!(peer = %peer, error = %format!("{error:#}"), "HELLO failed; will retry"); @@ -247,6 +257,7 @@ pub(super) async fn send_hello_reply( }; debug!(to = %peer, ready, "Sending HELLO_REPLY"); + say_what_we_speak(state, peer).await; state .transport diff --git a/src/swarm/peer/inbound.rs b/src/swarm/peer/inbound.rs index d266e5f..e23ec14 100644 --- a/src/swarm/peer/inbound.rs +++ b/src/swarm/peer/inbound.rs @@ -39,6 +39,11 @@ pub(crate) async fn handle_message( note_alive(state, &peer, &message).await; return Ok(()); } + // SPEAKS only says what the peer's build speaks and knows (`swarm::speaking`). + if let ParsedMessage::Speaks(speaks) = message { + state.speaking.heard(&peer, speaks); + return Ok(()); + } // WHO and ME end at the identity layer (`transport::identity`). A sync message names // its epoch by a tag (`EpochTag`): ours, or the sender's as we know it; one naming // neither is of an epoch we don't sync on, and says nothing we could record. @@ -317,7 +322,8 @@ pub(crate) async fn handle_message( transport::ParsedMessage::Who(_) | transport::ParsedMessage::Me(_) | transport::ParsedMessage::Ping(_) - | transport::ParsedMessage::Pong { .. } => {} + | transport::ParsedMessage::Pong { .. } + | transport::ParsedMessage::Speaks(_) => {} } Ok(()) diff --git a/src/swarm/speaking.rs b/src/swarm/speaking.rs new file mode 100644 index 0000000..dcbfac4 --- /dev/null +++ b/src/swarm/speaking.rs @@ -0,0 +1,46 @@ +//! What the peers' builds speak and know, as each says with its handshakes (SPEAKS, +//! `transport::Speaks`): the wire versions it speaks and the newest revision of the +//! federation it knows. A peer that sends none runs a build from before SPEAKS, or isn't +//! heard from yet. + +use std::collections::BTreeMap; + +use crate::locks::MutexExt; +use crate::transport::{Speaks, wire}; + +/// How many peers' SPEAKS are kept: anyone can send one. +const MOST: usize = 4096; + +/// What this build speaks and knows. +pub(crate) fn own() -> Speaks { + Speaks { + oldest: wire::OLDEST, + newest: wire::VERSION, + revision: newest_revision(), + } +} + +#[cfg(feature = "federation")] +fn newest_revision() -> Option<[u8; 32]> { + Some(crate::federation::revision::Revision::newest().id().0) +} + +#[cfg(not(feature = "federation"))] +fn newest_revision() -> Option<[u8; 32]> { + None +} + +/// The SPEAKS each peer sent last, by node ID. +#[derive(Default)] +pub(crate) struct Speaking(std::sync::Mutex>); + +impl Speaking { + /// `peer` said `speaks`. + pub(crate) fn heard(&self, peer: &str, speaks: Speaks) { + let mut heard = self.0.lock_or_recover(); + if heard.len() >= MOST && !heard.contains_key(peer) { + heard.clear(); + } + heard.insert(peer.to_owned(), speaks); + } +} diff --git a/src/transport/mod.rs b/src/transport/mod.rs index ad45ce9..7bc6ae0 100644 --- a/src/transport/mod.rs +++ b/src/transport/mod.rs @@ -254,6 +254,20 @@ pub(crate) enum ParsedMessage { Disconnect(NeighbourAnswer), /// Changes the sender refused, of those the recipient brought (`swarm::refusals`). Refusals(RefusalsMessage), + /// What the sender's build speaks and knows, in a layout no build changes + /// (`wire::SPEAKS`, `swarm::speaking`). + Speaks(Speaks), +} + +/// SPEAKS: the wire versions the sender speaks, the oldest and the newest, and the +/// newest revision of the federation it knows, by ID, where it is built with the +/// federation. Sent with every handshake, in a layout no build changes (`wire::SPEAKS`): +/// a build that doesn't know it ignores it, as a message of another wire version. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Speaks { + pub(crate) oldest: u8, + pub(crate) newest: u8, + pub(crate) revision: Option<[u8; 32]>, } /// REFUSALS: changes the sender refused by the swarm's rules (`access::screen`), of @@ -417,6 +431,7 @@ pub(crate) enum MessageKind { NeighbourReply, Disconnect, Refusals, + Speaks, } impl ParsedMessage { @@ -440,6 +455,7 @@ impl ParsedMessage { ParsedMessage::NeighbourReply(_) => MessageKind::NeighbourReply, ParsedMessage::Disconnect(_) => MessageKind::Disconnect, ParsedMessage::Refusals(_) => MessageKind::Refusals, + ParsedMessage::Speaks(_) => MessageKind::Speaks, } } @@ -469,7 +485,8 @@ impl ParsedMessage { ParsedMessage::Who(_) | ParsedMessage::Me(_) | ParsedMessage::Ping(_) - | ParsedMessage::Pong { .. } => None, + | ParsedMessage::Pong { .. } + | ParsedMessage::Speaks(_) => None, } } } diff --git a/src/transport/traffic.rs b/src/transport/traffic.rs index 5ff588e..89217c3 100644 --- a/src/transport/traffic.rs +++ b/src/transport/traffic.rs @@ -40,6 +40,7 @@ impl Counter { #[derive(Debug, Default, Clone, Copy, Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct ByKind { + /// HELLO and SPEAKS, which goes with each handshake. pub(crate) hello: Counter, pub(crate) hello_reply: Counter, pub(crate) sync: Counter, @@ -84,7 +85,7 @@ impl ByKind { fn get(&self, kind: MessageKind) -> &Counter { match kind { - MessageKind::Hello => &self.hello, + MessageKind::Hello | MessageKind::Speaks => &self.hello, MessageKind::HelloReply => &self.hello_reply, MessageKind::Sync => &self.sync, MessageKind::Status => &self.status, @@ -103,7 +104,7 @@ impl ByKind { fn get_mut(&mut self, kind: MessageKind) -> &mut Counter { match kind { - MessageKind::Hello => &mut self.hello, + MessageKind::Hello | MessageKind::Speaks => &mut self.hello, MessageKind::HelloReply => &mut self.hello_reply, MessageKind::Sync => &mut self.sync, MessageKind::Status => &mut self.status, @@ -366,6 +367,7 @@ fn names(kind: MessageKind) -> (&'static str, &'static str) { MessageKind::NeighbourReply => ("NEIGHBOUR_REPLY", "hyparview"), MessageKind::Disconnect => ("DISCONNECT", "hyparview"), MessageKind::Refusals => ("REFUSALS", "refusals"), + MessageKind::Speaks => ("SPEAKS", "hello"), } } diff --git a/src/transport/wire.rs b/src/transport/wire.rs index 0270e6c..abaa408 100644 --- a/src/transport/wire.rs +++ b/src/transport/wire.rs @@ -7,6 +7,11 @@ //! with older nodes to keep. But every change bumps `VERSION`, so nodes on different //! builds tell each other apart instead of misreading each other's messages. The //! transport's topic says which application a message is for, not which version. +//! +//! One message stands outside the versions: SPEAKS (`SPEAKS`), its first byte the +//! version none is, laid out as no build changes, saying which versions a node speaks +//! and the newest revision of the federation it knows. A build from before it reads it +//! as a message of another version, and ignores it. use thiserror::Error; @@ -16,6 +21,19 @@ use super::{EncodedMessage, MessageKind, ParsedMessage}; /// to a message type or a codec (`golden_bytes` fails then). pub(crate) const VERSION: u8 = 52; +/// The oldest version this build speaks: the messages of every version from it to +/// `VERSION` it reads. Its SPEAKS says so. +pub(crate) const OLDEST: u8 = VERSION; + +/// The first byte of a SPEAKS, which no version is. Its layout, never to change: that +/// byte, the oldest version the sender speaks, the newest, then the 32 bytes of the ID +/// of the newest revision of the federation it knows, where it knows one. A reader takes +/// those and leaves whatever follows, which a later build may add. +pub(crate) const SPEAKS: u8 = 0; + +// Whatever this build's versions, a SPEAKS reads as one. +const _: () = assert!(SPEAKS < OLDEST && OLDEST <= VERSION); + /// What the second byte of a payload says. fn tag(kind: MessageKind) -> u8 { match kind { @@ -37,6 +55,8 @@ fn tag(kind: MessageKind) -> u8 { MessageKind::NeighbourReply => 17, MessageKind::Disconnect => 18, MessageKind::Refusals => 20, + // None: a SPEAKS has no second byte (`SPEAKS`). + MessageKind::Speaks => u8::MAX, } } @@ -75,6 +95,11 @@ pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result Result, WireError> { + if let ParsedMessage::Speaks(speaks) = message { + let mut payload = vec![SPEAKS, speaks.oldest, speaks.newest]; + payload.extend(speaks.revision.iter().flatten()); + return Ok(payload); + } let mut payload = vec![VERSION, tag(message.kind())]; match message { ParsedMessage::Hello(hello) => rmp_serde::encode::write(&mut payload, hello)?, @@ -99,6 +124,7 @@ pub(crate) fn encode(message: &ParsedMessage) -> Result, WireError> { rmp_serde::encode::write(&mut payload, answer)? } ParsedMessage::Refusals(message) => rmp_serde::encode::write(&mut payload, message)?, + ParsedMessage::Speaks(_) => unreachable!("encoded above"), } Ok(payload) } @@ -138,7 +164,17 @@ impl OtherVersions { pub(crate) fn decode(payload: &[u8]) -> Result { let (&version, payload) = payload.split_first().ok_or(WireError::Empty)?; - if version != VERSION { + if version == SPEAKS { + let [oldest, newest, rest @ ..] = payload else { + return Err(WireError::Empty); + }; + return Ok(ParsedMessage::Speaks(super::Speaks { + oldest: *oldest, + newest: *newest, + revision: rest.get(..32).map(|id| id.try_into().expect("32 bytes")), + })); + } + if !(OLDEST..=VERSION).contains(&version) { return Err(WireError::Version(version)); } let (&kind, body) = payload.split_first().ok_or(WireError::Empty)?; @@ -729,6 +765,39 @@ mod tests { crate::transport::Hex(bytes).to_string() } + /// A SPEAKS is laid out as no build changes: its version byte, the oldest and newest + /// version, the revision's ID where there is one; what follows is left. + #[test] + fn speaks_keeps_its_layout() { + let speaks = super::super::Speaks { + oldest: 50, + newest: 52, + revision: Some([7; 32]), + }; + let bytes = encode(&ParsedMessage::Speaks(speaks)).unwrap(); + assert_eq!(hex(&bytes), format!("003234{}", "07".repeat(32))); + let without = super::super::Speaks { + revision: None, + ..speaks + }; + assert_eq!( + hex(&encode(&ParsedMessage::Speaks(without)).unwrap()), + "003234" + ); + for (payload, read) in [ + (bytes.clone(), speaks), + ([bytes.as_slice(), b"later"].concat(), speaks), + (vec![0, 50, 52], without), + (vec![0, 50, 52, 1, 2], without), + ] { + match decode(&payload) { + Ok(ParsedMessage::Speaks(decoded)) => assert_eq!(decoded, read), + other => panic!("{other:?}"), + } + } + assert!(decode(&[0, 50]).is_err()); + } + /// The exact bytes of a few messages. If this fails after a change to a message type /// or a codec, the change is a new wire format: bump `VERSION` and take the new bytes. #[test] -- cgit v1.3.1