From 59e889254b1fda2dcc8e3e6d596635d1b77f589f Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 19 Nov 2026 17:44:03 +0000 Subject: transport: wire versions spoken side by side A build speaks wire versions OLDEST to VERSION and writes each node the newest both speak (wire::choose), as the node's SPEAKS says; the oldest until it says. The identity layer keeps that per address (Book::speaks): a SPEAKS from an address sets it for the address, and for every address of the node proven there, and a new address of a node takes it over. Paths encodes each message for an address in its version (Transport::encode_in; Transport::encode is the newest). So that two nodes know what the other speaks once they have proven themselves to each other, before any other message goes between them, a SPEAKS goes ahead of every WHO and ME, and WHO and ME are read in every version from 52 on (wire::PROOFS): their tags and layout never change. Without that, a node whose oldest version is past the other's couldn't read its proofs, and the two would never get to the handshake that says what each speaks. A SPEAKS naming no version this build speaks is logged as a message of another version is, naming the node to upgrade, the versions as ranges. A 0.0.1 node takes a SPEAKS from an address not yet proven for any other message from there: it drops it, and asks WHO. This build speaks 52 alone, so nothing it sends changes but the SPEAKS ahead of the proofs. A release adding a version encodes and decodes the older ones from their own, frozen, types. --- CHANGELOG.md | 8 ++ docs/architecture.md | 17 +-- docs/guides/federation/upgrades.md | 9 +- docs/operations.md | 35 ++++--- docs/peers.md | 10 +- src/simtest/timing.rs | 8 ++ src/transport/identity.rs | 154 +++++++++++++++++++++++++-- src/transport/libp2p/mod.rs | 9 +- src/transport/mod.rs | 18 +++- src/transport/mycelium/mod.rs | 26 +++-- src/transport/paths.rs | 49 +++++++-- src/transport/sim.rs | 17 +-- src/transport/tor.rs | 9 +- src/transport/traffic.rs | 9 ++ src/transport/wire.rs | 206 ++++++++++++++++++++++++++++++------- 15 files changed, 481 insertions(+), 103 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d531c8..7754a3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ it from the release before ([`docs/operations.md`](docs/operations.md), ## Unreleased +* A node speaks several wire versions side by side and writes each peer the newest both + speak, as the peer's SPEAKS says; SPEAKS goes ahead of each WHO and ME now, and WHO + and ME read alike in every version. So a release adding a wire version and keeping + the older rolls out node by node ([`docs/operations.md`](docs/operations.md), + [Upgrading](docs/operations.md#upgrading); + [`src/transport/wire.rs`](src/transport/wire.rs), + [`src/transport/identity.rs`](src/transport/identity.rs)). + * `scripts/release` makes a release: it checks the tree, says from the last release's wire versions, Automerge and revisions whether it rolls out node by node, and commits and tags it, pushing nothing ([`docs/development.md`](docs/development.md), diff --git a/docs/architecture.md b/docs/architecture.md index f70f132..f462fe7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -79,9 +79,9 @@ it switched to the right document without trusting anyone. **[`transport/`](../src/transport/)** — [`mod.rs`](../src/transport/mod.rs) defines the message types (HELLO, HELLO_REPLY, sync envelope, STATUS/STATUS_REPLY, PING/PONG, blob messages, PROBE/PROBE_REPLY, REFUSALS, SPEAKS) and the `Transport` trait; -[`wire.rs`](../src/transport/wire.rs) is the wire format (a version byte, bumped with every change to the format, -and a kind byte, then the message in MessagePack, derived from the types in [`mod.rs`](../src/transport/mod.rs) -with codecs for hashes and IDs; SPEAKS, outside the versions, is laid out as no build changes); [`mycelium/`](../src/transport/mycelium/) (with the `mycelium` feature) is the REST +[`wire.rs`](../src/transport/wire.rs) is the wire format (a version byte, a new version with every change to the format, +several spoken side by side, and a kind byte, then the message in MessagePack, derived from the types in [`mod.rs`](../src/transport/mod.rs) +with codecs for hashes and IDs; SPEAKS, WHO and ME, outside the versions, are laid out as no build changes); [`mycelium/`](../src/transport/mycelium/) (with the `mycelium` feature) is the REST client (polling inbox, outbound POST, one topic, `continuum`, for every message), with [`mycelium/daemon.rs`](../src/transport/mycelium/daemon.rs) running the daemon (an external one by its URL, or one it starts and supervises); [`libp2p/`](../src/transport/libp2p/) (with the `libp2p` feature) is libp2p as a transport, @@ -288,11 +288,12 @@ places that rely on it. * **Conflicts are errors; overrides are named.** A contradiction in the configuration or at the API refuses with a message that names the `force` value that would override it, in one camelCase vocabulary everywhere. -* **Builds tell each other apart.** Every change of the wire format bumps its version, - and a node ignores messages of another version instead of misreading them: across a - change, the swarm is split until every node runs it, and merges then. A release's - version says whether it is such a change ([`operations.md`](operations.md), - [Upgrading](operations.md#upgrading)). +* **Builds tell each other apart.** Every change of the wire format is a new version; a + build speaks several side by side and writes each node the newest both speak, and a + node ignores messages of a version it doesn't speak instead of misreading them. Where + two releases share none, the swarm is split until every node runs the later, and + merges then. A release's version says whether it is such a change + ([`operations.md`](operations.md), [Upgrading](operations.md#upgrading)). ## Where the complexity sits diff --git a/docs/guides/federation/upgrades.md b/docs/guides/federation/upgrades.md index 6ffde67..9025164 100644 --- a/docs/guides/federation/upgrades.md +++ b/docs/guides/federation/upgrades.md @@ -10,10 +10,11 @@ design's ([`federation.md`](../../federation.md), A node of a federated swarm upgrades as any node does ([`operations.md`](../../operations.md), [Upgrading](../../operations.md#upgrading)), restarted with its files, and a release's version says how: a patch or a minor release one -node at a time, a major one every node close together. Across a change of the wire -format, a major one, upgraded nodes and the others don't talk, and the swarm is split -until all are upgraded; a remake needs the same Automerge on every node, and one with -another rejoins instead of switching. Before 1.0.0, take every upgrade as a major one. +node at a time, a major one every node close together. Nodes write each other the newest +wire version both speak; where two releases share none, a major one, upgraded nodes and +the others don't talk, and the swarm is split until all are upgraded. A remake needs the +same Automerge on every node, and one with another rejoins instead of switching. Before +1.0.0, a release's tag says whether it is a flag day. Nobody runs every node of a federated swarm: each party's operators upgrade its nodes. A patch or minor release each party takes when it likes; for a major one, agree on when, diff --git a/docs/operations.md b/docs/operations.md index 0f906eb..4f3d6f2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -125,20 +125,27 @@ continuum --version A node upgrades by restarting with its files. What to know: -* **The wire format.** Each change to it bumps a version byte, and nodes ignore - messages of another version. Across such a change, upgraded nodes and the others - don't talk: the swarm is split until every node is upgraded, both sides go on - writing, and they merge then. Each logs it, at warn, once per peer, naming both - versions and the node to upgrade (`Peer speaks another wire version: a newer build - (wire version 53; this one speaks 52): upgrade this node to talk with it; ignoring its - messages`, or of an older build, `upgrade that node`). With each handshake, a - node says which versions it speaks, and the newest revision of the federation it knows - (SPEAKS, outside the versions; [`peers.md`](peers.md), - [Handshake and join](peers.md#handshake-and-join)); a build from before SPEAKS, before - 0.0.1, logs it as another wire version, `version=0`, once of the peer, and reads the - peer's other messages where they are of its version. - A node speaks one version, so every change of the format is a major release; before - 1.0.0, take every upgrade as one, and upgrade all nodes close together. +* **The wire format.** Each change to it is a new version (the first byte of every + message), and a build speaks several side by side, from the oldest to the newest + `--version` names: it reads each, and writes to each node the newest both speak. A + node says which versions it speaks, and the newest revision of the federation it knows, + ahead of each proof of who it is and with each handshake (SPEAKS, outside the versions, + as are the proofs; [`peers.md`](peers.md), + [Handshake and join](peers.md#handshake-and-join)); it writes a node the oldest it + speaks until that node says. So a release whose versions meet the last one's rolls out + node by node, in any order: a release adds a version and keeps the older ones, and a + later one leaves the oldest out. A release's tag says whether it rolls out so + ([`development.md`](development.md), [Releasing](development.md#releasing)). + Nodes sharing no version don't talk: the swarm is split until every node is upgraded, + both sides go on writing, and they merge then. Each logs it, at warn, once per peer, + naming both versions and the node to upgrade (`Peer speaks another wire version: a + newer build (wire version 54; this one speaks 52): upgrade this node to talk with it; + ignoring its messages`, or of an older build, `upgrade that node`). 0.0.1 logs `Peer + speaks another wire version (another build); ignoring its messages`, with the peer's + version and its own (`ours`): upgrade the node whose version is the lower. A + build from before SPEAKS, before 0.0.1, logs SPEAKS as another wire version, + `version=0`, once of the peer, and reads the peer's other messages where they are of + its version. * **Automerge.** A remake needs every node to build the same snapshot, byte for byte, so the same Automerge version everywhere; a node with another one rejoins instead of switching ([`remakes.md`](remakes.md), [Limits](remakes.md#limits)). Don't remake while diff --git a/docs/peers.md b/docs/peers.md index 34023db..a03d15f 100644 --- a/docs/peers.md +++ b/docs/peers.md @@ -44,6 +44,12 @@ there are that node's; messages from an address no node proved itself at are dro and the address is asked WHO (at most every 5 s), so after a restart the first message from a peer brings its proof back. +Ahead of each WHO and each ME goes a SPEAKS ([Handshake and join](#handshake-and-join)), +and WHO and ME read alike in every wire version: so once two nodes proved themselves to +each other, each knows which versions the other speaks, and writes it the newest both +do, at each of its addresses ([`operations.md`](operations.md), +[Upgrading](operations.md#upgrading)). Until a node says, it is written the oldest. + A seed is an address (or something the transport resolves to one: a Mycelium public key, IPv6 address or host name); it is asked WHO, and its ME names the node. Where other nodes are comes from HELLO_REPLYs and the member list: hints, which are asked WHO before @@ -98,8 +104,8 @@ HELLO. Before each HELLO and each HELLO_REPLY, a node sends SPEAKS: the wire versions it speaks and the newest revision of the federation it knows, as its build has them. It stands outside the wire versions, laid out as no build changes, so that a peer on another build -reads it too, once that peer reads the node's WHO and ME -([Node IDs and addresses](#node-ids-and-addresses)): what each peer said shows as its +reads it too; it goes ahead of each WHO and ME as well +([Node IDs and addresses](#node-ids-and-addresses)). What each peer said shows as its `knows` in `GET /peers`, the node's own as `localKnows`, and in `GET /events/counts` for the nodes a federated store's parties list. A build from before SPEAKS takes it for a message of another wire version, and logs that once of the peer diff --git a/src/simtest/timing.rs b/src/simtest/timing.rs index ba85bb9..5f23c21 100644 --- a/src/simtest/timing.rs +++ b/src/simtest/timing.rs @@ -460,6 +460,14 @@ impl crate::transport::Transport for OutboundOnly { async fn resolve_seed(&self, seed: &str) -> anyhow::Result { self.0.resolve_seed(seed).await } + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> anyhow::Result { + self.0.encode_in(peer, message, version) + } fn encode( &self, peer: &str, diff --git a/src/transport/identity.rs b/src/transport/identity.rs index 5d3e2f5..29624db 100644 --- a/src/transport/identity.rs +++ b/src/transport/identity.rs @@ -26,6 +26,12 @@ //! //! Addresses and the transports behind them are the paths layer's //! (`transport::paths`): the books keep addresses as paths keep them. +//! +//! Each address is written the newest wire version its node and this build both speak +//! (`transport::wire`), as a SPEAKS from there, or from another address of its node, +//! said; the oldest this build speaks until one does. A SPEAKS goes ahead of every WHO +//! and ME, and WHO and ME read in every version, so two nodes know what the other speaks +//! once they proved themselves to each other, before anything else goes between them. use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex, MutexGuard}; @@ -40,12 +46,12 @@ use tracing::{debug, info}; use crate::identity::{self, NodeKey}; use crate::locks::MutexExt; use crate::transport::firewall::Firewall; -use crate::transport::paths::{Gate, PathHandler, Paths, scheme_of}; +use crate::transport::paths::{Gate, PathHandler, Paths, Versions, scheme_of}; use crate::transport::stats::{IdentityCounts, IdentityStats, TransportStats}; use crate::transport::traffic::Traffic; use crate::transport::{ Connection, Destination, EncodedMessage, HandlerFn, IdentityProof, MessageHandler, MessageKind, - ParsedMessage, Transport, + ParsedMessage, Transport, wire, }; pub(crate) const WHO_LABEL: &[u8] = b"continuum/who/1"; @@ -116,6 +122,8 @@ struct Shared { /// addresses they proved themselves at, their messages may be large /// (`transport::Admission`). admitted: Mutex>, + /// Addresses whose SPEAKS named no wire version we speak, logged once each. + other_versions: wire::OtherVersions, } /// Who is where, by address. @@ -132,6 +140,9 @@ struct Book { asked: HashMap, /// Seeds being resolved: who waits for a ME from where. waiting: HashMap>>, + /// The wire version to write at an address, where a SPEAKS from there or from its + /// node said what it speaks (`Book::speaks`). + spoken: HashMap, /// Since start (`stats`). counts: IdentityCounts, } @@ -148,11 +159,18 @@ impl Book { if *slot != address { let previous = std::mem::replace(slot, address.to_owned()); self.at.remove(&previous); + self.spoken.remove(&previous); self.counts.address_changes += 1; } } else { addresses.push(address.to_owned()); } + // What the node speaks at its other addresses, it speaks here: one build. + if !self.spoken.contains_key(address) + && let Some(version) = addresses.iter().find_map(|at| self.spoken.get(at).copied()) + { + self.spoken.insert(address.to_owned(), version); + } if let Some((previous, _)) = self.at.insert(address.to_owned(), (node.to_owned(), now)) && previous != node { @@ -166,6 +184,7 @@ impl Book { .map(|(address, _)| address.clone()) && let Some((node, _)) = self.at.remove(&oldest) { + self.spoken.remove(&oldest); self.forget_proof(&node, &oldest); } for waiter in self.waiting.remove(address).into_iter().flatten() { @@ -174,6 +193,39 @@ impl Book { news } + /// A SPEAKS from `address` said its build speaks wire versions `oldest` to `newest`: + /// it, and every address its node proved itself at, are written the newest both + /// speak (`wire::choose`), or the oldest this build speaks where there is none (and + /// the node says it can't read us). Before a proof, for `address` alone: a SPEAKS + /// goes ahead of each WHO and ME. Bounded as the books are: past `MAX_ENTRIES`, + /// addresses no node proved itself at are forgotten first. + fn speaks(&mut self, address: &str, oldest: u8, newest: u8) { + let version = wire::choose(oldest, newest); + let addresses = match self.at.get(address) { + Some((node, _)) => self.proven.get(node).cloned().unwrap_or_default(), + None => vec![address.to_owned()], + }; + for address in addresses { + let Some(version) = version else { + self.spoken.remove(&address); + continue; + }; + if self.spoken.len() >= MAX_ENTRIES && !self.spoken.contains_key(&address) { + let at = &self.at; + self.spoken.retain(|address, _| at.contains_key(address)); + if self.spoken.len() >= MAX_ENTRIES { + continue; + } + } + self.spoken.insert(address, version); + } + } + + /// The wire version to write at `address` (`speaks`). + fn version_at(&self, address: &str) -> u8 { + self.spoken.get(address).copied().unwrap_or(wire::OLDEST) + } + /// The node proven at `address`, which was heard from there `now`. fn heard(&mut self, address: &str, now: Instant) -> Option { self.at.get_mut(address).map(|(node, heard)| { @@ -293,6 +345,7 @@ impl IdentityTransport { let node_id = config.key.id(); let shared = Arc::new_cyclic(|shared: &std::sync::Weak| { let gate: std::sync::Weak = shared.clone(); + let versions: std::sync::Weak = shared.clone(); let admitting = shared.clone(); let admission = crate::transport::Admission::new(move |address| { admitting @@ -310,6 +363,7 @@ impl IdentityTransport { .collect(), Some(Arc::clone(&traffic)), firewall.is_some().then_some(gate), + Some(versions), clock, keepalive, )), @@ -319,6 +373,7 @@ impl IdentityTransport { traffic, book: Mutex::default(), admitted: Mutex::default(), + other_versions: wire::OtherVersions::default(), } }); // The node's messages are noted here, with the address each went to or came @@ -340,6 +395,13 @@ impl Gate for Shared { } } +/// Each address is written what its node speaks (`Book::speaks`). +impl Versions for Shared { + fn version_at(&self, address: &str) -> u8 { + self.lock().version_at(address) + } +} + impl Shared { /// Whether the node proven at `address` is admitted (`Transport::admit`). fn admits(&self, address: &str) -> bool { @@ -442,7 +504,9 @@ impl Shared { tokio::spawn(async move { shared.send_proof(MessageKind::Who, &address).await }); } - /// Sends a WHO or ME (`kind`) to `address`. + /// Sends a WHO or ME (`kind`) to `address`, a SPEAKS ahead of it: so the node there + /// knows what this build speaks once it knows who we are, and the first messages it + /// writes us are in a version we read. async fn send_proof(&self, kind: MessageKind, address: &str) { let Some(address) = self.paths.canonical(address) else { return; @@ -456,10 +520,21 @@ impl Shared { let Some(message) = message else { return; }; - let encoded = match self.paths.encode(address, &message) { + self.send_to( + address, + &ParsedMessage::Speaks(crate::swarm::speaking::own()), + ) + .await; + self.send_to(address, &message).await; + } + + /// Sends `message`, which ends at the identity layer, to `address`, counted under the + /// address (`traffic`). + async fn send_to(&self, address: &str, message: &ParsedMessage) { + let encoded = match self.paths.encode(address, message) { Ok(encoded) => encoded, Err(error) => { - debug!(to = %address, error = %format!("{error:#}"), "Couldn't encode an identity proof"); + debug!(to = %address, kind = ?message.kind(), error = %format!("{error:#}"), "Couldn't encode a message of the identity layer"); return; } }; @@ -470,7 +545,7 @@ impl Shared { Err(error) => { self.traffic .refused(address, encoded.kind, encoded.payload_len); - debug!(to = %address, error = %format!("{error:#}"), "Couldn't send an identity proof"); + debug!(to = %address, kind = ?message.kind(), error = %format!("{error:#}"), "Couldn't send a message of the identity layer"); } } } @@ -499,6 +574,29 @@ impl Shared { self.prove(&address, ME_LABEL, &proof); Ok(()) } + // What the address's build speaks, noted first (`Book::speaks`), and logged + // where we share no version; from a node proven there, it goes on up too + // (`swarm::speaking`). From an address no node proved itself at, it ends here: + // the WHO or ME it goes ahead of follows. + ParsedMessage::Speaks(speaks) => { + self.other_versions + .speaks(&address, speaks.oldest, speaks.newest); + let node = { + let mut book = self.lock(); + book.speaks(&address, speaks.oldest, speaks.newest); + book.heard(&address, Instant::now()) + }; + let Some(node) = node else { + self.traffic + .received(&address, MessageKind::Speaks, payload_len); + return Ok(()); + }; + let message = ParsedMessage::Speaks(speaks); + self.traffic + .log + .note("received", &node, Some(&address), &message, payload_len); + handler(node, message, payload_len).await + } message => { // Heard from: a node that talks isn't the one evicted when strangers' // proofs fill the book (`Book::bind`). @@ -774,7 +872,13 @@ impl Transport for IdentityTransport { /// For the address `destination` picks (a send there is refused until `peer` /// proved itself there, `send_encoded`). Only a look at the choice: the send makes /// it (encoding again for another path if it changes). - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result { + /// In the wire version the address speaks, whatever `_version` asks (`Paths::encode`). + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + _version: u8, + ) -> Result { let Some(address) = self.shared.destination(peer, false) else { bail!("no address known for {peer} on this node's transports"); }; @@ -855,6 +959,42 @@ mod tests { assert_eq!(book.at.len(), MAX_ENTRIES); } + /// An address is written what a SPEAKS from there said, and so is every address of + /// the node proven there; the oldest version this build speaks until one says. + #[test] + fn each_address_is_written_what_its_node_speaks() { + let now = Instant::now(); + let (oldest, newest) = (wire::OLDEST, wire::VERSION); + let mut book = Book::default(); + assert_eq!(book.version_at("s:x"), oldest); + // Before a proof, for the address alone. + book.speaks("s:x", oldest, newest + 3); + assert_eq!(book.version_at("s:x"), newest); + book.bind("s:x", "n1", now); + // Another address of the node is written the same: one build. + book.bind("t:x", "n1", now); + assert_eq!(book.version_at("t:x"), newest); + // Moved: the old address is forgotten, the new one written the same. + book.bind("t:y", "n1", now); + assert!(!book.spoken.contains_key("t:x")); + assert_eq!(book.version_at("t:y"), newest); + // Speaking none of this build's versions: the oldest, at each of its addresses. + book.speaks("s:x", newest + 1, newest + 2); + assert!(book.spoken.is_empty()); + assert_eq!(book.version_at("t:y"), oldest); + + // Full: addresses no node proved itself at go first. + let mut book = Book::default(); + book.bind("s:member", "member", now); + for i in 0..MAX_ENTRIES { + book.speaks(&format!("s:{i}"), oldest, newest); + } + assert_eq!(book.spoken.len(), MAX_ENTRIES); + book.speaks("s:member", oldest, newest); + assert_eq!(book.spoken.len(), 1); + assert_eq!(book.version_at("s:member"), newest); + } + #[test] fn one_address_per_node_and_scheme_and_one_node_per_address() { let now = Instant::now(); diff --git a/src/transport/libp2p/mod.rs b/src/transport/libp2p/mod.rs index 2290dea..24f5f34 100644 --- a/src/transport/libp2p/mod.rs +++ b/src/transport/libp2p/mod.rs @@ -572,8 +572,13 @@ impl Transport for Libp2pTransport { } } - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result { - wire::encoded(peer, message) + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result { + wire::encoded_in(peer, message, version) } async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { diff --git a/src/transport/mod.rs b/src/transport/mod.rs index 7bc6ae0..ee63988 100644 --- a/src/transport/mod.rs +++ b/src/transport/mod.rs @@ -261,8 +261,9 @@ pub(crate) enum ParsedMessage { /// SPEAKS: the wire versions the sender speaks, the oldest and the newest, and the /// newest revision of the federation it knows, by ID, where it is built with the -/// federation. Sent with every handshake, in a layout no build changes (`wire::SPEAKS`): -/// a build that doesn't know it ignores it, as a message of another wire version. +/// federation. Sent with every handshake, and ahead of every WHO and ME +/// (`transport::identity`), in a layout no build changes (`wire::SPEAKS`): a build that +/// doesn't know it ignores it, as a message of another wire version. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct Speaks { pub(crate) oldest: u8, @@ -1198,7 +1199,18 @@ pub(crate) trait Transport: Send + Sync { /// identity, as `local_id` gives it on that node. async fn resolve_seed(&self, seed: &str) -> Result; - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result; + /// `message` to `peer` in wire `version`, one this build speaks (`wire::OLDEST..= + /// wire::VERSION`). The identity layer writes to each node in the newest version both + /// speak, whatever is asked (`Paths::version_at`). + fn encode_in(&self, peer: &str, message: &ParsedMessage, version: u8) + -> Result; + + /// `message` to `peer` in the newest wire version; under the identity layer, in the one + /// `peer` speaks. + fn encode(&self, peer: &str, message: &ParsedMessage) -> Result { + self.encode_in(peer, message, wire::VERSION) + } + async fn send_encoded(&self, message: &EncodedMessage) -> Result<()>; /// Sends an outbound message payload to a targeted peer on a specific topic. diff --git a/src/transport/mycelium/mod.rs b/src/transport/mycelium/mod.rs index f1dbe40..d5d9271 100644 --- a/src/transport/mycelium/mod.rs +++ b/src/transport/mycelium/mod.rs @@ -355,8 +355,13 @@ impl Transport for MyceliumTransport { self.lookup_pubkey(&ip).await } - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result { - encode_message(peer, message) + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result { + encode_message(peer, message, version) } async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { @@ -573,9 +578,13 @@ async fn read_capped( /// A message as the Mycelium REST API takes it (the JSON body of an outbound message). /// Also the simulator's wire format (`transport::sim`), so tests go through the same -/// encoding. -pub(crate) fn encode_message(peer: &str, message: &ParsedMessage) -> Result { - let (outbound, payload_len) = format_message(peer, message)?; +/// encoding. Its payload is in wire `version`. +pub(crate) fn encode_message( + peer: &str, + message: &ParsedMessage, + version: u8, +) -> Result { + let (outbound, payload_len) = format_message(peer, message, version)?; Ok(EncodedMessage { peer: peer.to_owned(), address: peer.to_owned(), @@ -645,12 +654,13 @@ fn parse_message(message: &InboundMessage) -> Result<(String, ParsedMessage), Tr Ok((sender, parsed)) } -/// The message for Mycelium's API, and the length of its payload. +/// The message for Mycelium's API, its payload in wire `version`, and the payload's length. fn format_message( recipient: &str, message: &ParsedMessage, + version: u8, ) -> Result<(OutboundMessage, usize), TransportError> { - let payload_bytes = wire::encode(message)?; + let payload_bytes = wire::encode_in(message, version)?; let outbound = OutboundMessage { dst: Destination { pk: recipient.to_owned(), @@ -797,7 +807,7 @@ mod tests { #[test] fn every_kind_round_trips_through_the_api_body() { for message in wire::samples() { - let encoded = encode_message("peer", &message).unwrap(); + let encoded = encode_message("peer", &message, wire::VERSION).unwrap(); assert_eq!(encoded.kind, message.kind()); let decoded = decode_message(&encoded.body).unwrap(); assert_eq!( diff --git a/src/transport/paths.rs b/src/transport/paths.rs index f425216..e2d9ec9 100644 --- a/src/transport/paths.rs +++ b/src/transport/paths.rs @@ -58,7 +58,7 @@ use crate::clock::{Clock, UnixMillis}; use crate::locks::MutexExt; use crate::transport::stats::{PathCounts, PathInfo, PathsStats, TransportStats}; use crate::transport::traffic::Traffic; -use crate::transport::{Connection, EncodedMessage, ParsedMessage, Transport}; +use crate::transport::{Connection, EncodedMessage, ParsedMessage, Transport, wire}; /// Whether to drop what comes from or goes to an address (the debug firewall, /// `transport::firewall`; the identity layer knows which node is where). @@ -67,6 +67,12 @@ pub(crate) trait Gate: Send + Sync { fn blocks_outbound(&self, address: &str) -> bool; } +/// The wire version to write at an address (`transport::wire`): the newest the node there +/// and this build both speak, as the identity layer heard from it. +pub(crate) trait Versions: Send + Sync { + fn version_at(&self, address: &str) -> u8; +} + /// Called with the sender's address, the message and its `payload_len`. pub(crate) type PathHandler = dyn Fn(String, ParsedMessage, usize) -> std::pin::Pin> + Send>> + Send @@ -547,6 +553,8 @@ pub(crate) struct Paths { traffic: Option>, /// The debug firewall, if any. gate: Option>, + /// What each address speaks; without, every address is written `wire::OLDEST`. + versions: Option>, /// The node's wall clock, which its PONGs say the time by. clock: Arc, /// A std mutex: never held across an await. @@ -555,12 +563,14 @@ pub(crate) struct Paths { impl Paths { /// Over `transports`, by scheme, in order of preference; its own PINGs and PONGs - /// counted in `traffic`, the firewall through `gate`, PONGs stamped by `clock`; - /// paths pinged every `keepalive` at least (`Table::probe_every`). + /// counted in `traffic`, the firewall through `gate`, each address written the wire + /// version `versions` says, PONGs stamped by `clock`; paths pinged every `keepalive` + /// at least (`Table::probe_every`). pub(crate) fn new( transports: Vec<(&'static str, Arc)>, traffic: Option>, gate: Option>, + versions: Option>, clock: Arc, keepalive: Duration, ) -> Self { @@ -571,6 +581,7 @@ impl Paths { .collect(), traffic, gate, + versions, clock, table: Mutex::new(Table::new(keepalive)), } @@ -580,6 +591,15 @@ impl Paths { self.gate.as_ref()?.upgrade() } + /// The wire version to write at `address` (`Versions`): the oldest this build speaks, + /// until its node says what it speaks. + pub(crate) fn version_at(&self, address: &str) -> u8 { + self.versions + .as_ref() + .and_then(Weak::upgrade) + .map_or(wire::OLDEST, |versions| versions.version_at(address)) + } + /// The transport `address` is on, and the transport identity at it /// (`Transport::locate`, which notes where that is if the address says); None if no /// transport of ours has its scheme, or it isn't one of that transport's. @@ -825,13 +845,16 @@ impl Paths { Err(anyhow!("{}", failures.join("; "))) } - /// `message` encoded for `address`: `peer` and `address` of the result are - /// `address`, and it keeps the message (to encode it for another address). + /// `message` encoded for `address`, in the wire version it speaks (`version_at`): + /// `peer` and `address` of the result are `address`, and it keeps the message (to + /// encode it for another address). pub(crate) fn encode(&self, address: &str, message: &ParsedMessage) -> Result { let (link, target) = self .split(address) .with_context(|| format!("no transport for {address}"))?; - let mut encoded = link.inner.encode(&target, message)?; + let mut encoded = link + .inner + .encode_in(&target, message, self.version_at(address))?; encoded.peer = address.to_owned(); encoded.address = address.to_owned(); encoded.message = Some(Arc::new(message.clone())); @@ -1195,6 +1218,7 @@ mod tests { ], None, None, + None, Arc::default(), Duration::ZERO, )) @@ -1608,6 +1632,7 @@ mod tests { vec![("sim", net.transport("b") as Arc)], None, Some(Arc::downgrade(&gate)), + None, Arc::default(), Duration::ZERO, )); @@ -1615,6 +1640,7 @@ mod tests { vec![("sim", net.transport("a") as Arc)], None, None, + None, Arc::default(), Duration::ZERO, )); @@ -1673,8 +1699,13 @@ mod tests { async fn resolve_seed(&self, seed: &str) -> Result { Ok(seed.to_owned()) } - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result { - (crate::transport::sim::WIRE.encode)(peer, message) + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result { + (crate::transport::sim::WIRE.encode)(peer, message, version) } async fn send_encoded(&self, _message: &EncodedMessage) -> Result<()> { Err(anyhow!("broken")) @@ -1713,6 +1744,7 @@ mod tests { ], None, None, + None, Arc::default(), Duration::ZERO, ); @@ -1720,6 +1752,7 @@ mod tests { vec![("sim", net.transport("b") as Arc)], None, None, + None, Arc::default(), Duration::ZERO, )); diff --git a/src/transport/sim.rs b/src/transport/sim.rs index 40a1f7d..5d55e32 100644 --- a/src/transport/sim.rs +++ b/src/transport/sim.rs @@ -327,7 +327,7 @@ impl SimNet { /// net. For messages a real node wouldn't send at that moment, e.g. one sent long /// ago and delivered late. pub(crate) async fn inject(&self, from: &str, to: &str, message: &ParsedMessage) -> Result<()> { - let encoded = (self.codec.encode)(to, message)?; + let encoded = (self.codec.encode)(to, message, super::wire::VERSION)?; let (handled_tx, handled_rx) = oneshot::channel(); let delivery = Delivery { from: from.to_owned(), @@ -368,7 +368,7 @@ pub(crate) struct SimTransport { /// around it (Mycelium's: `mycelium::CODEC`), so a whole swarm runs on that envelope. #[derive(Clone, Copy)] pub(crate) struct Codec { - pub(crate) encode: fn(&str, &ParsedMessage) -> Result, + pub(crate) encode: fn(&str, &ParsedMessage, u8) -> Result, pub(crate) decode: fn(&[u8]) -> Result, } @@ -378,8 +378,8 @@ pub(crate) const WIRE: Codec = Codec { decode: wire_decode, }; -fn wire_encode(peer: &str, message: &ParsedMessage) -> Result { - super::wire::encoded(peer, message) +fn wire_encode(peer: &str, message: &ParsedMessage, version: u8) -> Result { + super::wire::encoded_in(peer, message, version) } fn wire_decode(body: &[u8]) -> Result { @@ -410,8 +410,13 @@ impl Transport for SimTransport { } } - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result { - (self.net.codec.encode)(peer, message) + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result { + (self.net.codec.encode)(peer, message, version) } /// Always accepted: the net loses messages silently, like Mycelium once it has diff --git a/src/transport/tor.rs b/src/transport/tor.rs index c194330..a98e1c4 100644 --- a/src/transport/tor.rs +++ b/src/transport/tor.rs @@ -658,8 +658,13 @@ impl Transport for TorTransport { Ok(location) } - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result { - wire::encoded(peer, message) + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result { + wire::encoded_in(peer, message, version) } async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { diff --git a/src/transport/traffic.rs b/src/transport/traffic.rs index 89217c3..33903ca 100644 --- a/src/transport/traffic.rs +++ b/src/transport/traffic.rs @@ -537,6 +537,15 @@ impl Transport for CountingTransport { self.inner.resolve_seed(seed).await } + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result { + self.inner.encode_in(peer, message, version) + } + fn encode(&self, peer: &str, message: &ParsedMessage) -> Result { self.inner.encode(peer, message) } diff --git a/src/transport/wire.rs b/src/transport/wire.rs index 0b661dd..56883e8 100644 --- a/src/transport/wire.rs +++ b/src/transport/wire.rs @@ -3,15 +3,26 @@ //! arrays, fields in declaration order). The codecs below make hashes, IDs and bytes //! compact (MessagePack binaries, not arrays of numbers) and check their lengths. //! -//! The wire format may change freely until the first release: there is no compatibility -//! with older nodes to keep. But every change bumps `VERSION`, so nodes on different -//! builds tell each other apart instead of misreading each other's messages. The +//! Every change to a message type or a codec is a new version (`VERSION`), and a build +//! speaks several side by side, `OLDEST` to `VERSION`: it reads each, and writes to each +//! node the newest both speak (`choose`), as the node's SPEAKS says, or `OLDEST` until +//! then (`transport::identity`). So releases whose versions meet roll out node by node: +//! a release adds a version and keeps the older ones, and a later one leaves out the +//! oldest. The messages of an older version keep their own types, frozen, encoded and +//! decoded here beside the newest's (`golden_bytes` keeps the bytes of each). The //! transport's topic says which application a message is for, not which version. //! -//! One message stands outside the versions: SPEAKS (`SPEAKS`), its first byte the -//! version none is, laid out as no build changes, saying which versions a node speaks -//! and the newest revision of the federation it knows. A build from before it reads it -//! as a message of another version, and ignores it. +//! Two kinds of messages stand outside the versions, as they are what lets two builds +//! learn what the other speaks: +//! +//! * SPEAKS (`SPEAKS`), its first byte the version none is, laid out as no build changes, +//! saying which versions a node speaks and the newest revision of the federation it +//! knows. A build from before it reads it as a message of another version, and +//! ignores it. +//! * WHO and ME, the proofs of who is at an address (`PROOFS`), laid out in every +//! version as in the first: a build reads them whatever version they say they are in, +//! from that first on, so that two nodes prove themselves to each other before they +//! know what the other speaks. use thiserror::Error; @@ -31,8 +42,14 @@ pub(crate) const OLDEST: u8 = VERSION; /// those and leaves whatever follows, which a later build may add. pub(crate) const SPEAKS: u8 = 0; -// Whatever this build's versions, a SPEAKS reads as one. -const _: () = assert!(SPEAKS < OLDEST && OLDEST <= VERSION); +/// The first version with SPEAKS, as WHO and ME are laid out in every version: their +/// tags and fields never change (a change would be a new kind of message), and a build +/// reads them in any version from this one on, older or newer than its own. +pub(crate) const PROOFS: u8 = 52; + +// Whatever this build's versions, a SPEAKS reads as one, and it reads the proofs of +// every build that speaks one of its versions. +const _: () = assert!(SPEAKS < PROOFS && PROOFS <= OLDEST && OLDEST <= VERSION); /// What the second byte of a payload says. fn tag(kind: MessageKind) -> u8 { @@ -78,11 +95,15 @@ pub(crate) enum WireError { Decode(#[from] rmp_serde::decode::Error), } -/// `message` to `peer`, encoded as it goes on the wire alone (no envelope: libp2p, -/// Tor, the simulator); `peer` is its address too. +/// `message` to `peer`, encoded in wire `version` as it goes on the wire alone (no +/// envelope: libp2p, Tor, the simulator); `peer` is its address too. #[cfg_attr(not(any(feature = "libp2p", feature = "tor", test)), allow(dead_code))] -pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result { - let body = encode(message)?; +pub(crate) fn encoded_in( + peer: &str, + message: &ParsedMessage, + version: u8, +) -> anyhow::Result { + let body = encode_in(message, version)?; Ok(EncodedMessage { peer: peer.to_owned(), address: peer.to_owned(), @@ -93,7 +114,31 @@ pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result Option { + let (low, high) = (oldest.max(OLDEST), newest.min(VERSION)); + (low <= high).then_some(high) +} + +/// The payload of `message` in wire `version`, one of `OLDEST..=VERSION`: of an older one, +/// as a build speaking only that one reads it. A SPEAKS is laid out alike in every one, +/// and so are WHO and ME (`PROOFS`), but for the version they say. +pub(crate) fn encode_in(message: &ParsedMessage, version: u8) -> Result, WireError> { + if matches!(message, ParsedMessage::Who(_) | ParsedMessage::Me(_)) && version >= PROOFS { + let mut payload = encode(message)?; + payload[0] = version; + return Ok(payload); + } + // Each older version this build still speaks is encoded here, from its own types, + // once its messages differ from the newest's; this build's are the newest's alone. + if version != VERSION && !matches!(message, ParsedMessage::Speaks(_)) { + return Err(WireError::Version(version)); + } + encode(message) +} + +/// The payload of `message`, in the newest wire version. pub(crate) fn encode(message: &ParsedMessage) -> Result, WireError> { if let ParsedMessage::Speaks(speaks) = message { let mut payload = vec![SPEAKS, speaks.oldest, speaks.newest]; @@ -134,16 +179,33 @@ pub(crate) fn encode(message: &ParsedMessage) -> Result, WireError> { /// (`OtherVersions`). const MAX_OTHER_VERSIONS: usize = 1024; -/// Senders whose messages are of another wire version (`WireError::Version`): each -/// logged once, at warn level, so two builds that don't speak to each other say so, and -/// which of them to upgrade, instead of silently retrying their HELLOs for good. Bounded: -/// anyone can send us anything. +/// Senders whose messages are of another wire version (`WireError::Version`), or whose +/// SPEAKS names none this build speaks: each logged once, at warn level, so two builds +/// that don't speak to each other say so, and which of them to upgrade, instead of +/// silently retrying their HELLOs for good. Bounded: anyone can send us anything. #[derive(Default)] pub(crate) struct OtherVersions(std::sync::Mutex>); impl OtherVersions { /// A message from `sender` (on the transport `scheme`) was of `version`. pub(crate) fn note(&self, scheme: &'static str, sender: &str, version: u8) { + self.warn( + &super::Addressed(scheme, sender).to_string(), + version, + version, + ); + } + + /// A SPEAKS from `address` (as paths keep it) said its build speaks wire versions + /// `oldest` to `newest`: logged where this build speaks none of them, as a message of + /// another version is. The proofs between the two still go (`PROOFS`), nothing else. + pub(crate) fn speaks(&self, address: &str, oldest: u8, newest: u8) { + if choose(oldest, newest).is_none() { + self.warn(address, oldest, newest); + } + } + + fn warn(&self, from: &str, oldest: u8, newest: u8) { let mut known = self .0 .lock() @@ -151,36 +213,41 @@ impl OtherVersions { if known.len() >= MAX_OTHER_VERSIONS { known.clear(); } - if known.insert(sender.to_owned()) { + if known.insert(from.to_owned()) { tracing::warn!( - from = %super::Addressed(scheme, &sender), - version, + from = %from, + version = newest, oldest = OLDEST, newest = VERSION, "Peer speaks another wire version: {}; ignoring its messages", - other_build(version) + other_build(oldest, newest) ); } } } -/// Of a peer whose messages are of wire `version`, which this build doesn't speak, what -/// an operator does: upgrade this node where the peer's build is newer, the peer's where -/// it is older. -fn other_build(version: u8) -> String { - let ours = if OLDEST == VERSION { - VERSION.to_string() - } else { - format!("{OLDEST} to {VERSION}") +/// Of a peer speaking wire versions `oldest` to `newest`, none of which this build +/// speaks, what an operator does: upgrade this node where the peer's build is newer, the +/// peer's where it is older. +fn other_build(oldest: u8, newest: u8) -> String { + let versions = |oldest: u8, newest: u8| { + if oldest == newest { + format!("{oldest}") + } else { + format!("{oldest} to {newest}") + } }; - if version > VERSION { + let (theirs, ours) = (versions(oldest, newest), versions(OLDEST, VERSION)); + let plural = if oldest == newest { "" } else { "s" }; + if oldest > VERSION { format!( - "a newer build (wire version {version}; this one speaks {ours}): upgrade this node \ - to talk with it" + "a newer build (wire version{plural} {theirs}; this one speaks {ours}): upgrade this \ + node to talk with it" ) } else { format!( - "an older build (wire version {version}; this one speaks {ours}): upgrade that node" + "an older build (wire version{plural} {theirs}; this one speaks {ours}): upgrade that \ + node" ) } } @@ -197,11 +264,14 @@ pub(crate) fn decode(payload: &[u8]) -> Result { revision: rest.get(..32).map(|id| id.try_into().expect("32 bytes")), })); } - if !(OLDEST..=VERSION).contains(&version) { - return Err(WireError::Version(version)); - } let (&kind, body) = payload.split_first().ok_or(WireError::Empty)?; let is = |wanted: MessageKind| kind == tag(wanted); + let proof = version >= PROOFS && (is(MessageKind::Who) || is(MessageKind::Me)); + // Each older version this build still speaks is decoded here, into its own types, + // once its messages differ from the newest's; this build's are the newest's alone. + if !(OLDEST..=VERSION).contains(&version) && !proof { + return Err(WireError::Version(version)); + } Ok(if is(MessageKind::Hello) { ParsedMessage::Hello(rmp_serde::from_slice(body)?) } else if is(MessageKind::HelloReply) { @@ -639,18 +709,76 @@ mod tests { assert!(other.0.lock().unwrap().len() <= MAX_OTHER_VERSIONS); } + /// A peer is written to in the newest version both speak, a version this build doesn't + /// speak never; in the newest one alone, while that is all it speaks. + #[test] + fn the_newest_version_both_speak_is_written() { + assert_eq!(choose(OLDEST, VERSION), Some(VERSION)); + assert_eq!(choose(1, VERSION + 5), Some(VERSION)); + assert_eq!(choose(OLDEST - 2, OLDEST - 1), None); + assert_eq!(choose(VERSION + 1, VERSION + 2), None); + let ping = ParsedMessage::Ping(7); + assert_eq!(encode_in(&ping, VERSION).unwrap(), encode(&ping).unwrap()); + assert!(encode_in(&ping, VERSION + 1).is_err()); + assert!(encoded_in("p", &ping, VERSION).is_ok()); + } + + /// WHO and ME are laid out alike in every version, as in the first with SPEAKS, and + /// read in any version from it on, older or newer than this build's: two builds prove + /// themselves to each other before either knows what the other speaks. These bytes + /// never change. + #[test] + fn proofs_are_read_in_every_version() { + let proof = IdentityProof { + key: [8; 32], + signature: [9; 64], + }; + let who = ParsedMessage::Who(proof.clone()); + let fields = format!("92c420{}c440{}", "08".repeat(32), "09".repeat(64)); + assert_eq!( + hex(&encode_in(&who, PROOFS).unwrap()), + format!("3406{fields}") + ); + let me = ParsedMessage::Me(proof); + assert_eq!( + hex(&encode_in(&me, PROOFS).unwrap()), + format!("3407{fields}") + ); + for version in [PROOFS, OLDEST, VERSION, VERSION + 1, u8::MAX] { + let who = encode_in(&who, version).unwrap(); + assert_eq!(who[0], version); + assert!( + matches!(decode(&who), Ok(ParsedMessage::Who(_))), + "{version}" + ); + let me = encode_in(&me, version).unwrap(); + assert!(matches!(decode(&me), Ok(ParsedMessage::Me(_))), "{version}"); + } + let mut before = encode(&who).unwrap(); + before[0] = PROOFS - 1; + assert!(matches!(decode(&before), Err(WireError::Version(_)))); + let mut ping = encode(&ParsedMessage::Ping(7)).unwrap(); + ping[0] = VERSION + 1; + assert!(matches!(decode(&ping), Err(WireError::Version(_)))); + } + /// The warning says which build is the older, the one to upgrade, and both versions. #[test] fn the_older_build_is_named() { - let newer = other_build(VERSION + 1); + let newer = other_build(VERSION + 1, VERSION + 1); assert!(newer.contains("upgrade this node"), "{newer}"); assert!( newer.contains(&format!("wire version {}", VERSION + 1)), "{newer}" ); assert!(newer.contains(&format!("speaks {VERSION}")), "{newer}"); - let older = other_build(OLDEST - 1); + let older = other_build(OLDEST - 1, OLDEST - 1); assert!(older.contains("upgrade that node"), "{older}"); + let newer = other_build(VERSION + 1, VERSION + 3); + assert!( + newer.contains(&format!("wire versions {} to {}", VERSION + 1, VERSION + 3)), + "{newer}" + ); } /// Every kind decodes to what was encoded: re-encoded, the same bytes. -- cgit v1.3.1