diff options
Diffstat (limited to 'src/transport')
| -rw-r--r-- | src/transport/identity.rs | 154 | ||||
| -rw-r--r-- | src/transport/libp2p/mod.rs | 9 | ||||
| -rw-r--r-- | src/transport/mod.rs | 40 | ||||
| -rw-r--r-- | src/transport/mycelium/mod.rs | 26 | ||||
| -rw-r--r-- | src/transport/paths.rs | 49 | ||||
| -rw-r--r-- | src/transport/sim.rs | 17 | ||||
| -rw-r--r-- | src/transport/tor.rs | 9 | ||||
| -rw-r--r-- | src/transport/traffic.rs | 15 | ||||
| -rw-r--r-- | src/transport/wire.rs | 285 |
9 files changed, 545 insertions, 59 deletions
diff --git a/src/transport/identity.rs b/src/transport/identity.rs index 5d3e2f5..29624db 100644 --- a/src/transport/identity.rs +++ b/src/transport/identity.rs @@ -26,6 +26,12 @@ //! //! Addresses and the transports behind them are the paths layer's //! (`transport::paths`): the books keep addresses as paths keep them. +//! +//! Each address is written the newest wire version its node and this build both speak +//! (`transport::wire`), as a SPEAKS from there, or from another address of its node, +//! said; the oldest this build speaks until one does. A SPEAKS goes ahead of every WHO +//! and ME, and WHO and ME read in every version, so two nodes know what the other speaks +//! once they proved themselves to each other, before anything else goes between them. use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex, MutexGuard}; @@ -40,12 +46,12 @@ use tracing::{debug, info}; use crate::identity::{self, NodeKey}; use crate::locks::MutexExt; use crate::transport::firewall::Firewall; -use crate::transport::paths::{Gate, PathHandler, Paths, scheme_of}; +use crate::transport::paths::{Gate, PathHandler, Paths, Versions, scheme_of}; use crate::transport::stats::{IdentityCounts, IdentityStats, TransportStats}; use crate::transport::traffic::Traffic; use crate::transport::{ Connection, Destination, EncodedMessage, HandlerFn, IdentityProof, MessageHandler, MessageKind, - ParsedMessage, Transport, + ParsedMessage, Transport, wire, }; pub(crate) const WHO_LABEL: &[u8] = b"continuum/who/1"; @@ -116,6 +122,8 @@ struct Shared { /// addresses they proved themselves at, their messages may be large /// (`transport::Admission`). admitted: Mutex<HashSet<String>>, + /// Addresses whose SPEAKS named no wire version we speak, logged once each. + other_versions: wire::OtherVersions, } /// Who is where, by address. @@ -132,6 +140,9 @@ struct Book { asked: HashMap<String, Instant>, /// Seeds being resolved: who waits for a ME from where. waiting: HashMap<String, Vec<oneshot::Sender<String>>>, + /// The wire version to write at an address, where a SPEAKS from there or from its + /// node said what it speaks (`Book::speaks`). + spoken: HashMap<String, u8>, /// Since start (`stats`). counts: IdentityCounts, } @@ -148,11 +159,18 @@ impl Book { if *slot != address { let previous = std::mem::replace(slot, address.to_owned()); self.at.remove(&previous); + self.spoken.remove(&previous); self.counts.address_changes += 1; } } else { addresses.push(address.to_owned()); } + // What the node speaks at its other addresses, it speaks here: one build. + if !self.spoken.contains_key(address) + && let Some(version) = addresses.iter().find_map(|at| self.spoken.get(at).copied()) + { + self.spoken.insert(address.to_owned(), version); + } if let Some((previous, _)) = self.at.insert(address.to_owned(), (node.to_owned(), now)) && previous != node { @@ -166,6 +184,7 @@ impl Book { .map(|(address, _)| address.clone()) && let Some((node, _)) = self.at.remove(&oldest) { + self.spoken.remove(&oldest); self.forget_proof(&node, &oldest); } for waiter in self.waiting.remove(address).into_iter().flatten() { @@ -174,6 +193,39 @@ impl Book { news } + /// A SPEAKS from `address` said its build speaks wire versions `oldest` to `newest`: + /// it, and every address its node proved itself at, are written the newest both + /// speak (`wire::choose`), or the oldest this build speaks where there is none (and + /// the node says it can't read us). Before a proof, for `address` alone: a SPEAKS + /// goes ahead of each WHO and ME. Bounded as the books are: past `MAX_ENTRIES`, + /// addresses no node proved itself at are forgotten first. + fn speaks(&mut self, address: &str, oldest: u8, newest: u8) { + let version = wire::choose(oldest, newest); + let addresses = match self.at.get(address) { + Some((node, _)) => self.proven.get(node).cloned().unwrap_or_default(), + None => vec![address.to_owned()], + }; + for address in addresses { + let Some(version) = version else { + self.spoken.remove(&address); + continue; + }; + if self.spoken.len() >= MAX_ENTRIES && !self.spoken.contains_key(&address) { + let at = &self.at; + self.spoken.retain(|address, _| at.contains_key(address)); + if self.spoken.len() >= MAX_ENTRIES { + continue; + } + } + self.spoken.insert(address, version); + } + } + + /// The wire version to write at `address` (`speaks`). + fn version_at(&self, address: &str) -> u8 { + self.spoken.get(address).copied().unwrap_or(wire::OLDEST) + } + /// The node proven at `address`, which was heard from there `now`. fn heard(&mut self, address: &str, now: Instant) -> Option<String> { self.at.get_mut(address).map(|(node, heard)| { @@ -293,6 +345,7 @@ impl IdentityTransport { let node_id = config.key.id(); let shared = Arc::new_cyclic(|shared: &std::sync::Weak<Shared>| { let gate: std::sync::Weak<dyn Gate> = shared.clone(); + let versions: std::sync::Weak<dyn Versions> = shared.clone(); let admitting = shared.clone(); let admission = crate::transport::Admission::new(move |address| { admitting @@ -310,6 +363,7 @@ impl IdentityTransport { .collect(), Some(Arc::clone(&traffic)), firewall.is_some().then_some(gate), + Some(versions), clock, keepalive, )), @@ -319,6 +373,7 @@ impl IdentityTransport { traffic, book: Mutex::default(), admitted: Mutex::default(), + other_versions: wire::OtherVersions::default(), } }); // The node's messages are noted here, with the address each went to or came @@ -340,6 +395,13 @@ impl Gate for Shared { } } +/// Each address is written what its node speaks (`Book::speaks`). +impl Versions for Shared { + fn version_at(&self, address: &str) -> u8 { + self.lock().version_at(address) + } +} + impl Shared { /// Whether the node proven at `address` is admitted (`Transport::admit`). fn admits(&self, address: &str) -> bool { @@ -442,7 +504,9 @@ impl Shared { tokio::spawn(async move { shared.send_proof(MessageKind::Who, &address).await }); } - /// Sends a WHO or ME (`kind`) to `address`. + /// Sends a WHO or ME (`kind`) to `address`, a SPEAKS ahead of it: so the node there + /// knows what this build speaks once it knows who we are, and the first messages it + /// writes us are in a version we read. async fn send_proof(&self, kind: MessageKind, address: &str) { let Some(address) = self.paths.canonical(address) else { return; @@ -456,10 +520,21 @@ impl Shared { let Some(message) = message else { return; }; - let encoded = match self.paths.encode(address, &message) { + self.send_to( + address, + &ParsedMessage::Speaks(crate::swarm::speaking::own()), + ) + .await; + self.send_to(address, &message).await; + } + + /// Sends `message`, which ends at the identity layer, to `address`, counted under the + /// address (`traffic`). + async fn send_to(&self, address: &str, message: &ParsedMessage) { + let encoded = match self.paths.encode(address, message) { Ok(encoded) => encoded, Err(error) => { - debug!(to = %address, error = %format!("{error:#}"), "Couldn't encode an identity proof"); + debug!(to = %address, kind = ?message.kind(), error = %format!("{error:#}"), "Couldn't encode a message of the identity layer"); return; } }; @@ -470,7 +545,7 @@ impl Shared { Err(error) => { self.traffic .refused(address, encoded.kind, encoded.payload_len); - debug!(to = %address, error = %format!("{error:#}"), "Couldn't send an identity proof"); + debug!(to = %address, kind = ?message.kind(), error = %format!("{error:#}"), "Couldn't send a message of the identity layer"); } } } @@ -499,6 +574,29 @@ impl Shared { self.prove(&address, ME_LABEL, &proof); Ok(()) } + // What the address's build speaks, noted first (`Book::speaks`), and logged + // where we share no version; from a node proven there, it goes on up too + // (`swarm::speaking`). From an address no node proved itself at, it ends here: + // the WHO or ME it goes ahead of follows. + ParsedMessage::Speaks(speaks) => { + self.other_versions + .speaks(&address, speaks.oldest, speaks.newest); + let node = { + let mut book = self.lock(); + book.speaks(&address, speaks.oldest, speaks.newest); + book.heard(&address, Instant::now()) + }; + let Some(node) = node else { + self.traffic + .received(&address, MessageKind::Speaks, payload_len); + return Ok(()); + }; + let message = ParsedMessage::Speaks(speaks); + self.traffic + .log + .note("received", &node, Some(&address), &message, payload_len); + handler(node, message, payload_len).await + } message => { // Heard from: a node that talks isn't the one evicted when strangers' // proofs fill the book (`Book::bind`). @@ -774,7 +872,13 @@ impl Transport for IdentityTransport { /// For the address `destination` picks (a send there is refused until `peer` /// proved itself there, `send_encoded`). Only a look at the choice: the send makes /// it (encoding again for another path if it changes). - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { + /// In the wire version the address speaks, whatever `_version` asks (`Paths::encode`). + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + _version: u8, + ) -> Result<EncodedMessage> { let Some(address) = self.shared.destination(peer, false) else { bail!("no address known for {peer} on this node's transports"); }; @@ -855,6 +959,42 @@ mod tests { assert_eq!(book.at.len(), MAX_ENTRIES); } + /// An address is written what a SPEAKS from there said, and so is every address of + /// the node proven there; the oldest version this build speaks until one says. + #[test] + fn each_address_is_written_what_its_node_speaks() { + let now = Instant::now(); + let (oldest, newest) = (wire::OLDEST, wire::VERSION); + let mut book = Book::default(); + assert_eq!(book.version_at("s:x"), oldest); + // Before a proof, for the address alone. + book.speaks("s:x", oldest, newest + 3); + assert_eq!(book.version_at("s:x"), newest); + book.bind("s:x", "n1", now); + // Another address of the node is written the same: one build. + book.bind("t:x", "n1", now); + assert_eq!(book.version_at("t:x"), newest); + // Moved: the old address is forgotten, the new one written the same. + book.bind("t:y", "n1", now); + assert!(!book.spoken.contains_key("t:x")); + assert_eq!(book.version_at("t:y"), newest); + // Speaking none of this build's versions: the oldest, at each of its addresses. + book.speaks("s:x", newest + 1, newest + 2); + assert!(book.spoken.is_empty()); + assert_eq!(book.version_at("t:y"), oldest); + + // Full: addresses no node proved itself at go first. + let mut book = Book::default(); + book.bind("s:member", "member", now); + for i in 0..MAX_ENTRIES { + book.speaks(&format!("s:{i}"), oldest, newest); + } + assert_eq!(book.spoken.len(), MAX_ENTRIES); + book.speaks("s:member", oldest, newest); + assert_eq!(book.spoken.len(), 1); + assert_eq!(book.version_at("s:member"), newest); + } + #[test] fn one_address_per_node_and_scheme_and_one_node_per_address() { let now = Instant::now(); diff --git a/src/transport/libp2p/mod.rs b/src/transport/libp2p/mod.rs index 2290dea..24f5f34 100644 --- a/src/transport/libp2p/mod.rs +++ b/src/transport/libp2p/mod.rs @@ -572,8 +572,13 @@ impl Transport for Libp2pTransport { } } - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - wire::encoded(peer, message) + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result<EncodedMessage> { + wire::encoded_in(peer, message, version) } async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { diff --git a/src/transport/mod.rs b/src/transport/mod.rs index e366f72..ee63988 100644 --- a/src/transport/mod.rs +++ b/src/transport/mod.rs @@ -254,6 +254,21 @@ pub(crate) enum ParsedMessage { Disconnect(NeighbourAnswer), /// Changes the sender refused, of those the recipient brought (`swarm::refusals`). Refusals(RefusalsMessage), + /// What the sender's build speaks and knows, in a layout no build changes + /// (`wire::SPEAKS`, `swarm::speaking`). + Speaks(Speaks), +} + +/// SPEAKS: the wire versions the sender speaks, the oldest and the newest, and the +/// newest revision of the federation it knows, by ID, where it is built with the +/// federation. Sent with every handshake, and ahead of every WHO and ME +/// (`transport::identity`), in a layout no build changes (`wire::SPEAKS`): a build that +/// doesn't know it ignores it, as a message of another wire version. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Speaks { + pub(crate) oldest: u8, + pub(crate) newest: u8, + pub(crate) revision: Option<[u8; 32]>, } /// REFUSALS: changes the sender refused by the swarm's rules (`access::screen`), of @@ -417,6 +432,7 @@ pub(crate) enum MessageKind { NeighbourReply, Disconnect, Refusals, + Speaks, } impl ParsedMessage { @@ -440,6 +456,7 @@ impl ParsedMessage { ParsedMessage::NeighbourReply(_) => MessageKind::NeighbourReply, ParsedMessage::Disconnect(_) => MessageKind::Disconnect, ParsedMessage::Refusals(_) => MessageKind::Refusals, + ParsedMessage::Speaks(_) => MessageKind::Speaks, } } @@ -469,7 +486,8 @@ impl ParsedMessage { ParsedMessage::Who(_) | ParsedMessage::Me(_) | ParsedMessage::Ping(_) - | ParsedMessage::Pong { .. } => None, + | ParsedMessage::Pong { .. } + | ParsedMessage::Speaks(_) => None, } } } @@ -566,6 +584,9 @@ pub(crate) struct HelloRequest { /// a joining node takes as the root, where those it hears agree. #[serde(with = "serde_bytes")] pub(crate) root: Option<Vec<u8>>, + /// The party the sender is of, in a swarm holding a store (`--store-party`): nodes of + /// one sync with the nodes its parties list, each as its party's. None elsewhere. + pub(crate) party: Option<String>, } /// A peer a HELLO_REPLY lists: who it is, and where the sender reaches it. @@ -654,6 +675,8 @@ pub(crate) struct HelloReply { /// As `HelloRequest::root`. #[serde(with = "serde_bytes")] pub(crate) root: Option<Vec<u8>>, + /// As `HelloRequest::party`. + pub(crate) party: Option<String>, } /// Bytes shown as lowercase hex, e.g. run IDs in logs. @@ -688,6 +711,7 @@ impl std::fmt::Debug for HelloRequest { .field("run_id", &Hex(&self.run_id)) .field("handshake", &self.handshake) .field("root", &self.root.as_deref().map(Hex)) + .field("party", &self.party) .finish() } } @@ -705,6 +729,7 @@ impl std::fmt::Debug for HelloReply { .field("away_ms", &self.away_ms) .field("sender_run_id", &Hex(&self.sender_run_id)) .field("root", &self.root.as_deref().map(Hex)) + .field("party", &self.party) .finish() } } @@ -1174,7 +1199,18 @@ pub(crate) trait Transport: Send + Sync { /// identity, as `local_id` gives it on that node. async fn resolve_seed(&self, seed: &str) -> Result<String>; - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage>; + /// `message` to `peer` in wire `version`, one this build speaks (`wire::OLDEST..= + /// wire::VERSION`). The identity layer writes to each node in the newest version both + /// speak, whatever is asked (`Paths::version_at`). + fn encode_in(&self, peer: &str, message: &ParsedMessage, version: u8) + -> Result<EncodedMessage>; + + /// `message` to `peer` in the newest wire version; under the identity layer, in the one + /// `peer` speaks. + fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { + self.encode_in(peer, message, wire::VERSION) + } + async fn send_encoded(&self, message: &EncodedMessage) -> Result<()>; /// Sends an outbound message payload to a targeted peer on a specific topic. diff --git a/src/transport/mycelium/mod.rs b/src/transport/mycelium/mod.rs index f1dbe40..d5d9271 100644 --- a/src/transport/mycelium/mod.rs +++ b/src/transport/mycelium/mod.rs @@ -355,8 +355,13 @@ impl Transport for MyceliumTransport { self.lookup_pubkey(&ip).await } - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - encode_message(peer, message) + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result<EncodedMessage> { + encode_message(peer, message, version) } async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { @@ -573,9 +578,13 @@ async fn read_capped( /// A message as the Mycelium REST API takes it (the JSON body of an outbound message). /// Also the simulator's wire format (`transport::sim`), so tests go through the same -/// encoding. -pub(crate) fn encode_message(peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - let (outbound, payload_len) = format_message(peer, message)?; +/// encoding. Its payload is in wire `version`. +pub(crate) fn encode_message( + peer: &str, + message: &ParsedMessage, + version: u8, +) -> Result<EncodedMessage> { + let (outbound, payload_len) = format_message(peer, message, version)?; Ok(EncodedMessage { peer: peer.to_owned(), address: peer.to_owned(), @@ -645,12 +654,13 @@ fn parse_message(message: &InboundMessage) -> Result<(String, ParsedMessage), Tr Ok((sender, parsed)) } -/// The message for Mycelium's API, and the length of its payload. +/// The message for Mycelium's API, its payload in wire `version`, and the payload's length. fn format_message( recipient: &str, message: &ParsedMessage, + version: u8, ) -> Result<(OutboundMessage, usize), TransportError> { - let payload_bytes = wire::encode(message)?; + let payload_bytes = wire::encode_in(message, version)?; let outbound = OutboundMessage { dst: Destination { pk: recipient.to_owned(), @@ -797,7 +807,7 @@ mod tests { #[test] fn every_kind_round_trips_through_the_api_body() { for message in wire::samples() { - let encoded = encode_message("peer", &message).unwrap(); + let encoded = encode_message("peer", &message, wire::VERSION).unwrap(); assert_eq!(encoded.kind, message.kind()); let decoded = decode_message(&encoded.body).unwrap(); assert_eq!( diff --git a/src/transport/paths.rs b/src/transport/paths.rs index f425216..e2d9ec9 100644 --- a/src/transport/paths.rs +++ b/src/transport/paths.rs @@ -58,7 +58,7 @@ use crate::clock::{Clock, UnixMillis}; use crate::locks::MutexExt; use crate::transport::stats::{PathCounts, PathInfo, PathsStats, TransportStats}; use crate::transport::traffic::Traffic; -use crate::transport::{Connection, EncodedMessage, ParsedMessage, Transport}; +use crate::transport::{Connection, EncodedMessage, ParsedMessage, Transport, wire}; /// Whether to drop what comes from or goes to an address (the debug firewall, /// `transport::firewall`; the identity layer knows which node is where). @@ -67,6 +67,12 @@ pub(crate) trait Gate: Send + Sync { fn blocks_outbound(&self, address: &str) -> bool; } +/// The wire version to write at an address (`transport::wire`): the newest the node there +/// and this build both speak, as the identity layer heard from it. +pub(crate) trait Versions: Send + Sync { + fn version_at(&self, address: &str) -> u8; +} + /// Called with the sender's address, the message and its `payload_len`. pub(crate) type PathHandler = dyn Fn(String, ParsedMessage, usize) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + Send>> + Send @@ -547,6 +553,8 @@ pub(crate) struct Paths { traffic: Option<Arc<Traffic>>, /// The debug firewall, if any. gate: Option<Weak<dyn Gate>>, + /// What each address speaks; without, every address is written `wire::OLDEST`. + versions: Option<Weak<dyn Versions>>, /// The node's wall clock, which its PONGs say the time by. clock: Arc<Clock>, /// A std mutex: never held across an await. @@ -555,12 +563,14 @@ pub(crate) struct Paths { impl Paths { /// Over `transports`, by scheme, in order of preference; its own PINGs and PONGs - /// counted in `traffic`, the firewall through `gate`, PONGs stamped by `clock`; - /// paths pinged every `keepalive` at least (`Table::probe_every`). + /// counted in `traffic`, the firewall through `gate`, each address written the wire + /// version `versions` says, PONGs stamped by `clock`; paths pinged every `keepalive` + /// at least (`Table::probe_every`). pub(crate) fn new( transports: Vec<(&'static str, Arc<dyn Transport>)>, traffic: Option<Arc<Traffic>>, gate: Option<Weak<dyn Gate>>, + versions: Option<Weak<dyn Versions>>, clock: Arc<Clock>, keepalive: Duration, ) -> Self { @@ -571,6 +581,7 @@ impl Paths { .collect(), traffic, gate, + versions, clock, table: Mutex::new(Table::new(keepalive)), } @@ -580,6 +591,15 @@ impl Paths { self.gate.as_ref()?.upgrade() } + /// The wire version to write at `address` (`Versions`): the oldest this build speaks, + /// until its node says what it speaks. + pub(crate) fn version_at(&self, address: &str) -> u8 { + self.versions + .as_ref() + .and_then(Weak::upgrade) + .map_or(wire::OLDEST, |versions| versions.version_at(address)) + } + /// The transport `address` is on, and the transport identity at it /// (`Transport::locate`, which notes where that is if the address says); None if no /// transport of ours has its scheme, or it isn't one of that transport's. @@ -825,13 +845,16 @@ impl Paths { Err(anyhow!("{}", failures.join("; "))) } - /// `message` encoded for `address`: `peer` and `address` of the result are - /// `address`, and it keeps the message (to encode it for another address). + /// `message` encoded for `address`, in the wire version it speaks (`version_at`): + /// `peer` and `address` of the result are `address`, and it keeps the message (to + /// encode it for another address). pub(crate) fn encode(&self, address: &str, message: &ParsedMessage) -> Result<EncodedMessage> { let (link, target) = self .split(address) .with_context(|| format!("no transport for {address}"))?; - let mut encoded = link.inner.encode(&target, message)?; + let mut encoded = link + .inner + .encode_in(&target, message, self.version_at(address))?; encoded.peer = address.to_owned(); encoded.address = address.to_owned(); encoded.message = Some(Arc::new(message.clone())); @@ -1195,6 +1218,7 @@ mod tests { ], None, None, + None, Arc::default(), Duration::ZERO, )) @@ -1608,6 +1632,7 @@ mod tests { vec![("sim", net.transport("b") as Arc<dyn Transport>)], None, Some(Arc::downgrade(&gate)), + None, Arc::default(), Duration::ZERO, )); @@ -1615,6 +1640,7 @@ mod tests { vec![("sim", net.transport("a") as Arc<dyn Transport>)], None, None, + None, Arc::default(), Duration::ZERO, )); @@ -1673,8 +1699,13 @@ mod tests { async fn resolve_seed(&self, seed: &str) -> Result<String> { Ok(seed.to_owned()) } - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - (crate::transport::sim::WIRE.encode)(peer, message) + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result<EncodedMessage> { + (crate::transport::sim::WIRE.encode)(peer, message, version) } async fn send_encoded(&self, _message: &EncodedMessage) -> Result<()> { Err(anyhow!("broken")) @@ -1713,6 +1744,7 @@ mod tests { ], None, None, + None, Arc::default(), Duration::ZERO, ); @@ -1720,6 +1752,7 @@ mod tests { vec![("sim", net.transport("b") as Arc<dyn Transport>)], None, None, + None, Arc::default(), Duration::ZERO, )); diff --git a/src/transport/sim.rs b/src/transport/sim.rs index 40a1f7d..5d55e32 100644 --- a/src/transport/sim.rs +++ b/src/transport/sim.rs @@ -327,7 +327,7 @@ impl SimNet { /// net. For messages a real node wouldn't send at that moment, e.g. one sent long /// ago and delivered late. pub(crate) async fn inject(&self, from: &str, to: &str, message: &ParsedMessage) -> Result<()> { - let encoded = (self.codec.encode)(to, message)?; + let encoded = (self.codec.encode)(to, message, super::wire::VERSION)?; let (handled_tx, handled_rx) = oneshot::channel(); let delivery = Delivery { from: from.to_owned(), @@ -368,7 +368,7 @@ pub(crate) struct SimTransport { /// around it (Mycelium's: `mycelium::CODEC`), so a whole swarm runs on that envelope. #[derive(Clone, Copy)] pub(crate) struct Codec { - pub(crate) encode: fn(&str, &ParsedMessage) -> Result<EncodedMessage>, + pub(crate) encode: fn(&str, &ParsedMessage, u8) -> Result<EncodedMessage>, pub(crate) decode: fn(&[u8]) -> Result<ParsedMessage>, } @@ -378,8 +378,8 @@ pub(crate) const WIRE: Codec = Codec { decode: wire_decode, }; -fn wire_encode(peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - super::wire::encoded(peer, message) +fn wire_encode(peer: &str, message: &ParsedMessage, version: u8) -> Result<EncodedMessage> { + super::wire::encoded_in(peer, message, version) } fn wire_decode(body: &[u8]) -> Result<ParsedMessage> { @@ -410,8 +410,13 @@ impl Transport for SimTransport { } } - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - (self.net.codec.encode)(peer, message) + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result<EncodedMessage> { + (self.net.codec.encode)(peer, message, version) } /// Always accepted: the net loses messages silently, like Mycelium once it has diff --git a/src/transport/tor.rs b/src/transport/tor.rs index c194330..a98e1c4 100644 --- a/src/transport/tor.rs +++ b/src/transport/tor.rs @@ -658,8 +658,13 @@ impl Transport for TorTransport { Ok(location) } - fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { - wire::encoded(peer, message) + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result<EncodedMessage> { + wire::encoded_in(peer, message, version) } async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { diff --git a/src/transport/traffic.rs b/src/transport/traffic.rs index 5ff588e..33903ca 100644 --- a/src/transport/traffic.rs +++ b/src/transport/traffic.rs @@ -40,6 +40,7 @@ impl Counter { #[derive(Debug, Default, Clone, Copy, Serialize)] #[serde(rename_all = "camelCase")] pub(crate) struct ByKind { + /// HELLO and SPEAKS, which goes with each handshake. pub(crate) hello: Counter, pub(crate) hello_reply: Counter, pub(crate) sync: Counter, @@ -84,7 +85,7 @@ impl ByKind { fn get(&self, kind: MessageKind) -> &Counter { match kind { - MessageKind::Hello => &self.hello, + MessageKind::Hello | MessageKind::Speaks => &self.hello, MessageKind::HelloReply => &self.hello_reply, MessageKind::Sync => &self.sync, MessageKind::Status => &self.status, @@ -103,7 +104,7 @@ impl ByKind { fn get_mut(&mut self, kind: MessageKind) -> &mut Counter { match kind { - MessageKind::Hello => &mut self.hello, + MessageKind::Hello | MessageKind::Speaks => &mut self.hello, MessageKind::HelloReply => &mut self.hello_reply, MessageKind::Sync => &mut self.sync, MessageKind::Status => &mut self.status, @@ -366,6 +367,7 @@ fn names(kind: MessageKind) -> (&'static str, &'static str) { MessageKind::NeighbourReply => ("NEIGHBOUR_REPLY", "hyparview"), MessageKind::Disconnect => ("DISCONNECT", "hyparview"), MessageKind::Refusals => ("REFUSALS", "refusals"), + MessageKind::Speaks => ("SPEAKS", "hello"), } } @@ -535,6 +537,15 @@ impl Transport for CountingTransport { self.inner.resolve_seed(seed).await } + fn encode_in( + &self, + peer: &str, + message: &ParsedMessage, + version: u8, + ) -> Result<EncodedMessage> { + self.inner.encode_in(peer, message, version) + } + fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> { self.inner.encode(peer, message) } diff --git a/src/transport/wire.rs b/src/transport/wire.rs index 97bf6cc..56883e8 100644 --- a/src/transport/wire.rs +++ b/src/transport/wire.rs @@ -3,10 +3,26 @@ //! arrays, fields in declaration order). The codecs below make hashes, IDs and bytes //! compact (MessagePack binaries, not arrays of numbers) and check their lengths. //! -//! The wire format may change freely until the first release: there is no compatibility -//! with older nodes to keep. But every change bumps `VERSION`, so nodes on different -//! builds tell each other apart instead of misreading each other's messages. The +//! Every change to a message type or a codec is a new version (`VERSION`), and a build +//! speaks several side by side, `OLDEST` to `VERSION`: it reads each, and writes to each +//! node the newest both speak (`choose`), as the node's SPEAKS says, or `OLDEST` until +//! then (`transport::identity`). So releases whose versions meet roll out node by node: +//! a release adds a version and keeps the older ones, and a later one leaves out the +//! oldest. The messages of an older version keep their own types, frozen, encoded and +//! decoded here beside the newest's (`golden_bytes` keeps the bytes of each). The //! transport's topic says which application a message is for, not which version. +//! +//! Two kinds of messages stand outside the versions, as they are what lets two builds +//! learn what the other speaks: +//! +//! * SPEAKS (`SPEAKS`), its first byte the version none is, laid out as no build changes, +//! saying which versions a node speaks and the newest revision of the federation it +//! knows. A build from before it reads it as a message of another version, and +//! ignores it. +//! * WHO and ME, the proofs of who is at an address (`PROOFS`), laid out in every +//! version as in the first: a build reads them whatever version they say they are in, +//! from that first on, so that two nodes prove themselves to each other before they +//! know what the other speaks. use thiserror::Error; @@ -14,7 +30,26 @@ use super::{EncodedMessage, MessageKind, ParsedMessage}; /// The wire format's version, the first byte of every payload. Bump it with every change /// to a message type or a codec (`golden_bytes` fails then). -pub(crate) const VERSION: u8 = 51; +pub(crate) const VERSION: u8 = 52; + +/// The oldest version this build speaks: the messages of every version from it to +/// `VERSION` it reads. Its SPEAKS says so. +pub(crate) const OLDEST: u8 = VERSION; + +/// The first byte of a SPEAKS, which no version is. Its layout, never to change: that +/// byte, the oldest version the sender speaks, the newest, then the 32 bytes of the ID +/// of the newest revision of the federation it knows, where it knows one. A reader takes +/// those and leaves whatever follows, which a later build may add. +pub(crate) const SPEAKS: u8 = 0; + +/// The first version with SPEAKS, as WHO and ME are laid out in every version: their +/// tags and fields never change (a change would be a new kind of message), and a build +/// reads them in any version from this one on, older or newer than its own. +pub(crate) const PROOFS: u8 = 52; + +// Whatever this build's versions, a SPEAKS reads as one, and it reads the proofs of +// every build that speaks one of its versions. +const _: () = assert!(SPEAKS < PROOFS && PROOFS <= OLDEST && OLDEST <= VERSION); /// What the second byte of a payload says. fn tag(kind: MessageKind) -> u8 { @@ -37,6 +72,8 @@ fn tag(kind: MessageKind) -> u8 { MessageKind::NeighbourReply => 17, MessageKind::Disconnect => 18, MessageKind::Refusals => 20, + // None: a SPEAKS has no second byte (`SPEAKS`). + MessageKind::Speaks => u8::MAX, } } @@ -58,11 +95,15 @@ pub(crate) enum WireError { Decode(#[from] rmp_serde::decode::Error), } -/// `message` to `peer`, encoded as it goes on the wire alone (no envelope: libp2p, -/// Tor, the simulator); `peer` is its address too. +/// `message` to `peer`, encoded in wire `version` as it goes on the wire alone (no +/// envelope: libp2p, Tor, the simulator); `peer` is its address too. #[cfg_attr(not(any(feature = "libp2p", feature = "tor", test)), allow(dead_code))] -pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result<EncodedMessage> { - let body = encode(message)?; +pub(crate) fn encoded_in( + peer: &str, + message: &ParsedMessage, + version: u8, +) -> anyhow::Result<EncodedMessage> { + let body = encode_in(message, version)?; Ok(EncodedMessage { peer: peer.to_owned(), address: peer.to_owned(), @@ -73,8 +114,37 @@ pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result<Enc }) } -/// The payload of `message`. +/// The newest wire version a peer speaking `oldest` to `newest` (its SPEAKS) and this +/// build both speak: what this build writes to it. None where they share none. +pub(crate) fn choose(oldest: u8, newest: u8) -> Option<u8> { + let (low, high) = (oldest.max(OLDEST), newest.min(VERSION)); + (low <= high).then_some(high) +} + +/// The payload of `message` in wire `version`, one of `OLDEST..=VERSION`: of an older one, +/// as a build speaking only that one reads it. A SPEAKS is laid out alike in every one, +/// and so are WHO and ME (`PROOFS`), but for the version they say. +pub(crate) fn encode_in(message: &ParsedMessage, version: u8) -> Result<Vec<u8>, WireError> { + if matches!(message, ParsedMessage::Who(_) | ParsedMessage::Me(_)) && version >= PROOFS { + let mut payload = encode(message)?; + payload[0] = version; + return Ok(payload); + } + // Each older version this build still speaks is encoded here, from its own types, + // once its messages differ from the newest's; this build's are the newest's alone. + if version != VERSION && !matches!(message, ParsedMessage::Speaks(_)) { + return Err(WireError::Version(version)); + } + encode(message) +} + +/// The payload of `message`, in the newest wire version. pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> { + if let ParsedMessage::Speaks(speaks) = message { + let mut payload = vec![SPEAKS, speaks.oldest, speaks.newest]; + payload.extend(speaks.revision.iter().flatten()); + return Ok(payload); + } let mut payload = vec![VERSION, tag(message.kind())]; match message { ParsedMessage::Hello(hello) => rmp_serde::encode::write(&mut payload, hello)?, @@ -99,6 +169,7 @@ pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> { rmp_serde::encode::write(&mut payload, answer)? } ParsedMessage::Refusals(message) => rmp_serde::encode::write(&mut payload, message)?, + ParsedMessage::Speaks(_) => unreachable!("encoded above"), } Ok(payload) } @@ -108,16 +179,33 @@ pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> { /// (`OtherVersions`). const MAX_OTHER_VERSIONS: usize = 1024; -/// Senders whose messages are of another wire version (`WireError::Version`): each -/// logged once, at warn level, so two builds that don't speak to each other say so, -/// instead of silently retrying their HELLOs for good. Bounded: anyone can send us -/// anything. +/// Senders whose messages are of another wire version (`WireError::Version`), or whose +/// SPEAKS names none this build speaks: each logged once, at warn level, so two builds +/// that don't speak to each other say so, and which of them to upgrade, instead of +/// silently retrying their HELLOs for good. Bounded: anyone can send us anything. #[derive(Default)] pub(crate) struct OtherVersions(std::sync::Mutex<std::collections::HashSet<String>>); impl OtherVersions { /// A message from `sender` (on the transport `scheme`) was of `version`. pub(crate) fn note(&self, scheme: &'static str, sender: &str, version: u8) { + self.warn( + &super::Addressed(scheme, sender).to_string(), + version, + version, + ); + } + + /// A SPEAKS from `address` (as paths keep it) said its build speaks wire versions + /// `oldest` to `newest`: logged where this build speaks none of them, as a message of + /// another version is. The proofs between the two still go (`PROOFS`), nothing else. + pub(crate) fn speaks(&self, address: &str, oldest: u8, newest: u8) { + if choose(oldest, newest).is_none() { + self.warn(address, oldest, newest); + } + } + + fn warn(&self, from: &str, oldest: u8, newest: u8) { let mut known = self .0 .lock() @@ -125,24 +213,65 @@ impl OtherVersions { if known.len() >= MAX_OTHER_VERSIONS { known.clear(); } - if known.insert(sender.to_owned()) { + if known.insert(from.to_owned()) { tracing::warn!( - from = %super::Addressed(scheme, &sender), - version, - ours = VERSION, - "Peer speaks another wire version (another build); ignoring its messages" + from = %from, + version = newest, + oldest = OLDEST, + newest = VERSION, + "Peer speaks another wire version: {}; ignoring its messages", + other_build(oldest, newest) ); } } } +/// Of a peer speaking wire versions `oldest` to `newest`, none of which this build +/// speaks, what an operator does: upgrade this node where the peer's build is newer, the +/// peer's where it is older. +fn other_build(oldest: u8, newest: u8) -> String { + let versions = |oldest: u8, newest: u8| { + if oldest == newest { + format!("{oldest}") + } else { + format!("{oldest} to {newest}") + } + }; + let (theirs, ours) = (versions(oldest, newest), versions(OLDEST, VERSION)); + let plural = if oldest == newest { "" } else { "s" }; + if oldest > VERSION { + format!( + "a newer build (wire version{plural} {theirs}; this one speaks {ours}): upgrade this \ + node to talk with it" + ) + } else { + format!( + "an older build (wire version{plural} {theirs}; this one speaks {ours}): upgrade that \ + node" + ) + } +} + pub(crate) fn decode(payload: &[u8]) -> Result<ParsedMessage, WireError> { let (&version, payload) = payload.split_first().ok_or(WireError::Empty)?; - if version != VERSION { - return Err(WireError::Version(version)); + if version == SPEAKS { + let [oldest, newest, rest @ ..] = payload else { + return Err(WireError::Empty); + }; + return Ok(ParsedMessage::Speaks(super::Speaks { + oldest: *oldest, + newest: *newest, + revision: rest.get(..32).map(|id| id.try_into().expect("32 bytes")), + })); } let (&kind, body) = payload.split_first().ok_or(WireError::Empty)?; let is = |wanted: MessageKind| kind == tag(wanted); + let proof = version >= PROOFS && (is(MessageKind::Who) || is(MessageKind::Me)); + // Each older version this build still speaks is decoded here, into its own types, + // once its messages differ from the newest's; this build's are the newest's alone. + if !(OLDEST..=VERSION).contains(&version) && !proof { + return Err(WireError::Version(version)); + } Ok(if is(MessageKind::Hello) { ParsedMessage::Hello(rmp_serde::from_slice(body)?) } else if is(MessageKind::HelloReply) { @@ -458,6 +587,7 @@ pub(crate) fn samples() -> Vec<ParsedMessage> { run_id: [4; 16], handshake: 7, root: None, + party: None, }), ParsedMessage::HelloReply(HelloReply { swarm_id: "swarm".into(), @@ -473,6 +603,7 @@ pub(crate) fn samples() -> Vec<ParsedMessage> { away_ms: Some(5), sender_run_id: [6; 16], root: None, + party: None, }), ParsedMessage::Sync(SyncEnvelope { epoch: crate::transport::EpochTag::of(epoch), @@ -578,6 +709,78 @@ mod tests { assert!(other.0.lock().unwrap().len() <= MAX_OTHER_VERSIONS); } + /// A peer is written to in the newest version both speak, a version this build doesn't + /// speak never; in the newest one alone, while that is all it speaks. + #[test] + fn the_newest_version_both_speak_is_written() { + assert_eq!(choose(OLDEST, VERSION), Some(VERSION)); + assert_eq!(choose(1, VERSION + 5), Some(VERSION)); + assert_eq!(choose(OLDEST - 2, OLDEST - 1), None); + assert_eq!(choose(VERSION + 1, VERSION + 2), None); + let ping = ParsedMessage::Ping(7); + assert_eq!(encode_in(&ping, VERSION).unwrap(), encode(&ping).unwrap()); + assert!(encode_in(&ping, VERSION + 1).is_err()); + assert!(encoded_in("p", &ping, VERSION).is_ok()); + } + + /// WHO and ME are laid out alike in every version, as in the first with SPEAKS, and + /// read in any version from it on, older or newer than this build's: two builds prove + /// themselves to each other before either knows what the other speaks. These bytes + /// never change. + #[test] + fn proofs_are_read_in_every_version() { + let proof = IdentityProof { + key: [8; 32], + signature: [9; 64], + }; + let who = ParsedMessage::Who(proof.clone()); + let fields = format!("92c420{}c440{}", "08".repeat(32), "09".repeat(64)); + assert_eq!( + hex(&encode_in(&who, PROOFS).unwrap()), + format!("3406{fields}") + ); + let me = ParsedMessage::Me(proof); + assert_eq!( + hex(&encode_in(&me, PROOFS).unwrap()), + format!("3407{fields}") + ); + for version in [PROOFS, OLDEST, VERSION, VERSION + 1, u8::MAX] { + let who = encode_in(&who, version).unwrap(); + assert_eq!(who[0], version); + assert!( + matches!(decode(&who), Ok(ParsedMessage::Who(_))), + "{version}" + ); + let me = encode_in(&me, version).unwrap(); + assert!(matches!(decode(&me), Ok(ParsedMessage::Me(_))), "{version}"); + } + let mut before = encode(&who).unwrap(); + before[0] = PROOFS - 1; + assert!(matches!(decode(&before), Err(WireError::Version(_)))); + let mut ping = encode(&ParsedMessage::Ping(7)).unwrap(); + ping[0] = VERSION + 1; + assert!(matches!(decode(&ping), Err(WireError::Version(_)))); + } + + /// The warning says which build is the older, the one to upgrade, and both versions. + #[test] + fn the_older_build_is_named() { + let newer = other_build(VERSION + 1, VERSION + 1); + assert!(newer.contains("upgrade this node"), "{newer}"); + assert!( + newer.contains(&format!("wire version {}", VERSION + 1)), + "{newer}" + ); + assert!(newer.contains(&format!("speaks {VERSION}")), "{newer}"); + let older = other_build(OLDEST - 1, OLDEST - 1); + assert!(older.contains("upgrade that node"), "{older}"); + let newer = other_build(VERSION + 1, VERSION + 3); + assert!( + newer.contains(&format!("wire versions {} to {}", VERSION + 1, VERSION + 3)), + "{newer}" + ); + } + /// Every kind decodes to what was encoded: re-encoded, the same bytes. #[test] fn every_kind_round_trips() { @@ -606,6 +809,7 @@ mod tests { run_id: [4; 16], handshake: 7, root: None, + party: Some("tv".into()), }); let ParsedMessage::Hello(decoded) = decode(&encode(&hello).unwrap()).unwrap() else { panic!("not a HELLO"); @@ -614,6 +818,7 @@ mod tests { assert_eq!((decoded.epoch, decoded.epoch_seq), (epoch, 1)); assert_eq!(decoded.auth, vec![1, 2, 3]); assert_eq!(decoded.run_id, [4; 16]); + assert_eq!(decoded.party.as_deref(), Some("tv")); assert_eq!(decoded.handshake, 7); let reply = ParsedMessage::HelloReply(HelloReply { @@ -630,6 +835,7 @@ mod tests { away_ms: Some(5), sender_run_id: [6; 16], root: None, + party: None, }); let ParsedMessage::HelloReply(decoded) = decode(&encode(&reply).unwrap()).unwrap() else { panic!("not a HELLO_REPLY"); @@ -724,6 +930,39 @@ mod tests { crate::transport::Hex(bytes).to_string() } + /// A SPEAKS is laid out as no build changes: its version byte, the oldest and newest + /// version, the revision's ID where there is one; what follows is left. + #[test] + fn speaks_keeps_its_layout() { + let speaks = super::super::Speaks { + oldest: 50, + newest: 52, + revision: Some([7; 32]), + }; + let bytes = encode(&ParsedMessage::Speaks(speaks)).unwrap(); + assert_eq!(hex(&bytes), format!("003234{}", "07".repeat(32))); + let without = super::super::Speaks { + revision: None, + ..speaks + }; + assert_eq!( + hex(&encode(&ParsedMessage::Speaks(without)).unwrap()), + "003234" + ); + for (payload, read) in [ + (bytes.clone(), speaks), + ([bytes.as_slice(), b"later"].concat(), speaks), + (vec![0, 50, 52], without), + (vec![0, 50, 52, 1, 2], without), + ] { + match decode(&payload) { + Ok(ParsedMessage::Speaks(decoded)) => assert_eq!(decoded, read), + other => panic!("{other:?}"), + } + } + assert!(decode(&[0, 50]).is_err()); + } + /// The exact bytes of a few messages. If this fails after a change to a message type /// or a codec, the change is a new wire format: bump `VERSION` and take the new bytes. #[test] @@ -746,11 +985,12 @@ mod tests { run_id: [4; 16], handshake: 7, root: None, + party: None, }); assert_eq!( hex(&encode(&hello).unwrap()), format!( - "{v}0098a5737761726d{epoch_hex}01c403010203c3{}07c0", + "{v}0099a5737761726d{epoch_hex}01c403010203c3{}07c0c0", run("04") ) ); @@ -769,11 +1009,12 @@ mod tests { away_ms: Some(5), sender_run_id: [6; 16], root: None, + party: None, }); assert_eq!( hex(&encode(&reply).unwrap()), format!( - "{v}019aa5737761726d9192a17091a161{epoch_hex}01c400c2{}05{}c0", + "{v}019ba5737761726d9192a17091a161{epoch_hex}01c400c2{}05{}c0c0", run("04"), run("06") ) |
