aboutsummaryrefslogtreecommitdiffstats
path: root/src/transport
diff options
context:
space:
mode:
authortv <tv@krebsco.de>2026-11-19 17:44:03 +0000
committertv <tv@krebsco.de>2026-11-19 19:26:01 +0100
commit59e889254b1fda2dcc8e3e6d596635d1b77f589f (patch)
tree2412dd262e7ae729498eb3300912395b601cd0b6 /src/transport
parent180aba9ff89a4a294e284aa281d96adbcfe1b85b (diff)
transport: wire versions spoken side by side
A build speaks wire versions OLDEST to VERSION and writes each node the newest both speak (wire::choose), as the node's SPEAKS says; the oldest until it says. The identity layer keeps that per address (Book::speaks): a SPEAKS from an address sets it for the address, and for every address of the node proven there, and a new address of a node takes it over. Paths encodes each message for an address in its version (Transport::encode_in; Transport::encode is the newest). So that two nodes know what the other speaks once they have proven themselves to each other, before any other message goes between them, a SPEAKS goes ahead of every WHO and ME, and WHO and ME are read in every version from 52 on (wire::PROOFS): their tags and layout never change. Without that, a node whose oldest version is past the other's couldn't read its proofs, and the two would never get to the handshake that says what each speaks. A SPEAKS naming no version this build speaks is logged as a message of another version is, naming the node to upgrade, the versions as ranges. A 0.0.1 node takes a SPEAKS from an address not yet proven for any other message from there: it drops it, and asks WHO. This build speaks 52 alone, so nothing it sends changes but the SPEAKS ahead of the proofs. A release adding a version encodes and decodes the older ones from their own, frozen, types.
Diffstat (limited to 'src/transport')
-rw-r--r--src/transport/identity.rs154
-rw-r--r--src/transport/libp2p/mod.rs9
-rw-r--r--src/transport/mod.rs18
-rw-r--r--src/transport/mycelium/mod.rs26
-rw-r--r--src/transport/paths.rs49
-rw-r--r--src/transport/sim.rs17
-rw-r--r--src/transport/tor.rs9
-rw-r--r--src/transport/traffic.rs9
-rw-r--r--src/transport/wire.rs206
9 files changed, 422 insertions, 75 deletions
diff --git a/src/transport/identity.rs b/src/transport/identity.rs
index 5d3e2f5..29624db 100644
--- a/src/transport/identity.rs
+++ b/src/transport/identity.rs
@@ -26,6 +26,12 @@
//!
//! Addresses and the transports behind them are the paths layer's
//! (`transport::paths`): the books keep addresses as paths keep them.
+//!
+//! Each address is written the newest wire version its node and this build both speak
+//! (`transport::wire`), as a SPEAKS from there, or from another address of its node,
+//! said; the oldest this build speaks until one does. A SPEAKS goes ahead of every WHO
+//! and ME, and WHO and ME read in every version, so two nodes know what the other speaks
+//! once they proved themselves to each other, before anything else goes between them.
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex, MutexGuard};
@@ -40,12 +46,12 @@ use tracing::{debug, info};
use crate::identity::{self, NodeKey};
use crate::locks::MutexExt;
use crate::transport::firewall::Firewall;
-use crate::transport::paths::{Gate, PathHandler, Paths, scheme_of};
+use crate::transport::paths::{Gate, PathHandler, Paths, Versions, scheme_of};
use crate::transport::stats::{IdentityCounts, IdentityStats, TransportStats};
use crate::transport::traffic::Traffic;
use crate::transport::{
Connection, Destination, EncodedMessage, HandlerFn, IdentityProof, MessageHandler, MessageKind,
- ParsedMessage, Transport,
+ ParsedMessage, Transport, wire,
};
pub(crate) const WHO_LABEL: &[u8] = b"continuum/who/1";
@@ -116,6 +122,8 @@ struct Shared {
/// addresses they proved themselves at, their messages may be large
/// (`transport::Admission`).
admitted: Mutex<HashSet<String>>,
+ /// Addresses whose SPEAKS named no wire version we speak, logged once each.
+ other_versions: wire::OtherVersions,
}
/// Who is where, by address.
@@ -132,6 +140,9 @@ struct Book {
asked: HashMap<String, Instant>,
/// Seeds being resolved: who waits for a ME from where.
waiting: HashMap<String, Vec<oneshot::Sender<String>>>,
+ /// The wire version to write at an address, where a SPEAKS from there or from its
+ /// node said what it speaks (`Book::speaks`).
+ spoken: HashMap<String, u8>,
/// Since start (`stats`).
counts: IdentityCounts,
}
@@ -148,11 +159,18 @@ impl Book {
if *slot != address {
let previous = std::mem::replace(slot, address.to_owned());
self.at.remove(&previous);
+ self.spoken.remove(&previous);
self.counts.address_changes += 1;
}
} else {
addresses.push(address.to_owned());
}
+ // What the node speaks at its other addresses, it speaks here: one build.
+ if !self.spoken.contains_key(address)
+ && let Some(version) = addresses.iter().find_map(|at| self.spoken.get(at).copied())
+ {
+ self.spoken.insert(address.to_owned(), version);
+ }
if let Some((previous, _)) = self.at.insert(address.to_owned(), (node.to_owned(), now))
&& previous != node
{
@@ -166,6 +184,7 @@ impl Book {
.map(|(address, _)| address.clone())
&& let Some((node, _)) = self.at.remove(&oldest)
{
+ self.spoken.remove(&oldest);
self.forget_proof(&node, &oldest);
}
for waiter in self.waiting.remove(address).into_iter().flatten() {
@@ -174,6 +193,39 @@ impl Book {
news
}
+ /// A SPEAKS from `address` said its build speaks wire versions `oldest` to `newest`:
+ /// it, and every address its node proved itself at, are written the newest both
+ /// speak (`wire::choose`), or the oldest this build speaks where there is none (and
+ /// the node says it can't read us). Before a proof, for `address` alone: a SPEAKS
+ /// goes ahead of each WHO and ME. Bounded as the books are: past `MAX_ENTRIES`,
+ /// addresses no node proved itself at are forgotten first.
+ fn speaks(&mut self, address: &str, oldest: u8, newest: u8) {
+ let version = wire::choose(oldest, newest);
+ let addresses = match self.at.get(address) {
+ Some((node, _)) => self.proven.get(node).cloned().unwrap_or_default(),
+ None => vec![address.to_owned()],
+ };
+ for address in addresses {
+ let Some(version) = version else {
+ self.spoken.remove(&address);
+ continue;
+ };
+ if self.spoken.len() >= MAX_ENTRIES && !self.spoken.contains_key(&address) {
+ let at = &self.at;
+ self.spoken.retain(|address, _| at.contains_key(address));
+ if self.spoken.len() >= MAX_ENTRIES {
+ continue;
+ }
+ }
+ self.spoken.insert(address, version);
+ }
+ }
+
+ /// The wire version to write at `address` (`speaks`).
+ fn version_at(&self, address: &str) -> u8 {
+ self.spoken.get(address).copied().unwrap_or(wire::OLDEST)
+ }
+
/// The node proven at `address`, which was heard from there `now`.
fn heard(&mut self, address: &str, now: Instant) -> Option<String> {
self.at.get_mut(address).map(|(node, heard)| {
@@ -293,6 +345,7 @@ impl IdentityTransport {
let node_id = config.key.id();
let shared = Arc::new_cyclic(|shared: &std::sync::Weak<Shared>| {
let gate: std::sync::Weak<dyn Gate> = shared.clone();
+ let versions: std::sync::Weak<dyn Versions> = shared.clone();
let admitting = shared.clone();
let admission = crate::transport::Admission::new(move |address| {
admitting
@@ -310,6 +363,7 @@ impl IdentityTransport {
.collect(),
Some(Arc::clone(&traffic)),
firewall.is_some().then_some(gate),
+ Some(versions),
clock,
keepalive,
)),
@@ -319,6 +373,7 @@ impl IdentityTransport {
traffic,
book: Mutex::default(),
admitted: Mutex::default(),
+ other_versions: wire::OtherVersions::default(),
}
});
// The node's messages are noted here, with the address each went to or came
@@ -340,6 +395,13 @@ impl Gate for Shared {
}
}
+/// Each address is written what its node speaks (`Book::speaks`).
+impl Versions for Shared {
+ fn version_at(&self, address: &str) -> u8 {
+ self.lock().version_at(address)
+ }
+}
+
impl Shared {
/// Whether the node proven at `address` is admitted (`Transport::admit`).
fn admits(&self, address: &str) -> bool {
@@ -442,7 +504,9 @@ impl Shared {
tokio::spawn(async move { shared.send_proof(MessageKind::Who, &address).await });
}
- /// Sends a WHO or ME (`kind`) to `address`.
+ /// Sends a WHO or ME (`kind`) to `address`, a SPEAKS ahead of it: so the node there
+ /// knows what this build speaks once it knows who we are, and the first messages it
+ /// writes us are in a version we read.
async fn send_proof(&self, kind: MessageKind, address: &str) {
let Some(address) = self.paths.canonical(address) else {
return;
@@ -456,10 +520,21 @@ impl Shared {
let Some(message) = message else {
return;
};
- let encoded = match self.paths.encode(address, &message) {
+ self.send_to(
+ address,
+ &ParsedMessage::Speaks(crate::swarm::speaking::own()),
+ )
+ .await;
+ self.send_to(address, &message).await;
+ }
+
+ /// Sends `message`, which ends at the identity layer, to `address`, counted under the
+ /// address (`traffic`).
+ async fn send_to(&self, address: &str, message: &ParsedMessage) {
+ let encoded = match self.paths.encode(address, message) {
Ok(encoded) => encoded,
Err(error) => {
- debug!(to = %address, error = %format!("{error:#}"), "Couldn't encode an identity proof");
+ debug!(to = %address, kind = ?message.kind(), error = %format!("{error:#}"), "Couldn't encode a message of the identity layer");
return;
}
};
@@ -470,7 +545,7 @@ impl Shared {
Err(error) => {
self.traffic
.refused(address, encoded.kind, encoded.payload_len);
- debug!(to = %address, error = %format!("{error:#}"), "Couldn't send an identity proof");
+ debug!(to = %address, kind = ?message.kind(), error = %format!("{error:#}"), "Couldn't send a message of the identity layer");
}
}
}
@@ -499,6 +574,29 @@ impl Shared {
self.prove(&address, ME_LABEL, &proof);
Ok(())
}
+ // What the address's build speaks, noted first (`Book::speaks`), and logged
+ // where we share no version; from a node proven there, it goes on up too
+ // (`swarm::speaking`). From an address no node proved itself at, it ends here:
+ // the WHO or ME it goes ahead of follows.
+ ParsedMessage::Speaks(speaks) => {
+ self.other_versions
+ .speaks(&address, speaks.oldest, speaks.newest);
+ let node = {
+ let mut book = self.lock();
+ book.speaks(&address, speaks.oldest, speaks.newest);
+ book.heard(&address, Instant::now())
+ };
+ let Some(node) = node else {
+ self.traffic
+ .received(&address, MessageKind::Speaks, payload_len);
+ return Ok(());
+ };
+ let message = ParsedMessage::Speaks(speaks);
+ self.traffic
+ .log
+ .note("received", &node, Some(&address), &message, payload_len);
+ handler(node, message, payload_len).await
+ }
message => {
// Heard from: a node that talks isn't the one evicted when strangers'
// proofs fill the book (`Book::bind`).
@@ -774,7 +872,13 @@ impl Transport for IdentityTransport {
/// For the address `destination` picks (a send there is refused until `peer`
/// proved itself there, `send_encoded`). Only a look at the choice: the send makes
/// it (encoding again for another path if it changes).
- fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
+ /// In the wire version the address speaks, whatever `_version` asks (`Paths::encode`).
+ fn encode_in(
+ &self,
+ peer: &str,
+ message: &ParsedMessage,
+ _version: u8,
+ ) -> Result<EncodedMessage> {
let Some(address) = self.shared.destination(peer, false) else {
bail!("no address known for {peer} on this node's transports");
};
@@ -855,6 +959,42 @@ mod tests {
assert_eq!(book.at.len(), MAX_ENTRIES);
}
+ /// An address is written what a SPEAKS from there said, and so is every address of
+ /// the node proven there; the oldest version this build speaks until one says.
+ #[test]
+ fn each_address_is_written_what_its_node_speaks() {
+ let now = Instant::now();
+ let (oldest, newest) = (wire::OLDEST, wire::VERSION);
+ let mut book = Book::default();
+ assert_eq!(book.version_at("s:x"), oldest);
+ // Before a proof, for the address alone.
+ book.speaks("s:x", oldest, newest + 3);
+ assert_eq!(book.version_at("s:x"), newest);
+ book.bind("s:x", "n1", now);
+ // Another address of the node is written the same: one build.
+ book.bind("t:x", "n1", now);
+ assert_eq!(book.version_at("t:x"), newest);
+ // Moved: the old address is forgotten, the new one written the same.
+ book.bind("t:y", "n1", now);
+ assert!(!book.spoken.contains_key("t:x"));
+ assert_eq!(book.version_at("t:y"), newest);
+ // Speaking none of this build's versions: the oldest, at each of its addresses.
+ book.speaks("s:x", newest + 1, newest + 2);
+ assert!(book.spoken.is_empty());
+ assert_eq!(book.version_at("t:y"), oldest);
+
+ // Full: addresses no node proved itself at go first.
+ let mut book = Book::default();
+ book.bind("s:member", "member", now);
+ for i in 0..MAX_ENTRIES {
+ book.speaks(&format!("s:{i}"), oldest, newest);
+ }
+ assert_eq!(book.spoken.len(), MAX_ENTRIES);
+ book.speaks("s:member", oldest, newest);
+ assert_eq!(book.spoken.len(), 1);
+ assert_eq!(book.version_at("s:member"), newest);
+ }
+
#[test]
fn one_address_per_node_and_scheme_and_one_node_per_address() {
let now = Instant::now();
diff --git a/src/transport/libp2p/mod.rs b/src/transport/libp2p/mod.rs
index 2290dea..24f5f34 100644
--- a/src/transport/libp2p/mod.rs
+++ b/src/transport/libp2p/mod.rs
@@ -572,8 +572,13 @@ impl Transport for Libp2pTransport {
}
}
- fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- wire::encoded(peer, message)
+ fn encode_in(
+ &self,
+ peer: &str,
+ message: &ParsedMessage,
+ version: u8,
+ ) -> Result<EncodedMessage> {
+ wire::encoded_in(peer, message, version)
}
async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> {
diff --git a/src/transport/mod.rs b/src/transport/mod.rs
index 7bc6ae0..ee63988 100644
--- a/src/transport/mod.rs
+++ b/src/transport/mod.rs
@@ -261,8 +261,9 @@ pub(crate) enum ParsedMessage {
/// SPEAKS: the wire versions the sender speaks, the oldest and the newest, and the
/// newest revision of the federation it knows, by ID, where it is built with the
-/// federation. Sent with every handshake, in a layout no build changes (`wire::SPEAKS`):
-/// a build that doesn't know it ignores it, as a message of another wire version.
+/// federation. Sent with every handshake, and ahead of every WHO and ME
+/// (`transport::identity`), in a layout no build changes (`wire::SPEAKS`): a build that
+/// doesn't know it ignores it, as a message of another wire version.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct Speaks {
pub(crate) oldest: u8,
@@ -1198,7 +1199,18 @@ pub(crate) trait Transport: Send + Sync {
/// identity, as `local_id` gives it on that node.
async fn resolve_seed(&self, seed: &str) -> Result<String>;
- fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage>;
+ /// `message` to `peer` in wire `version`, one this build speaks (`wire::OLDEST..=
+ /// wire::VERSION`). The identity layer writes to each node in the newest version both
+ /// speak, whatever is asked (`Paths::version_at`).
+ fn encode_in(&self, peer: &str, message: &ParsedMessage, version: u8)
+ -> Result<EncodedMessage>;
+
+ /// `message` to `peer` in the newest wire version; under the identity layer, in the one
+ /// `peer` speaks.
+ fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
+ self.encode_in(peer, message, wire::VERSION)
+ }
+
async fn send_encoded(&self, message: &EncodedMessage) -> Result<()>;
/// Sends an outbound message payload to a targeted peer on a specific topic.
diff --git a/src/transport/mycelium/mod.rs b/src/transport/mycelium/mod.rs
index f1dbe40..d5d9271 100644
--- a/src/transport/mycelium/mod.rs
+++ b/src/transport/mycelium/mod.rs
@@ -355,8 +355,13 @@ impl Transport for MyceliumTransport {
self.lookup_pubkey(&ip).await
}
- fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- encode_message(peer, message)
+ fn encode_in(
+ &self,
+ peer: &str,
+ message: &ParsedMessage,
+ version: u8,
+ ) -> Result<EncodedMessage> {
+ encode_message(peer, message, version)
}
async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> {
@@ -573,9 +578,13 @@ async fn read_capped(
/// A message as the Mycelium REST API takes it (the JSON body of an outbound message).
/// Also the simulator's wire format (`transport::sim`), so tests go through the same
-/// encoding.
-pub(crate) fn encode_message(peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- let (outbound, payload_len) = format_message(peer, message)?;
+/// encoding. Its payload is in wire `version`.
+pub(crate) fn encode_message(
+ peer: &str,
+ message: &ParsedMessage,
+ version: u8,
+) -> Result<EncodedMessage> {
+ let (outbound, payload_len) = format_message(peer, message, version)?;
Ok(EncodedMessage {
peer: peer.to_owned(),
address: peer.to_owned(),
@@ -645,12 +654,13 @@ fn parse_message(message: &InboundMessage) -> Result<(String, ParsedMessage), Tr
Ok((sender, parsed))
}
-/// The message for Mycelium's API, and the length of its payload.
+/// The message for Mycelium's API, its payload in wire `version`, and the payload's length.
fn format_message(
recipient: &str,
message: &ParsedMessage,
+ version: u8,
) -> Result<(OutboundMessage, usize), TransportError> {
- let payload_bytes = wire::encode(message)?;
+ let payload_bytes = wire::encode_in(message, version)?;
let outbound = OutboundMessage {
dst: Destination {
pk: recipient.to_owned(),
@@ -797,7 +807,7 @@ mod tests {
#[test]
fn every_kind_round_trips_through_the_api_body() {
for message in wire::samples() {
- let encoded = encode_message("peer", &message).unwrap();
+ let encoded = encode_message("peer", &message, wire::VERSION).unwrap();
assert_eq!(encoded.kind, message.kind());
let decoded = decode_message(&encoded.body).unwrap();
assert_eq!(
diff --git a/src/transport/paths.rs b/src/transport/paths.rs
index f425216..e2d9ec9 100644
--- a/src/transport/paths.rs
+++ b/src/transport/paths.rs
@@ -58,7 +58,7 @@ use crate::clock::{Clock, UnixMillis};
use crate::locks::MutexExt;
use crate::transport::stats::{PathCounts, PathInfo, PathsStats, TransportStats};
use crate::transport::traffic::Traffic;
-use crate::transport::{Connection, EncodedMessage, ParsedMessage, Transport};
+use crate::transport::{Connection, EncodedMessage, ParsedMessage, Transport, wire};
/// Whether to drop what comes from or goes to an address (the debug firewall,
/// `transport::firewall`; the identity layer knows which node is where).
@@ -67,6 +67,12 @@ pub(crate) trait Gate: Send + Sync {
fn blocks_outbound(&self, address: &str) -> bool;
}
+/// The wire version to write at an address (`transport::wire`): the newest the node there
+/// and this build both speak, as the identity layer heard from it.
+pub(crate) trait Versions: Send + Sync {
+ fn version_at(&self, address: &str) -> u8;
+}
+
/// Called with the sender's address, the message and its `payload_len`.
pub(crate) type PathHandler = dyn Fn(String, ParsedMessage, usize) -> std::pin::Pin<Box<dyn Future<Output = Result<()>> + Send>>
+ Send
@@ -547,6 +553,8 @@ pub(crate) struct Paths {
traffic: Option<Arc<Traffic>>,
/// The debug firewall, if any.
gate: Option<Weak<dyn Gate>>,
+ /// What each address speaks; without, every address is written `wire::OLDEST`.
+ versions: Option<Weak<dyn Versions>>,
/// The node's wall clock, which its PONGs say the time by.
clock: Arc<Clock>,
/// A std mutex: never held across an await.
@@ -555,12 +563,14 @@ pub(crate) struct Paths {
impl Paths {
/// Over `transports`, by scheme, in order of preference; its own PINGs and PONGs
- /// counted in `traffic`, the firewall through `gate`, PONGs stamped by `clock`;
- /// paths pinged every `keepalive` at least (`Table::probe_every`).
+ /// counted in `traffic`, the firewall through `gate`, each address written the wire
+ /// version `versions` says, PONGs stamped by `clock`; paths pinged every `keepalive`
+ /// at least (`Table::probe_every`).
pub(crate) fn new(
transports: Vec<(&'static str, Arc<dyn Transport>)>,
traffic: Option<Arc<Traffic>>,
gate: Option<Weak<dyn Gate>>,
+ versions: Option<Weak<dyn Versions>>,
clock: Arc<Clock>,
keepalive: Duration,
) -> Self {
@@ -571,6 +581,7 @@ impl Paths {
.collect(),
traffic,
gate,
+ versions,
clock,
table: Mutex::new(Table::new(keepalive)),
}
@@ -580,6 +591,15 @@ impl Paths {
self.gate.as_ref()?.upgrade()
}
+ /// The wire version to write at `address` (`Versions`): the oldest this build speaks,
+ /// until its node says what it speaks.
+ pub(crate) fn version_at(&self, address: &str) -> u8 {
+ self.versions
+ .as_ref()
+ .and_then(Weak::upgrade)
+ .map_or(wire::OLDEST, |versions| versions.version_at(address))
+ }
+
/// The transport `address` is on, and the transport identity at it
/// (`Transport::locate`, which notes where that is if the address says); None if no
/// transport of ours has its scheme, or it isn't one of that transport's.
@@ -825,13 +845,16 @@ impl Paths {
Err(anyhow!("{}", failures.join("; ")))
}
- /// `message` encoded for `address`: `peer` and `address` of the result are
- /// `address`, and it keeps the message (to encode it for another address).
+ /// `message` encoded for `address`, in the wire version it speaks (`version_at`):
+ /// `peer` and `address` of the result are `address`, and it keeps the message (to
+ /// encode it for another address).
pub(crate) fn encode(&self, address: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
let (link, target) = self
.split(address)
.with_context(|| format!("no transport for {address}"))?;
- let mut encoded = link.inner.encode(&target, message)?;
+ let mut encoded = link
+ .inner
+ .encode_in(&target, message, self.version_at(address))?;
encoded.peer = address.to_owned();
encoded.address = address.to_owned();
encoded.message = Some(Arc::new(message.clone()));
@@ -1195,6 +1218,7 @@ mod tests {
],
None,
None,
+ None,
Arc::default(),
Duration::ZERO,
))
@@ -1608,6 +1632,7 @@ mod tests {
vec![("sim", net.transport("b") as Arc<dyn Transport>)],
None,
Some(Arc::downgrade(&gate)),
+ None,
Arc::default(),
Duration::ZERO,
));
@@ -1615,6 +1640,7 @@ mod tests {
vec![("sim", net.transport("a") as Arc<dyn Transport>)],
None,
None,
+ None,
Arc::default(),
Duration::ZERO,
));
@@ -1673,8 +1699,13 @@ mod tests {
async fn resolve_seed(&self, seed: &str) -> Result<String> {
Ok(seed.to_owned())
}
- fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- (crate::transport::sim::WIRE.encode)(peer, message)
+ fn encode_in(
+ &self,
+ peer: &str,
+ message: &ParsedMessage,
+ version: u8,
+ ) -> Result<EncodedMessage> {
+ (crate::transport::sim::WIRE.encode)(peer, message, version)
}
async fn send_encoded(&self, _message: &EncodedMessage) -> Result<()> {
Err(anyhow!("broken"))
@@ -1713,6 +1744,7 @@ mod tests {
],
None,
None,
+ None,
Arc::default(),
Duration::ZERO,
);
@@ -1720,6 +1752,7 @@ mod tests {
vec![("sim", net.transport("b") as Arc<dyn Transport>)],
None,
None,
+ None,
Arc::default(),
Duration::ZERO,
));
diff --git a/src/transport/sim.rs b/src/transport/sim.rs
index 40a1f7d..5d55e32 100644
--- a/src/transport/sim.rs
+++ b/src/transport/sim.rs
@@ -327,7 +327,7 @@ impl SimNet {
/// net. For messages a real node wouldn't send at that moment, e.g. one sent long
/// ago and delivered late.
pub(crate) async fn inject(&self, from: &str, to: &str, message: &ParsedMessage) -> Result<()> {
- let encoded = (self.codec.encode)(to, message)?;
+ let encoded = (self.codec.encode)(to, message, super::wire::VERSION)?;
let (handled_tx, handled_rx) = oneshot::channel();
let delivery = Delivery {
from: from.to_owned(),
@@ -368,7 +368,7 @@ pub(crate) struct SimTransport {
/// around it (Mycelium's: `mycelium::CODEC`), so a whole swarm runs on that envelope.
#[derive(Clone, Copy)]
pub(crate) struct Codec {
- pub(crate) encode: fn(&str, &ParsedMessage) -> Result<EncodedMessage>,
+ pub(crate) encode: fn(&str, &ParsedMessage, u8) -> Result<EncodedMessage>,
pub(crate) decode: fn(&[u8]) -> Result<ParsedMessage>,
}
@@ -378,8 +378,8 @@ pub(crate) const WIRE: Codec = Codec {
decode: wire_decode,
};
-fn wire_encode(peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- super::wire::encoded(peer, message)
+fn wire_encode(peer: &str, message: &ParsedMessage, version: u8) -> Result<EncodedMessage> {
+ super::wire::encoded_in(peer, message, version)
}
fn wire_decode(body: &[u8]) -> Result<ParsedMessage> {
@@ -410,8 +410,13 @@ impl Transport for SimTransport {
}
}
- fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- (self.net.codec.encode)(peer, message)
+ fn encode_in(
+ &self,
+ peer: &str,
+ message: &ParsedMessage,
+ version: u8,
+ ) -> Result<EncodedMessage> {
+ (self.net.codec.encode)(peer, message, version)
}
/// Always accepted: the net loses messages silently, like Mycelium once it has
diff --git a/src/transport/tor.rs b/src/transport/tor.rs
index c194330..a98e1c4 100644
--- a/src/transport/tor.rs
+++ b/src/transport/tor.rs
@@ -658,8 +658,13 @@ impl Transport for TorTransport {
Ok(location)
}
- fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
- wire::encoded(peer, message)
+ fn encode_in(
+ &self,
+ peer: &str,
+ message: &ParsedMessage,
+ version: u8,
+ ) -> Result<EncodedMessage> {
+ wire::encoded_in(peer, message, version)
}
async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> {
diff --git a/src/transport/traffic.rs b/src/transport/traffic.rs
index 89217c3..33903ca 100644
--- a/src/transport/traffic.rs
+++ b/src/transport/traffic.rs
@@ -537,6 +537,15 @@ impl Transport for CountingTransport {
self.inner.resolve_seed(seed).await
}
+ fn encode_in(
+ &self,
+ peer: &str,
+ message: &ParsedMessage,
+ version: u8,
+ ) -> Result<EncodedMessage> {
+ self.inner.encode_in(peer, message, version)
+ }
+
fn encode(&self, peer: &str, message: &ParsedMessage) -> Result<EncodedMessage> {
self.inner.encode(peer, message)
}
diff --git a/src/transport/wire.rs b/src/transport/wire.rs
index 0b661dd..56883e8 100644
--- a/src/transport/wire.rs
+++ b/src/transport/wire.rs
@@ -3,15 +3,26 @@
//! arrays, fields in declaration order). The codecs below make hashes, IDs and bytes
//! compact (MessagePack binaries, not arrays of numbers) and check their lengths.
//!
-//! The wire format may change freely until the first release: there is no compatibility
-//! with older nodes to keep. But every change bumps `VERSION`, so nodes on different
-//! builds tell each other apart instead of misreading each other's messages. The
+//! Every change to a message type or a codec is a new version (`VERSION`), and a build
+//! speaks several side by side, `OLDEST` to `VERSION`: it reads each, and writes to each
+//! node the newest both speak (`choose`), as the node's SPEAKS says, or `OLDEST` until
+//! then (`transport::identity`). So releases whose versions meet roll out node by node:
+//! a release adds a version and keeps the older ones, and a later one leaves out the
+//! oldest. The messages of an older version keep their own types, frozen, encoded and
+//! decoded here beside the newest's (`golden_bytes` keeps the bytes of each). The
//! transport's topic says which application a message is for, not which version.
//!
-//! One message stands outside the versions: SPEAKS (`SPEAKS`), its first byte the
-//! version none is, laid out as no build changes, saying which versions a node speaks
-//! and the newest revision of the federation it knows. A build from before it reads it
-//! as a message of another version, and ignores it.
+//! Two kinds of messages stand outside the versions, as they are what lets two builds
+//! learn what the other speaks:
+//!
+//! * SPEAKS (`SPEAKS`), its first byte the version none is, laid out as no build changes,
+//! saying which versions a node speaks and the newest revision of the federation it
+//! knows. A build from before it reads it as a message of another version, and
+//! ignores it.
+//! * WHO and ME, the proofs of who is at an address (`PROOFS`), laid out in every
+//! version as in the first: a build reads them whatever version they say they are in,
+//! from that first on, so that two nodes prove themselves to each other before they
+//! know what the other speaks.
use thiserror::Error;
@@ -31,8 +42,14 @@ pub(crate) const OLDEST: u8 = VERSION;
/// those and leaves whatever follows, which a later build may add.
pub(crate) const SPEAKS: u8 = 0;
-// Whatever this build's versions, a SPEAKS reads as one.
-const _: () = assert!(SPEAKS < OLDEST && OLDEST <= VERSION);
+/// The first version with SPEAKS, as WHO and ME are laid out in every version: their
+/// tags and fields never change (a change would be a new kind of message), and a build
+/// reads them in any version from this one on, older or newer than its own.
+pub(crate) const PROOFS: u8 = 52;
+
+// Whatever this build's versions, a SPEAKS reads as one, and it reads the proofs of
+// every build that speaks one of its versions.
+const _: () = assert!(SPEAKS < PROOFS && PROOFS <= OLDEST && OLDEST <= VERSION);
/// What the second byte of a payload says.
fn tag(kind: MessageKind) -> u8 {
@@ -78,11 +95,15 @@ pub(crate) enum WireError {
Decode(#[from] rmp_serde::decode::Error),
}
-/// `message` to `peer`, encoded as it goes on the wire alone (no envelope: libp2p,
-/// Tor, the simulator); `peer` is its address too.
+/// `message` to `peer`, encoded in wire `version` as it goes on the wire alone (no
+/// envelope: libp2p, Tor, the simulator); `peer` is its address too.
#[cfg_attr(not(any(feature = "libp2p", feature = "tor", test)), allow(dead_code))]
-pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result<EncodedMessage> {
- let body = encode(message)?;
+pub(crate) fn encoded_in(
+ peer: &str,
+ message: &ParsedMessage,
+ version: u8,
+) -> anyhow::Result<EncodedMessage> {
+ let body = encode_in(message, version)?;
Ok(EncodedMessage {
peer: peer.to_owned(),
address: peer.to_owned(),
@@ -93,7 +114,31 @@ pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result<Enc
})
}
-/// The payload of `message`.
+/// The newest wire version a peer speaking `oldest` to `newest` (its SPEAKS) and this
+/// build both speak: what this build writes to it. None where they share none.
+pub(crate) fn choose(oldest: u8, newest: u8) -> Option<u8> {
+ let (low, high) = (oldest.max(OLDEST), newest.min(VERSION));
+ (low <= high).then_some(high)
+}
+
+/// The payload of `message` in wire `version`, one of `OLDEST..=VERSION`: of an older one,
+/// as a build speaking only that one reads it. A SPEAKS is laid out alike in every one,
+/// and so are WHO and ME (`PROOFS`), but for the version they say.
+pub(crate) fn encode_in(message: &ParsedMessage, version: u8) -> Result<Vec<u8>, WireError> {
+ if matches!(message, ParsedMessage::Who(_) | ParsedMessage::Me(_)) && version >= PROOFS {
+ let mut payload = encode(message)?;
+ payload[0] = version;
+ return Ok(payload);
+ }
+ // Each older version this build still speaks is encoded here, from its own types,
+ // once its messages differ from the newest's; this build's are the newest's alone.
+ if version != VERSION && !matches!(message, ParsedMessage::Speaks(_)) {
+ return Err(WireError::Version(version));
+ }
+ encode(message)
+}
+
+/// The payload of `message`, in the newest wire version.
pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> {
if let ParsedMessage::Speaks(speaks) = message {
let mut payload = vec![SPEAKS, speaks.oldest, speaks.newest];
@@ -134,16 +179,33 @@ pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> {
/// (`OtherVersions`).
const MAX_OTHER_VERSIONS: usize = 1024;
-/// Senders whose messages are of another wire version (`WireError::Version`): each
-/// logged once, at warn level, so two builds that don't speak to each other say so, and
-/// which of them to upgrade, instead of silently retrying their HELLOs for good. Bounded:
-/// anyone can send us anything.
+/// Senders whose messages are of another wire version (`WireError::Version`), or whose
+/// SPEAKS names none this build speaks: each logged once, at warn level, so two builds
+/// that don't speak to each other say so, and which of them to upgrade, instead of
+/// silently retrying their HELLOs for good. Bounded: anyone can send us anything.
#[derive(Default)]
pub(crate) struct OtherVersions(std::sync::Mutex<std::collections::HashSet<String>>);
impl OtherVersions {
/// A message from `sender` (on the transport `scheme`) was of `version`.
pub(crate) fn note(&self, scheme: &'static str, sender: &str, version: u8) {
+ self.warn(
+ &super::Addressed(scheme, sender).to_string(),
+ version,
+ version,
+ );
+ }
+
+ /// A SPEAKS from `address` (as paths keep it) said its build speaks wire versions
+ /// `oldest` to `newest`: logged where this build speaks none of them, as a message of
+ /// another version is. The proofs between the two still go (`PROOFS`), nothing else.
+ pub(crate) fn speaks(&self, address: &str, oldest: u8, newest: u8) {
+ if choose(oldest, newest).is_none() {
+ self.warn(address, oldest, newest);
+ }
+ }
+
+ fn warn(&self, from: &str, oldest: u8, newest: u8) {
let mut known = self
.0
.lock()
@@ -151,36 +213,41 @@ impl OtherVersions {
if known.len() >= MAX_OTHER_VERSIONS {
known.clear();
}
- if known.insert(sender.to_owned()) {
+ if known.insert(from.to_owned()) {
tracing::warn!(
- from = %super::Addressed(scheme, &sender),
- version,
+ from = %from,
+ version = newest,
oldest = OLDEST,
newest = VERSION,
"Peer speaks another wire version: {}; ignoring its messages",
- other_build(version)
+ other_build(oldest, newest)
);
}
}
}
-/// Of a peer whose messages are of wire `version`, which this build doesn't speak, what
-/// an operator does: upgrade this node where the peer's build is newer, the peer's where
-/// it is older.
-fn other_build(version: u8) -> String {
- let ours = if OLDEST == VERSION {
- VERSION.to_string()
- } else {
- format!("{OLDEST} to {VERSION}")
+/// Of a peer speaking wire versions `oldest` to `newest`, none of which this build
+/// speaks, what an operator does: upgrade this node where the peer's build is newer, the
+/// peer's where it is older.
+fn other_build(oldest: u8, newest: u8) -> String {
+ let versions = |oldest: u8, newest: u8| {
+ if oldest == newest {
+ format!("{oldest}")
+ } else {
+ format!("{oldest} to {newest}")
+ }
};
- if version > VERSION {
+ let (theirs, ours) = (versions(oldest, newest), versions(OLDEST, VERSION));
+ let plural = if oldest == newest { "" } else { "s" };
+ if oldest > VERSION {
format!(
- "a newer build (wire version {version}; this one speaks {ours}): upgrade this node \
- to talk with it"
+ "a newer build (wire version{plural} {theirs}; this one speaks {ours}): upgrade this \
+ node to talk with it"
)
} else {
format!(
- "an older build (wire version {version}; this one speaks {ours}): upgrade that node"
+ "an older build (wire version{plural} {theirs}; this one speaks {ours}): upgrade that \
+ node"
)
}
}
@@ -197,11 +264,14 @@ pub(crate) fn decode(payload: &[u8]) -> Result<ParsedMessage, WireError> {
revision: rest.get(..32).map(|id| id.try_into().expect("32 bytes")),
}));
}
- if !(OLDEST..=VERSION).contains(&version) {
- return Err(WireError::Version(version));
- }
let (&kind, body) = payload.split_first().ok_or(WireError::Empty)?;
let is = |wanted: MessageKind| kind == tag(wanted);
+ let proof = version >= PROOFS && (is(MessageKind::Who) || is(MessageKind::Me));
+ // Each older version this build still speaks is decoded here, into its own types,
+ // once its messages differ from the newest's; this build's are the newest's alone.
+ if !(OLDEST..=VERSION).contains(&version) && !proof {
+ return Err(WireError::Version(version));
+ }
Ok(if is(MessageKind::Hello) {
ParsedMessage::Hello(rmp_serde::from_slice(body)?)
} else if is(MessageKind::HelloReply) {
@@ -639,18 +709,76 @@ mod tests {
assert!(other.0.lock().unwrap().len() <= MAX_OTHER_VERSIONS);
}
+ /// A peer is written to in the newest version both speak, a version this build doesn't
+ /// speak never; in the newest one alone, while that is all it speaks.
+ #[test]
+ fn the_newest_version_both_speak_is_written() {
+ assert_eq!(choose(OLDEST, VERSION), Some(VERSION));
+ assert_eq!(choose(1, VERSION + 5), Some(VERSION));
+ assert_eq!(choose(OLDEST - 2, OLDEST - 1), None);
+ assert_eq!(choose(VERSION + 1, VERSION + 2), None);
+ let ping = ParsedMessage::Ping(7);
+ assert_eq!(encode_in(&ping, VERSION).unwrap(), encode(&ping).unwrap());
+ assert!(encode_in(&ping, VERSION + 1).is_err());
+ assert!(encoded_in("p", &ping, VERSION).is_ok());
+ }
+
+ /// WHO and ME are laid out alike in every version, as in the first with SPEAKS, and
+ /// read in any version from it on, older or newer than this build's: two builds prove
+ /// themselves to each other before either knows what the other speaks. These bytes
+ /// never change.
+ #[test]
+ fn proofs_are_read_in_every_version() {
+ let proof = IdentityProof {
+ key: [8; 32],
+ signature: [9; 64],
+ };
+ let who = ParsedMessage::Who(proof.clone());
+ let fields = format!("92c420{}c440{}", "08".repeat(32), "09".repeat(64));
+ assert_eq!(
+ hex(&encode_in(&who, PROOFS).unwrap()),
+ format!("3406{fields}")
+ );
+ let me = ParsedMessage::Me(proof);
+ assert_eq!(
+ hex(&encode_in(&me, PROOFS).unwrap()),
+ format!("3407{fields}")
+ );
+ for version in [PROOFS, OLDEST, VERSION, VERSION + 1, u8::MAX] {
+ let who = encode_in(&who, version).unwrap();
+ assert_eq!(who[0], version);
+ assert!(
+ matches!(decode(&who), Ok(ParsedMessage::Who(_))),
+ "{version}"
+ );
+ let me = encode_in(&me, version).unwrap();
+ assert!(matches!(decode(&me), Ok(ParsedMessage::Me(_))), "{version}");
+ }
+ let mut before = encode(&who).unwrap();
+ before[0] = PROOFS - 1;
+ assert!(matches!(decode(&before), Err(WireError::Version(_))));
+ let mut ping = encode(&ParsedMessage::Ping(7)).unwrap();
+ ping[0] = VERSION + 1;
+ assert!(matches!(decode(&ping), Err(WireError::Version(_))));
+ }
+
/// The warning says which build is the older, the one to upgrade, and both versions.
#[test]
fn the_older_build_is_named() {
- let newer = other_build(VERSION + 1);
+ let newer = other_build(VERSION + 1, VERSION + 1);
assert!(newer.contains("upgrade this node"), "{newer}");
assert!(
newer.contains(&format!("wire version {}", VERSION + 1)),
"{newer}"
);
assert!(newer.contains(&format!("speaks {VERSION}")), "{newer}");
- let older = other_build(OLDEST - 1);
+ let older = other_build(OLDEST - 1, OLDEST - 1);
assert!(older.contains("upgrade that node"), "{older}");
+ let newer = other_build(VERSION + 1, VERSION + 3);
+ assert!(
+ newer.contains(&format!("wire versions {} to {}", VERSION + 1, VERSION + 3)),
+ "{newer}"
+ );
}
/// Every kind decodes to what was encoded: re-encoded, the same bytes.