aboutsummaryrefslogtreecommitdiffstats
path: root/src/transport/wire.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/transport/wire.rs')
-rw-r--r--src/transport/wire.rs285
1 files changed, 263 insertions, 22 deletions
diff --git a/src/transport/wire.rs b/src/transport/wire.rs
index 97bf6cc..56883e8 100644
--- a/src/transport/wire.rs
+++ b/src/transport/wire.rs
@@ -3,10 +3,26 @@
//! arrays, fields in declaration order). The codecs below make hashes, IDs and bytes
//! compact (MessagePack binaries, not arrays of numbers) and check their lengths.
//!
-//! The wire format may change freely until the first release: there is no compatibility
-//! with older nodes to keep. But every change bumps `VERSION`, so nodes on different
-//! builds tell each other apart instead of misreading each other's messages. The
+//! Every change to a message type or a codec is a new version (`VERSION`), and a build
+//! speaks several side by side, `OLDEST` to `VERSION`: it reads each, and writes to each
+//! node the newest both speak (`choose`), as the node's SPEAKS says, or `OLDEST` until
+//! then (`transport::identity`). So releases whose versions meet roll out node by node:
+//! a release adds a version and keeps the older ones, and a later one leaves out the
+//! oldest. The messages of an older version keep their own types, frozen, encoded and
+//! decoded here beside the newest's (`golden_bytes` keeps the bytes of each). The
//! transport's topic says which application a message is for, not which version.
+//!
+//! Two kinds of messages stand outside the versions, as they are what lets two builds
+//! learn what the other speaks:
+//!
+//! * SPEAKS (`SPEAKS`), its first byte the version none is, laid out as no build changes,
+//! saying which versions a node speaks and the newest revision of the federation it
+//! knows. A build from before it reads it as a message of another version, and
+//! ignores it.
+//! * WHO and ME, the proofs of who is at an address (`PROOFS`), laid out in every
+//! version as in the first: a build reads them whatever version they say they are in,
+//! from that first on, so that two nodes prove themselves to each other before they
+//! know what the other speaks.
use thiserror::Error;
@@ -14,7 +30,26 @@ use super::{EncodedMessage, MessageKind, ParsedMessage};
/// The wire format's version, the first byte of every payload. Bump it with every change
/// to a message type or a codec (`golden_bytes` fails then).
-pub(crate) const VERSION: u8 = 51;
+pub(crate) const VERSION: u8 = 52;
+
+/// The oldest version this build speaks: the messages of every version from it to
+/// `VERSION` it reads. Its SPEAKS says so.
+pub(crate) const OLDEST: u8 = VERSION;
+
+/// The first byte of a SPEAKS, which no version is. Its layout, never to change: that
+/// byte, the oldest version the sender speaks, the newest, then the 32 bytes of the ID
+/// of the newest revision of the federation it knows, where it knows one. A reader takes
+/// those and leaves whatever follows, which a later build may add.
+pub(crate) const SPEAKS: u8 = 0;
+
+/// The first version with SPEAKS, as WHO and ME are laid out in every version: their
+/// tags and fields never change (a change would be a new kind of message), and a build
+/// reads them in any version from this one on, older or newer than its own.
+pub(crate) const PROOFS: u8 = 52;
+
+// Whatever this build's versions, a SPEAKS reads as one, and it reads the proofs of
+// every build that speaks one of its versions.
+const _: () = assert!(SPEAKS < PROOFS && PROOFS <= OLDEST && OLDEST <= VERSION);
/// What the second byte of a payload says.
fn tag(kind: MessageKind) -> u8 {
@@ -37,6 +72,8 @@ fn tag(kind: MessageKind) -> u8 {
MessageKind::NeighbourReply => 17,
MessageKind::Disconnect => 18,
MessageKind::Refusals => 20,
+ // None: a SPEAKS has no second byte (`SPEAKS`).
+ MessageKind::Speaks => u8::MAX,
}
}
@@ -58,11 +95,15 @@ pub(crate) enum WireError {
Decode(#[from] rmp_serde::decode::Error),
}
-/// `message` to `peer`, encoded as it goes on the wire alone (no envelope: libp2p,
-/// Tor, the simulator); `peer` is its address too.
+/// `message` to `peer`, encoded in wire `version` as it goes on the wire alone (no
+/// envelope: libp2p, Tor, the simulator); `peer` is its address too.
#[cfg_attr(not(any(feature = "libp2p", feature = "tor", test)), allow(dead_code))]
-pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result<EncodedMessage> {
- let body = encode(message)?;
+pub(crate) fn encoded_in(
+ peer: &str,
+ message: &ParsedMessage,
+ version: u8,
+) -> anyhow::Result<EncodedMessage> {
+ let body = encode_in(message, version)?;
Ok(EncodedMessage {
peer: peer.to_owned(),
address: peer.to_owned(),
@@ -73,8 +114,37 @@ pub(crate) fn encoded(peer: &str, message: &ParsedMessage) -> anyhow::Result<Enc
})
}
-/// The payload of `message`.
+/// The newest wire version a peer speaking `oldest` to `newest` (its SPEAKS) and this
+/// build both speak: what this build writes to it. None where they share none.
+pub(crate) fn choose(oldest: u8, newest: u8) -> Option<u8> {
+ let (low, high) = (oldest.max(OLDEST), newest.min(VERSION));
+ (low <= high).then_some(high)
+}
+
+/// The payload of `message` in wire `version`, one of `OLDEST..=VERSION`: of an older one,
+/// as a build speaking only that one reads it. A SPEAKS is laid out alike in every one,
+/// and so are WHO and ME (`PROOFS`), but for the version they say.
+pub(crate) fn encode_in(message: &ParsedMessage, version: u8) -> Result<Vec<u8>, WireError> {
+ if matches!(message, ParsedMessage::Who(_) | ParsedMessage::Me(_)) && version >= PROOFS {
+ let mut payload = encode(message)?;
+ payload[0] = version;
+ return Ok(payload);
+ }
+ // Each older version this build still speaks is encoded here, from its own types,
+ // once its messages differ from the newest's; this build's are the newest's alone.
+ if version != VERSION && !matches!(message, ParsedMessage::Speaks(_)) {
+ return Err(WireError::Version(version));
+ }
+ encode(message)
+}
+
+/// The payload of `message`, in the newest wire version.
pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> {
+ if let ParsedMessage::Speaks(speaks) = message {
+ let mut payload = vec![SPEAKS, speaks.oldest, speaks.newest];
+ payload.extend(speaks.revision.iter().flatten());
+ return Ok(payload);
+ }
let mut payload = vec![VERSION, tag(message.kind())];
match message {
ParsedMessage::Hello(hello) => rmp_serde::encode::write(&mut payload, hello)?,
@@ -99,6 +169,7 @@ pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> {
rmp_serde::encode::write(&mut payload, answer)?
}
ParsedMessage::Refusals(message) => rmp_serde::encode::write(&mut payload, message)?,
+ ParsedMessage::Speaks(_) => unreachable!("encoded above"),
}
Ok(payload)
}
@@ -108,16 +179,33 @@ pub(crate) fn encode(message: &ParsedMessage) -> Result<Vec<u8>, WireError> {
/// (`OtherVersions`).
const MAX_OTHER_VERSIONS: usize = 1024;
-/// Senders whose messages are of another wire version (`WireError::Version`): each
-/// logged once, at warn level, so two builds that don't speak to each other say so,
-/// instead of silently retrying their HELLOs for good. Bounded: anyone can send us
-/// anything.
+/// Senders whose messages are of another wire version (`WireError::Version`), or whose
+/// SPEAKS names none this build speaks: each logged once, at warn level, so two builds
+/// that don't speak to each other say so, and which of them to upgrade, instead of
+/// silently retrying their HELLOs for good. Bounded: anyone can send us anything.
#[derive(Default)]
pub(crate) struct OtherVersions(std::sync::Mutex<std::collections::HashSet<String>>);
impl OtherVersions {
/// A message from `sender` (on the transport `scheme`) was of `version`.
pub(crate) fn note(&self, scheme: &'static str, sender: &str, version: u8) {
+ self.warn(
+ &super::Addressed(scheme, sender).to_string(),
+ version,
+ version,
+ );
+ }
+
+ /// A SPEAKS from `address` (as paths keep it) said its build speaks wire versions
+ /// `oldest` to `newest`: logged where this build speaks none of them, as a message of
+ /// another version is. The proofs between the two still go (`PROOFS`), nothing else.
+ pub(crate) fn speaks(&self, address: &str, oldest: u8, newest: u8) {
+ if choose(oldest, newest).is_none() {
+ self.warn(address, oldest, newest);
+ }
+ }
+
+ fn warn(&self, from: &str, oldest: u8, newest: u8) {
let mut known = self
.0
.lock()
@@ -125,24 +213,65 @@ impl OtherVersions {
if known.len() >= MAX_OTHER_VERSIONS {
known.clear();
}
- if known.insert(sender.to_owned()) {
+ if known.insert(from.to_owned()) {
tracing::warn!(
- from = %super::Addressed(scheme, &sender),
- version,
- ours = VERSION,
- "Peer speaks another wire version (another build); ignoring its messages"
+ from = %from,
+ version = newest,
+ oldest = OLDEST,
+ newest = VERSION,
+ "Peer speaks another wire version: {}; ignoring its messages",
+ other_build(oldest, newest)
);
}
}
}
+/// Of a peer speaking wire versions `oldest` to `newest`, none of which this build
+/// speaks, what an operator does: upgrade this node where the peer's build is newer, the
+/// peer's where it is older.
+fn other_build(oldest: u8, newest: u8) -> String {
+ let versions = |oldest: u8, newest: u8| {
+ if oldest == newest {
+ format!("{oldest}")
+ } else {
+ format!("{oldest} to {newest}")
+ }
+ };
+ let (theirs, ours) = (versions(oldest, newest), versions(OLDEST, VERSION));
+ let plural = if oldest == newest { "" } else { "s" };
+ if oldest > VERSION {
+ format!(
+ "a newer build (wire version{plural} {theirs}; this one speaks {ours}): upgrade this \
+ node to talk with it"
+ )
+ } else {
+ format!(
+ "an older build (wire version{plural} {theirs}; this one speaks {ours}): upgrade that \
+ node"
+ )
+ }
+}
+
pub(crate) fn decode(payload: &[u8]) -> Result<ParsedMessage, WireError> {
let (&version, payload) = payload.split_first().ok_or(WireError::Empty)?;
- if version != VERSION {
- return Err(WireError::Version(version));
+ if version == SPEAKS {
+ let [oldest, newest, rest @ ..] = payload else {
+ return Err(WireError::Empty);
+ };
+ return Ok(ParsedMessage::Speaks(super::Speaks {
+ oldest: *oldest,
+ newest: *newest,
+ revision: rest.get(..32).map(|id| id.try_into().expect("32 bytes")),
+ }));
}
let (&kind, body) = payload.split_first().ok_or(WireError::Empty)?;
let is = |wanted: MessageKind| kind == tag(wanted);
+ let proof = version >= PROOFS && (is(MessageKind::Who) || is(MessageKind::Me));
+ // Each older version this build still speaks is decoded here, into its own types,
+ // once its messages differ from the newest's; this build's are the newest's alone.
+ if !(OLDEST..=VERSION).contains(&version) && !proof {
+ return Err(WireError::Version(version));
+ }
Ok(if is(MessageKind::Hello) {
ParsedMessage::Hello(rmp_serde::from_slice(body)?)
} else if is(MessageKind::HelloReply) {
@@ -458,6 +587,7 @@ pub(crate) fn samples() -> Vec<ParsedMessage> {
run_id: [4; 16],
handshake: 7,
root: None,
+ party: None,
}),
ParsedMessage::HelloReply(HelloReply {
swarm_id: "swarm".into(),
@@ -473,6 +603,7 @@ pub(crate) fn samples() -> Vec<ParsedMessage> {
away_ms: Some(5),
sender_run_id: [6; 16],
root: None,
+ party: None,
}),
ParsedMessage::Sync(SyncEnvelope {
epoch: crate::transport::EpochTag::of(epoch),
@@ -578,6 +709,78 @@ mod tests {
assert!(other.0.lock().unwrap().len() <= MAX_OTHER_VERSIONS);
}
+ /// A peer is written to in the newest version both speak, a version this build doesn't
+ /// speak never; in the newest one alone, while that is all it speaks.
+ #[test]
+ fn the_newest_version_both_speak_is_written() {
+ assert_eq!(choose(OLDEST, VERSION), Some(VERSION));
+ assert_eq!(choose(1, VERSION + 5), Some(VERSION));
+ assert_eq!(choose(OLDEST - 2, OLDEST - 1), None);
+ assert_eq!(choose(VERSION + 1, VERSION + 2), None);
+ let ping = ParsedMessage::Ping(7);
+ assert_eq!(encode_in(&ping, VERSION).unwrap(), encode(&ping).unwrap());
+ assert!(encode_in(&ping, VERSION + 1).is_err());
+ assert!(encoded_in("p", &ping, VERSION).is_ok());
+ }
+
+ /// WHO and ME are laid out alike in every version, as in the first with SPEAKS, and
+ /// read in any version from it on, older or newer than this build's: two builds prove
+ /// themselves to each other before either knows what the other speaks. These bytes
+ /// never change.
+ #[test]
+ fn proofs_are_read_in_every_version() {
+ let proof = IdentityProof {
+ key: [8; 32],
+ signature: [9; 64],
+ };
+ let who = ParsedMessage::Who(proof.clone());
+ let fields = format!("92c420{}c440{}", "08".repeat(32), "09".repeat(64));
+ assert_eq!(
+ hex(&encode_in(&who, PROOFS).unwrap()),
+ format!("3406{fields}")
+ );
+ let me = ParsedMessage::Me(proof);
+ assert_eq!(
+ hex(&encode_in(&me, PROOFS).unwrap()),
+ format!("3407{fields}")
+ );
+ for version in [PROOFS, OLDEST, VERSION, VERSION + 1, u8::MAX] {
+ let who = encode_in(&who, version).unwrap();
+ assert_eq!(who[0], version);
+ assert!(
+ matches!(decode(&who), Ok(ParsedMessage::Who(_))),
+ "{version}"
+ );
+ let me = encode_in(&me, version).unwrap();
+ assert!(matches!(decode(&me), Ok(ParsedMessage::Me(_))), "{version}");
+ }
+ let mut before = encode(&who).unwrap();
+ before[0] = PROOFS - 1;
+ assert!(matches!(decode(&before), Err(WireError::Version(_))));
+ let mut ping = encode(&ParsedMessage::Ping(7)).unwrap();
+ ping[0] = VERSION + 1;
+ assert!(matches!(decode(&ping), Err(WireError::Version(_))));
+ }
+
+ /// The warning says which build is the older, the one to upgrade, and both versions.
+ #[test]
+ fn the_older_build_is_named() {
+ let newer = other_build(VERSION + 1, VERSION + 1);
+ assert!(newer.contains("upgrade this node"), "{newer}");
+ assert!(
+ newer.contains(&format!("wire version {}", VERSION + 1)),
+ "{newer}"
+ );
+ assert!(newer.contains(&format!("speaks {VERSION}")), "{newer}");
+ let older = other_build(OLDEST - 1, OLDEST - 1);
+ assert!(older.contains("upgrade that node"), "{older}");
+ let newer = other_build(VERSION + 1, VERSION + 3);
+ assert!(
+ newer.contains(&format!("wire versions {} to {}", VERSION + 1, VERSION + 3)),
+ "{newer}"
+ );
+ }
+
/// Every kind decodes to what was encoded: re-encoded, the same bytes.
#[test]
fn every_kind_round_trips() {
@@ -606,6 +809,7 @@ mod tests {
run_id: [4; 16],
handshake: 7,
root: None,
+ party: Some("tv".into()),
});
let ParsedMessage::Hello(decoded) = decode(&encode(&hello).unwrap()).unwrap() else {
panic!("not a HELLO");
@@ -614,6 +818,7 @@ mod tests {
assert_eq!((decoded.epoch, decoded.epoch_seq), (epoch, 1));
assert_eq!(decoded.auth, vec![1, 2, 3]);
assert_eq!(decoded.run_id, [4; 16]);
+ assert_eq!(decoded.party.as_deref(), Some("tv"));
assert_eq!(decoded.handshake, 7);
let reply = ParsedMessage::HelloReply(HelloReply {
@@ -630,6 +835,7 @@ mod tests {
away_ms: Some(5),
sender_run_id: [6; 16],
root: None,
+ party: None,
});
let ParsedMessage::HelloReply(decoded) = decode(&encode(&reply).unwrap()).unwrap() else {
panic!("not a HELLO_REPLY");
@@ -724,6 +930,39 @@ mod tests {
crate::transport::Hex(bytes).to_string()
}
+ /// A SPEAKS is laid out as no build changes: its version byte, the oldest and newest
+ /// version, the revision's ID where there is one; what follows is left.
+ #[test]
+ fn speaks_keeps_its_layout() {
+ let speaks = super::super::Speaks {
+ oldest: 50,
+ newest: 52,
+ revision: Some([7; 32]),
+ };
+ let bytes = encode(&ParsedMessage::Speaks(speaks)).unwrap();
+ assert_eq!(hex(&bytes), format!("003234{}", "07".repeat(32)));
+ let without = super::super::Speaks {
+ revision: None,
+ ..speaks
+ };
+ assert_eq!(
+ hex(&encode(&ParsedMessage::Speaks(without)).unwrap()),
+ "003234"
+ );
+ for (payload, read) in [
+ (bytes.clone(), speaks),
+ ([bytes.as_slice(), b"later"].concat(), speaks),
+ (vec![0, 50, 52], without),
+ (vec![0, 50, 52, 1, 2], without),
+ ] {
+ match decode(&payload) {
+ Ok(ParsedMessage::Speaks(decoded)) => assert_eq!(decoded, read),
+ other => panic!("{other:?}"),
+ }
+ }
+ assert!(decode(&[0, 50]).is_err());
+ }
+
/// The exact bytes of a few messages. If this fails after a change to a message type
/// or a codec, the change is a new wire format: bump `VERSION` and take the new bytes.
#[test]
@@ -746,11 +985,12 @@ mod tests {
run_id: [4; 16],
handshake: 7,
root: None,
+ party: None,
});
assert_eq!(
hex(&encode(&hello).unwrap()),
format!(
- "{v}0098a5737761726d{epoch_hex}01c403010203c3{}07c0",
+ "{v}0099a5737761726d{epoch_hex}01c403010203c3{}07c0c0",
run("04")
)
);
@@ -769,11 +1009,12 @@ mod tests {
away_ms: Some(5),
sender_run_id: [6; 16],
root: None,
+ party: None,
});
assert_eq!(
hex(&encode(&reply).unwrap()),
format!(
- "{v}019aa5737761726d9192a17091a161{epoch_hex}01c400c2{}05{}c0",
+ "{v}019ba5737761726d9192a17091a161{epoch_hex}01c400c2{}05{}c0c0",
run("04"),
run("06")
)