//! The identity layer: between the node and a transport whose identities are addresses //! (Mycelium's public keys), so that the rest of the node deals in node IDs //! (`crate::identity`) only, whatever transport reaches a peer. //! //! A node proves who it is to an address with WHO (which asks back) or ME (the answer): //! its public node key, and a signature over its own address and the recipient's //! (`signed`). The transport authenticates the sender's address, so a proof is good for //! that pair of addresses only: replayed from another address, or to another recipient, //! it fails. Once a node proved itself at an address, messages from there are its. //! //! * A message from an address no node proved itself at is dropped, and the address is //! asked WHO (at most every `ASK_INTERVAL`): after a restart, the first message from //! a peer brings its proof back, and the peer's node retries what was lost. //! * A seed (`resolve_seed`) is an address, or something the transport resolves to one; //! it is asked WHO, and its ME says which node is there. //! * Where other nodes are comes from the member list and HELLO_REPLYs //! (`learn_addresses`): hints, not proofs. A message to a node goes to the address it //! proved itself at; with only hints, those are asked WHO, and the send is refused //! until one answers (the node retries refused sends). //! //! Addresses are `:`, e.g. `mycelium:`. use std::collections::HashMap; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::Duration; use anyhow::{Result, anyhow, bail}; use async_trait::async_trait; use tokio::sync::oneshot; use tokio::time::Instant; use tracing::{debug, info}; use crate::identity::{self, NodeKey}; use crate::locks::MutexExt; use crate::transport::traffic::Traffic; use crate::transport::{ EncodedMessage, HandlerFn, IdentityProof, MessageHandler, MessageKind, ParsedMessage, Transport, }; pub(crate) const WHO_LABEL: &[u8] = b"continuum/who/1"; pub(crate) const ME_LABEL: &[u8] = b"continuum/me/1"; /// An address is asked WHO at most this often, unless a seed is being resolved. const ASK_INTERVAL: Duration = Duration::from_secs(5); /// How long resolving a seed waits for its ME. const RESOLVE_TIMEOUT: Duration = Duration::from_secs(10); /// Addresses kept per node from hints: about one per transport. const MAX_HINTS: usize = 8; /// Addresses with a proven node, nodes with hints, and addresses asked, each at most: /// anyone can send us proofs (keys are free), so the books are bounded. const MAX_ENTRIES: usize = 4096; /// What a node under the identity layer needs (`node::NodeConfig::identity`). #[derive(Clone)] pub(crate) struct IdentityConfig { pub(crate) key: Arc, /// The transport's address scheme, e.g. `mycelium`. pub(crate) scheme: &'static str, } /// The bytes a WHO (`label`: `WHO_LABEL`) or ME proof signs: the label, then the /// sender's and the recipient's address, each with its length. pub(crate) fn signed(label: &[u8], from: &str, to: &str) -> Vec { let mut bytes = label.to_vec(); for address in [from, to] { bytes.extend_from_slice(&(address.len() as u32).to_be_bytes()); bytes.extend_from_slice(address.as_bytes()); } bytes } pub(crate) struct IdentityTransport { shared: Arc, } struct Shared { inner: Arc, scheme: &'static str, key: Arc, node_id: String, /// WHO and ME are counted here, under the address (`traffic`). traffic: Arc, // A std mutex: every access is short and never spans an await. book: Mutex, } /// Who is where. Keys are transport identities (an address without its scheme). #[derive(Default)] struct Book { /// The node that proved itself at a transport identity, and when. at: HashMap, /// The transport identity each node last proved itself at. proven: HashMap, /// Where others say a node is. hints: HashMap>, /// When each transport identity was last asked WHO. asked: HashMap, /// Seeds being resolved: who waits for a ME from where. waiting: HashMap>>, } impl Book { /// `node` proved itself at `target`. Returns whether that is news. fn bind(&mut self, target: &str, node: &str, now: Instant) -> bool { let news = self.at.get(target).is_none_or(|(known, _)| known != node); // One address per node and one node per address: a node that moved, or an // address taken over by another node, leaves no stale entry behind. if let Some(previous) = self.proven.insert(node.to_owned(), target.to_owned()) && previous != target { self.at.remove(&previous); } if let Some((previous, _)) = self.at.insert(target.to_owned(), (node.to_owned(), now)) && previous != node && self.proven.get(&previous).is_some_and(|at| at == target) { self.proven.remove(&previous); } if self.at.len() > MAX_ENTRIES && let Some(oldest) = self .at .iter() .min_by_key(|(_, (_, at))| *at) .map(|(target, _)| target.clone()) && let Some((node, _)) = self.at.remove(&oldest) { self.proven.remove(&node); } for waiter in self.waiting.remove(target).into_iter().flatten() { let _ = waiter.send(node.to_owned()); } news } /// Whether to ask `target` WHO now (always, if `force`), noting that it was asked. fn ask(&mut self, target: &str, now: Instant, force: bool) -> bool { if !force && self .asked .get(target) .is_some_and(|at| now.duration_since(*at) < ASK_INTERVAL) { return false; } if self.asked.len() >= MAX_ENTRIES { self.asked .retain(|_, at| now.duration_since(*at) < ASK_INTERVAL); } self.asked.insert(target.to_owned(), now); true } } impl IdentityTransport { pub(crate) fn new( inner: Arc, config: IdentityConfig, traffic: Arc, ) -> Self { let node_id = config.key.id(); Self { shared: Arc::new(Shared { inner, scheme: config.scheme, key: config.key, node_id, traffic, book: Mutex::default(), }), } } } impl Shared { fn lock(&self) -> MutexGuard<'_, Book> { self.book.lock_or_recover() } fn address(&self, target: &str) -> String { format!("{}:{target}", self.scheme) } /// The transport identity in `address`, if it is one of this transport's. fn target_of<'a>(&self, address: &'a str) -> Option<&'a str> { address .strip_prefix(self.scheme) .and_then(|rest| rest.strip_prefix(':')) .filter(|target| self.inner.is_peer_id(target)) } fn proof(&self, label: &[u8], target: &str) -> IdentityProof { let message = signed( label, &self.address(self.inner.local_id()), &self.address(target), ); IdentityProof { key: self.key.public(), signature: self.key.sign(&message), } } /// Notes the node `proof` proves to be at `sender`; None if it proves nothing. fn prove(&self, sender: &str, label: &[u8], proof: &IdentityProof) -> Option { let message = signed( label, &self.address(sender), &self.address(self.inner.local_id()), ); if !identity::verify(&proof.key, &message, &proof.signature) { debug!(from = %self.address(sender), "Ignoring an identity proof that doesn't hold"); return None; } let node = identity::node_id_of(&proof.key); if node == self.node_id { debug!(from = %self.address(sender), "Ignoring a proof of our own node key"); return None; } if self.lock().bind(sender, &node, Instant::now()) { debug!(node = %node, address = %self.address(sender), "Node proved its address"); } Some(node) } /// Asks `target` WHO, unless it was asked recently (`force`: anyway). fn ask(self: &Arc, target: &str, force: bool) { if !self.lock().ask(target, Instant::now(), force) { return; } let shared = Arc::clone(self); let target = target.to_owned(); tokio::spawn(async move { shared.send_proof(MessageKind::Who, &target).await }); } /// Sends a WHO or ME (`kind`) to `target`. async fn send_proof(&self, kind: MessageKind, target: &str) { let message = if kind == MessageKind::Who { ParsedMessage::Who(self.proof(WHO_LABEL, target)) } else { ParsedMessage::Me(self.proof(ME_LABEL, target)) }; let encoded = match self.inner.encode(target, &message) { Ok(encoded) => encoded, Err(error) => { debug!(to = %target, error = %format!("{error:#}"), "Couldn't encode an identity proof"); return; } }; let address = self.address(target); match self.inner.send_encoded(&encoded).await { Ok(()) => self .traffic .sent(&address, encoded.kind, encoded.payload_len), Err(error) => { self.traffic .refused(&address, encoded.kind, encoded.payload_len); debug!(to = %address, error = %format!("{error:#}"), "Couldn't send an identity proof"); } } } async fn receive( self: Arc, sender: String, message: ParsedMessage, payload_len: usize, handler: &HandlerFn, ) -> Result<()> { match message { ParsedMessage::Who(proof) => { self.traffic .received(&self.address(&sender), MessageKind::Who, payload_len); if self.prove(&sender, WHO_LABEL, &proof).is_some() { let shared = Arc::clone(&self); tokio::spawn(async move { shared.send_proof(MessageKind::Me, &sender).await }); } Ok(()) } ParsedMessage::Me(proof) => { self.traffic .received(&self.address(&sender), MessageKind::Me, payload_len); self.prove(&sender, ME_LABEL, &proof); Ok(()) } message => { let node = self.lock().at.get(&sender).map(|(node, _)| node.clone()); match node { Some(node) => handler(node, message, payload_len).await, None => { debug!( from = %self.address(&sender), kind = ?message.kind(), "Dropping a message from an address no node proved itself at; asking who is there" ); self.ask(&sender, false); Ok(()) } } } } } } #[async_trait] impl Transport for IdentityTransport { fn local_id(&self) -> &str { &self.shared.node_id } fn is_peer_id(&self, id: &str) -> bool { identity::is_node_id(id) } fn delivery_limit(&self) -> Duration { self.shared.inner.delivery_limit() } fn local_addresses(&self) -> Vec { vec![self.shared.address(self.shared.inner.local_id())] } fn addresses_of(&self, peer: &str) -> Vec { let book = self.shared.lock(); match book.proven.get(peer) { Some(target) => vec![self.shared.address(target)], None => book .hints .get(peer) .into_iter() .flatten() .map(|target| self.shared.address(target)) .collect(), } } fn learn_addresses(&self, peer: &str, addresses: &[String]) { if peer == self.shared.node_id || !identity::is_node_id(peer) { return; } let local = self.shared.inner.local_id(); let mut targets: Vec = addresses .iter() .filter_map(|address| self.shared.target_of(address)) .filter(|target| *target != local) .map(str::to_owned) .collect(); targets.dedup(); targets.truncate(MAX_HINTS); if targets.is_empty() { return; } let mut book = self.shared.lock(); if book.hints.len() < MAX_ENTRIES || book.hints.contains_key(peer) { book.hints.insert(peer.to_owned(), targets); } } /// A node ID resolves to itself if this node knows where it is; anything else is /// an address (`:...`) or what the transport resolves to one, asked WHO. async fn resolve_seed(&self, seed: &str) -> Result { if identity::is_node_id(seed) { let book = self.shared.lock(); if book.proven.contains_key(seed) || book.hints.contains_key(seed) { return Ok(seed.to_owned()); } bail!("a node ID alone doesn't say where that node is; give its address"); } let target = match self.shared.target_of(seed) { Some(target) => target.to_owned(), None => self.shared.inner.resolve_seed(seed).await?, }; if target == self.shared.inner.local_id() { bail!("{seed} is this node"); } let receiver = { let mut book = self.shared.lock(); if let Some((node, _)) = book.at.get(&target) { return Ok(node.clone()); } let (sender, receiver) = oneshot::channel(); let waiting = book.waiting.entry(target.clone()).or_default(); waiting.retain(|waiter| !waiter.is_closed()); waiting.push(sender); receiver }; self.shared.ask(&target, true); match tokio::time::timeout(RESOLVE_TIMEOUT, receiver).await { Ok(Ok(node)) => { info!(seed = %seed, node = %node, "Seed is node"); Ok(node) } _ => Err(anyhow!( "no node answered at {} within {RESOLVE_TIMEOUT:?}", self.shared.address(&target) )), } } /// For the address `peer` proved itself at, or else one it is said to be at (a send /// there is refused until it proves itself, `send_encoded`). fn encode(&self, peer: &str, message: &ParsedMessage) -> Result { let target = { let book = self.shared.lock(); book.proven .get(peer) .or_else(|| book.hints.get(peer).and_then(|hints| hints.first())) .cloned() }; let Some(target) = target else { bail!("no address known for {peer}"); }; let mut encoded = self.shared.inner.encode(&target, message)?; encoded.peer = peer.to_owned(); Ok(encoded) } async fn send_encoded(&self, message: &EncodedMessage) -> Result<()> { let hints = { let book = self.shared.lock(); if book.proven.get(&message.peer) == Some(&message.address) { None } else { Some(book.hints.get(&message.peer).cloned().unwrap_or_default()) } }; if let Some(hints) = hints { for target in hints { self.shared.ask(&target, false); } bail!( "{} hasn't proven where it is yet; asked where it is said to be", message.peer ); } let mut inner = message.clone(); inner.peer = message.address.clone(); self.shared.inner.send_encoded(&inner).await } async fn start_consumer(&self, handler: MessageHandler) -> Result<()> { let handler: Arc = Arc::from(handler); let shared = Arc::clone(&self.shared); self.shared .inner .start_consumer(Box::new(move |sender, message, payload_len| { let shared = Arc::clone(&shared); let handler = Arc::clone(&handler); Box::pin(async move { shared .receive(sender, message, payload_len, &*handler) .await }) })) .await } async fn wait_for_exit(&self) -> std::io::Result { self.shared.inner.wait_for_exit().await } } #[cfg(test)] mod tests { use super::*; #[test] fn signed_bytes_say_both_addresses() { // A length before each address: "a:bc" to "d" isn't "a:b" to "cd". assert_ne!( signed(WHO_LABEL, "a:bc", "d"), signed(WHO_LABEL, "a:b", "cd") ); assert_ne!(signed(WHO_LABEL, "a", "b"), signed(ME_LABEL, "a", "b")); assert_ne!(signed(WHO_LABEL, "a", "b"), signed(WHO_LABEL, "b", "a")); } #[test] fn one_address_per_node_and_one_node_per_address() { let now = Instant::now(); let mut book = Book::default(); assert!(book.bind("x", "n1", now)); assert!(!book.bind("x", "n1", now)); // n1 moved to y: x is no longer n1's. assert!(book.bind("y", "n1", now)); assert!(!book.at.contains_key("x")); assert_eq!(book.proven["n1"], "y"); // n2 took over y: n1 is nowhere proven. assert!(book.bind("y", "n2", now)); assert!(!book.proven.contains_key("n1")); assert_eq!(book.at["y"].0, "n2"); } #[test] fn asking_is_rate_limited() { let now = Instant::now(); let mut book = Book::default(); assert!(book.ask("x", now, false)); assert!(!book.ask("x", now, false)); assert!(book.ask("x", now, true)); assert!(book.ask("x", now + ASK_INTERVAL, false)); } #[tokio::test] async fn a_waiting_seed_learns_the_node() { let mut book = Book::default(); let (sender, receiver) = oneshot::channel(); book.waiting.insert("x".into(), vec![sender]); book.bind("x", "n1", Instant::now()); assert_eq!(receiver.await.unwrap(), "n1"); } }