//! Configuration: built-in defaults, then configuration files (`--config`, in order), //! then command-line arguments. Every option can be set in each. //! //! An argument that replaces a value from a file is logged (info), as is a file that //! replaces one from another file (debug; warning between two includes of one file). //! Values can be imported instead of written out (`import`): imported values are never //! logged, only where they come from. //! //! All options are described once, in [`OPTIONS`]: the command line, the file keys and //! the conversion of text values all derive from it. The merged values are then read //! into [`Settings`] and checked. mod file; mod import; use std::collections::BTreeMap; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; use clap::parser::ValueSource; use clap::{Arg, ArgAction, ArgMatches, Command}; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; use tracing::{debug, info, warn}; use import::{Imported, Raw}; /// How an option's value is written as text (command line, text imports). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Kind { /// `true`/`false`; on the command line, present or not. Flag, Text, Integer, /// Seconds, or with units: `500ms`, `30s`, `1h30m` (`units::parse_duration`). Duration, /// Bytes, or with a unit: `64KiB` (`units::parse_size`). Size, /// Comma- or line-separated; repeatable on the command line. List, } #[derive(Debug)] pub(crate) struct OptionSpec { pub(crate) name: &'static str, pub(crate) kind: Kind, pub(crate) value_name: &'static str, /// For `--help` only; the effective defaults are in [`Settings`]. pub(crate) default: Option<&'static str>, pub(crate) help: &'static str, pub(crate) hidden: bool, /// Its value is never shown (logs, `--print-config`, `GET /admin/config`), however /// it was given. pub(crate) secret: bool, } impl OptionSpec { const fn secret(self) -> Self { Self { secret: true, ..self } } } const fn option( name: &'static str, kind: Kind, value_name: &'static str, default: Option<&'static str>, help: &'static str, ) -> OptionSpec { OptionSpec { name, kind, value_name, default, help, hidden: false, secret: false, } } /// What `--force` may override at start (`force`). pub(crate) const FORCE: &[&str] = &["swarmId", "badDocumentFile", "replaceDocumentFile"]; /// Every option. Names are the long option names and the configuration file keys. pub(crate) const OPTIONS: &[OptionSpec] = &[ option( "bind-addr", Kind::Text, "HOSTNAME:PORT", Some("127.0.0.1:3000"), "Address of the HTTP API.", ), option( "htpasswd", Kind::Text, "PATH", None, "Users of the HTTP API (htpasswd file with Argon2 hashes). Required.", ), option( "protocol-writers", Kind::List, "USER,[USER...]", None, "HTTP API users (from htpasswd) who may write protocol data under /store/_continuum (remake records, members, forget records), e.g. to repair a stuck remake. Such writes act on the whole swarm and are logged. Epoch metadata and the swarm ID are never writable.", ), option( "debug-api", Kind::Flag, "BOOL", Some("false"), "Enable the /debug endpoints (tests only).", ), option( "seeds", Kind::List, "STRING,[STRING...]", None, "How to reach a node of the swarm to join: its address (`mycelium:`, as GET /admin/stats lists it), a Mycelium public key, a Mycelium IPv6 address, or a host name with an AAAA record for one. A node keeps trying until it has joined.", ), option( "genesis", Kind::Flag, "BOOL", Some("false"), "Start a new swarm, as its genesis node, instead of joining one through seeds. Every start with it starts another new swarm: drop it once the swarm exists, and give the node seeds instead. Can't be combined with seeds. With a --state-file that remembers a swarm, it starts that swarm again instead (empty; nodes of it still running merge with this one).", ), option( "force", Kind::List, "NAME,[NAME...]", None, "Override checks that otherwise refuse to start (command line only: it is for one start): true for all, or names: swarmId (a document to start on is of another swarm than the state file remembers), badDocumentFile (an unusable document file: moved aside, start without it), replaceDocumentFile (recover-from while the document file holds a document).", ), option( "recover-from", Kind::Text, "PATH", None, "Start the swarm again from a backup (as GET /admin/document saves it): this node loads it, on the swarm and epoch it names, and is ready at once; other nodes join it. Members of the swarm still running merge with it, or it rejoins them if they moved on to a newer epoch (the backup is then dropped).", ), option( "remake", Kind::Text, "KIND[:PATH]", None, "With --genesis: make the new swarm's document as a remake of this kind would, instead of starting it empty: import:PATH (an Automerge document, e.g. a backup, without its history and protocol data) or replace:PATH (a JSON document, the store root); compact changes nothing.", ), option( "document-file", Kind::Text, "PATH", None, "This node's document on disk. Read at start: the node resumes with it, ready at once. Kept up to date while ready: changes are appended every --document-append, and the file is rewritten after a switch, a rejoin or a restart, and when the appended changes outgrew it.", ), option( "document-append", Kind::Duration, "DURATION", Some("5s"), "How often changes are appended to the document file.", ), option( "document-rewrite", Kind::Duration, "DURATION", Some("1h"), "Shortest time between rewrites of the document file because its appended changes outgrew its full save (a new epoch or a restart rewrites it at once).", ), option( "state-file", Kind::Text, "PATH", None, "File where this node remembers its swarm across restarts: the swarm ID and some members to rejoin through (not the document). Written once ready, then rarely. Without it, a restarted node knows only its configured seeds.", ), option( "node-key-file", Kind::Text, "PATH", None, "This node's key (Ed25519, 32 raw bytes): who the node is, whatever transport reaches it. Keep it with the node: a new key is a new node. Required.", ), option( "create-node-key", Kind::Flag, "BOOL", Some("false"), "Create a new node key at node-key-file if there is no file there (a new node). Without it, a missing key file is an error.", ), option( "swarm-key-file", Kind::Text, "PATH", None, "Pre-shared swarm key: only nodes that know it may join (raw bytes, at least 16; e.g. `head -c 32 /dev/urandom`). Without it, any node that can reach a member may join.", ), option( "alias", Kind::Text, "NAME", None, "A name for this node, shown next to its public key (/peers, metrics). Published in the swarm's member list; not unique, never an identity.", ), option( "keepalive", Kind::Duration, "DURATION", Some("1m"), "Keepalive interval; dormancy, resync and probe timings derive from it.", ), option( "neighbours", Kind::Integer, "COUNT", Some("12"), "Neighbours each node keeps (at least 2). Swarms of up to this many + 1 nodes are a full mesh; larger ones sync over several hops.", ), option( "electorate-window", Kind::Duration, "DURATION", Some("7d"), "Members that renewed their alive stamp within this time vote on remakes (a majority of them must agree). Matters on the node that coordinates a remake.", ), option( "blob-chunk", Kind::Size, "SIZE", Some("64KiB"), "Chunk size when serving large transfers to other nodes (e.g. an epoch's snapshot to a rejoining node), 1KiB to 4MiB.", ), option( "transports", Kind::List, "NAME,[NAME...]", Some("mycelium"), "The transports this node uses, in order of preference: mycelium, libp2p (in builds with the libp2p feature). A peer is reached on the first one both have; nodes on several carry the traffic between nodes that share none.", ), option( "libp2p-listen", Kind::List, "MULTIADDR,[MULTIADDR...]", None, "With the libp2p transport: where to listen, e.g. /ip4/0.0.0.0/tcp/9661 and /ip4/0.0.0.0/udp/9661/quic-v1. Without, the node only connects out. The addresses it listens on are published in the member list.", ), option( "libp2p-announce", Kind::List, "MULTIADDR,[MULTIADDR...]", None, "With the libp2p transport: the addresses to publish in the member list instead of those listened on, e.g. a public address forwarded to this node's listen port. Without, the listen addresses are published, loopback and link-local ones left out.", ), option( "libp2p-relay", Kind::Flag, "BOOL", Some("false"), "With the libp2p transport: relay for other nodes of the swarm that can't be dialled (bounded: 64 circuits, 8 per peer, each up to an hour and 1 GiB), and say so in the member list, where nodes that need a relay find it. Needs a listen address others can dial.", ), option( "libp2p-relays", Kind::List, "MULTIADDR,[MULTIADDR...]", None, "With the libp2p transport: relays to be reachable through always (multiaddrs, with or without the relay's peer ID). Without, a node that can't be dialled picks members that relay (libp2p-relay) by itself. Never public ones.", ), option( "libp2p-mdns", Kind::Flag, "BOOL", Some("false"), "With the libp2p transport: find nodes on the LAN (multicast DNS, IPv4). Those that speak continuum are seeds while joining, so nodes on one LAN find each other without configured seeds; a swarm key keeps a new node from joining another swarm on the same LAN.", ), option( "libp2p-public", Kind::Flag, "BOOL", Some("false"), "With the libp2p transport: use public libp2p infrastructure: join the IPFS DHT (through the IPFS bootstrap nodes) to find peers' addresses and public relays, picked after members that relay. Public relays only keep circuits for minutes and some KiB: enough to punch holes through NATs. Makes this node's peer ID and addresses known to the DHT.", ), option( "libp2p-key-file", Kind::Text, "PATH", None, "With the libp2p transport: its key (Ed25519, 32 raw bytes), which makes its peer ID; a new one each start if absent (the node ID stays the same either way).", ), option( "mycelium-rest-url", Kind::Text, "SCHEME://HOSTNAME:PORT", None, "Use a Mycelium daemon running elsewhere (its REST API) instead of supervising one. Excludes the other mycelium options except mycelium-peers.", ) // A URL may carry credentials. .secret(), option( "mycelium-key-file", Kind::Text, "PATH", None, "Private key of the supervised Mycelium daemon (its identity); a new one each start if absent.", ), option( "mycelium-network-key-file", Kind::Text, "PATH", None, "Key of a private Mycelium network for the supervised daemon (with mycelium-network-name).", ), option( "mycelium-network-name", Kind::Text, "NAME", None, "Name of that private Mycelium network (with mycelium-network-key-file).", ), option( "mycelium-tcp-listen-port", Kind::Integer, "PORT", None, "TCP port the supervised Mycelium daemon listens on for overlay peers; a free port if absent.", ), option( "mycelium-peers", Kind::List, "URL,[URL...]", Some("tls://127.0.0.1:9651"), "Peers of the supervised Mycelium daemon (overlay links such as `tls://host:9651`), not continuum peers.", ), OptionSpec { name: "debug-epoch", kind: Kind::Integer, value_name: "GENERATION", default: None, help: "Test hook: start on a non-genesis epoch of this generation (with a fixed dummy id). Needs debug-api.", hidden: true, secret: false, }, ]; pub(crate) fn spec(name: &str) -> Option<&'static OptionSpec> { OPTIONS.iter().find(|spec| spec.name == name) } /// The effective configuration. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] pub(crate) struct Settings { #[serde(default = "default_bind_addr")] pub(crate) bind_addr: SocketAddr, pub(crate) htpasswd: String, #[serde(default)] pub(crate) protocol_writers: Vec, #[serde(default)] pub(crate) debug_api: bool, #[serde(default)] pub(crate) seeds: Vec, #[serde(default)] pub(crate) genesis: bool, pub(crate) state_file: Option, pub(crate) document_file: Option, #[serde( default = "default_document_append", with = "crate::units::duration_option" )] pub(crate) document_append: Duration, #[serde( default = "default_document_rewrite", with = "crate::units::duration_option" )] pub(crate) document_rewrite: Duration, pub(crate) recover_from: Option, pub(crate) remake: Option, #[serde(default)] pub(crate) force: Vec, pub(crate) node_key_file: PathBuf, #[serde(default)] pub(crate) create_node_key: bool, pub(crate) swarm_key_file: Option, pub(crate) alias: Option, #[serde(default = "default_keepalive", with = "crate::units::duration_option")] pub(crate) keepalive: Duration, #[serde(default = "default_neighbours")] pub(crate) neighbours: usize, #[serde( default = "default_electorate_window", with = "crate::units::duration_option" )] pub(crate) electorate_window: Duration, /// In bytes. #[serde(default = "default_blob_chunk", with = "crate::units::size_option")] pub(crate) blob_chunk: u64, #[serde(default = "default_transports")] pub(crate) transports: Vec, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_listen: Vec, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_announce: Vec, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_relay: bool, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_relays: Vec, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_mdns: bool, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_public: bool, #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_key_file: Option, pub(crate) mycelium_rest_url: Option, pub(crate) mycelium_key_file: Option, pub(crate) mycelium_network_key_file: Option, pub(crate) mycelium_network_name: Option, pub(crate) mycelium_tcp_listen_port: Option, #[serde(default = "default_mycelium_peers")] pub(crate) mycelium_peers: Vec, pub(crate) debug_epoch: Option, } fn default_bind_addr() -> SocketAddr { SocketAddr::from(([127, 0, 0, 1], 3000)) } fn default_document_append() -> Duration { Duration::from_secs(5) } fn default_document_rewrite() -> Duration { Duration::from_secs(3600) } fn default_keepalive() -> Duration { Duration::from_secs(60) } fn default_neighbours() -> usize { 12 } fn default_electorate_window() -> Duration { Duration::from_secs(7 * 24 * 3600) } fn default_blob_chunk() -> u64 { 64 * 1024 } fn default_transports() -> Vec { vec!["mycelium".to_owned()] } /// The transports `transports` may name, and whether this build has each. pub(crate) const TRANSPORTS: &[(&str, bool)] = &[("mycelium", true), ("libp2p", cfg!(feature = "libp2p"))]; fn default_mycelium_peers() -> Vec { vec!["tls://127.0.0.1:9651".to_owned()] } impl Settings { /// `mycelium-rest-url`, parsed (checked when loading). pub(crate) fn mycelium_rest_url(&self) -> Option { self.mycelium_rest_url .as_deref() .and_then(|url| url::Url::parse(url).ok()) } /// The options only a supervised Mycelium daemon takes, and whether each is set: the /// other side of `CONFLICTS`' `mycelium-rest-url` (a test keeps them the same). fn supervised_only(&self) -> [(&'static str, bool); 4] { [ ("mycelium-key-file", self.mycelium_key_file.is_some()), ( "mycelium-network-key-file", self.mycelium_network_key_file.is_some(), ), ( "mycelium-network-name", self.mycelium_network_name.is_some(), ), ( "mycelium-tcp-listen-port", self.mycelium_tcp_listen_port.is_some(), ), ] } fn check(&self) -> Result<()> { if let Some(url) = &self.mycelium_rest_url { url::Url::parse(url).with_context(|| format!("mycelium-rest-url {url:?}"))?; } if self.mycelium_rest_url.is_some() && let Some((name, _)) = self.supervised_only().iter().find(|(_, set)| *set) { bail!( "{name} is for a supervised Mycelium daemon and can't be combined with mycelium-rest-url" ); } // A private network has both; one alone would be ignored or invented. if self.mycelium_network_key_file.is_some() != self.mycelium_network_name.is_some() { bail!("mycelium-network-key-file and mycelium-network-name go together"); } crate::force::Force::parse(self.force.iter().map(String::as_str), FORCE) .map_err(|reason| anyhow!("force: {reason}"))?; // Contradictions, not overrides: an error wherever the options come from. if self.genesis && !self.seeds.is_empty() { bail!( "genesis starts a new swarm, so there are no seeds to join through; unset seeds (--unset seeds) or genesis" ); } if self.remake.is_some() && !self.genesis { bail!("remake makes a new swarm's document; it needs genesis"); } if self.genesis && self.recover_from.is_some() { bail!("recover-from starts the swarm from a backup; it can't be combined with genesis"); } if self.transports.is_empty() { bail!("transports: a node needs at least one"); } for (index, name) in self.transports.iter().enumerate() { match TRANSPORTS.iter().find(|(known, _)| known == name) { None => bail!( "transports: unknown transport {name:?} (known: {})", TRANSPORTS .iter() .map(|(known, _)| *known) .collect::>() .join(", ") ), Some((_, false)) => { bail!("transports: this build has no {name} (built without the {name} feature)") } Some(_) if self.transports[..index].contains(name) => { bail!("transports: {name} is named twice") } Some(_) => {} } } if self.debug_epoch.is_some() && !self.debug_api { bail!("debug-epoch needs debug-api"); } if self.neighbours < 2 { bail!("neighbours must be at least 2"); } if self.document_append.is_zero() || self.document_rewrite.is_zero() { bail!("document-append and document-rewrite must be longer than 0"); } if self.keepalive < Duration::from_millis(100) { bail!("keepalive must be at least 100ms"); } if !(1024..=4 << 20).contains(&self.blob_chunk) { bail!("blob-chunk must be between 1KiB and 4MiB"); } if self.electorate_window.is_zero() { bail!("electorate-window must be longer than 0"); } if let Some(alias) = &self.alias { check_alias(alias)?; } Ok(()) } } /// Up to 64 characters, no control characters, not blank. fn check_alias(alias: &str) -> Result<()> { if alias.trim().is_empty() { bail!("alias must not be blank"); } if alias.chars().count() > 64 { bail!("alias must be at most 64 characters"); } if alias.chars().any(char::is_control) { bail!("alias must not contain control characters"); } Ok(()) } // --- values and where they came from ------------------------------------------- #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum Provenance { File(PathBuf), Argument, } #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Origin { pub(crate) provenance: Provenance, /// The option was unset here (back to its default): `--unset`, `unset = [...]` in a /// file, or `null` in a JSON file. pub(crate) unset: bool, /// Importers the value (or some list elements) came from, e.g. `env:HOSTNAME`. The /// value is never shown. pub(crate) imports: Vec, /// Text with placeholders the value (or some list elements) came from, e.g. /// `node-${env:HOSTNAME}`. The value is shown. pub(crate) templates: Vec, /// With placeholders: the value as written, placeholders and all. pub(crate) written: Option, } impl Origin { pub(crate) fn is_imported(&self) -> bool { !self.imports.is_empty() } } impl std::fmt::Display for Origin { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self.provenance { Provenance::File(path) => write!(f, "{}", path.display())?, Provenance::Argument => write!(f, "the command line")?, } if self.unset { write!(f, " (unset)")?; } // Not the text with placeholders: of a secret, it may hold some of it. if self.is_imported() { write!(f, " ({})", self.imports.join(", "))?; } Ok(()) } } #[derive(Debug, Clone)] pub(crate) struct Entry { pub(crate) value: JsonValue, pub(crate) origin: Origin, } impl Entry { /// The value of `option` for logs: imported values and secrets are never shown. pub(crate) fn shown(&self, option: &str) -> String { if self.origin.unset { "".to_owned() } else if spec(option).is_some_and(|spec| spec.secret) { "".to_owned() } else if self.origin.is_imported() { "".to_owned() } else { self.value.to_string() } } } /// Option name to value. A `null` value unsets the option (back to its default). pub(crate) type Layer = BTreeMap; /// An unset of `name` from `provenance`. fn unset_entry(provenance: Provenance) -> Entry { Entry { value: JsonValue::Null, origin: Origin { provenance, unset: true, imports: Vec::new(), templates: Vec::new(), written: None, }, } } /// Options that can't be combined. A source that sets one side drops the other side's /// values from earlier sources (lower precedence), so e.g. `--mycelium-rest-url` on the /// command line overrides `mycelium-key-file` from a file. Both set in one source is an /// error (`Settings::check`). const CONFLICTS: &[(&[&str], &[&str])] = &[( &["mycelium-rest-url"], &[ "mycelium-key-file", "mycelium-network-key-file", "mycelium-network-name", "mycelium-tcp-listen-port", ], )]; /// The options `name` can't be combined with. fn conflicting(name: &str) -> impl Iterator + '_ { CONFLICTS.iter().flat_map(move |(a, b)| { let other: &'static [&'static str] = if a.contains(&name) { b } else if b.contains(&name) { a } else { &[] }; other.iter().copied() }) } /// How a later source relates to the earlier ones it overrides (for logging). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Overlay { /// A later include over an earlier include of the same file: usually a mistake. IncludeOverInclude, /// A file's own values over its includes, or a later `--config` over an earlier. FileOverFile, /// The command line over files. ArgumentOverFile, } /// Puts `entry` for `name` into `layer`, which holds earlier sources, logging what it /// replaces and dropping earlier values it can't be combined with. pub(crate) fn overlay(layer: &mut Layer, name: String, entry: Entry, how: Overlay) { let log = |message: &str, option: &str, previous: &Entry| match how { Overlay::IncludeOverInclude => warn!( option, value = %entry.shown(&name), from = %entry.origin, overrides = %previous.shown(option), overridden_from = %previous.origin, "{message}" ), Overlay::FileOverFile => debug!( option, from = %entry.origin, overrides = %previous.origin, "{message}" ), Overlay::ArgumentOverFile => info!( option, value = %entry.shown(&name), overrides = %previous.shown(option), from = %previous.origin, "{message}" ), }; if !entry.value.is_null() { for other in conflicting(&name) { let earlier = layer .get(other) .is_some_and(|previous| previous.origin.provenance != entry.origin.provenance); if earlier && let Some(previous) = layer.remove(other) { log( &format!("{name} can't be combined with {other}; dropped the earlier value"), other, &previous, ); } } } if let Some(previous) = layer.get(&name) { let message = match how { Overlay::IncludeOverInclude => { "Two included configuration files set the same option; the later one wins" } Overlay::FileOverFile => "Configuration file overrides an earlier one", Overlay::ArgumentOverFile => "Command-line argument overrides the configuration file", }; log(message, &name, previous); } layer.insert(name, entry); } /// The value of a text, by the option's kind. fn from_text(kind: Kind, text: &str) -> Result { Ok(match kind { Kind::Text => JsonValue::String(text.to_owned()), Kind::Integer => { let number: u64 = text .trim() .parse() .map_err(|_| anyhow!("{text:?} is not a non-negative integer"))?; JsonValue::from(number) } Kind::Duration => { crate::units::parse_duration(text).map_err(|reason| anyhow!(reason))?; JsonValue::String(text.trim().to_owned()) } Kind::Size => { crate::units::parse_size(text).map_err(|reason| anyhow!(reason))?; JsonValue::String(text.trim().to_owned()) } Kind::Flag => match text.trim() { "true" | "yes" | "1" => JsonValue::Bool(true), "false" | "no" | "0" => JsonValue::Bool(false), _ => bail!("{text:?} is not true or false"), }, Kind::List => JsonValue::Array( text.split([',', '\n']) .map(str::trim) .filter(|item| !item.is_empty()) .map(|item| JsonValue::String(item.to_owned())) .collect(), ), }) } fn from_import(kind: Kind, importer: &import::Importer) -> Result { match importer.fetch()? { Imported::Text(text) => from_text(kind, &text), Imported::Json(value) => Ok(value), } } /// A value from a file in `dir`: importer tables are resolved, also as list elements. /// Collects where the parts of a value came from, for its [`Origin`]. #[derive(Default)] struct Sources { imports: Vec, templates: Vec, /// The parts as written: literal values, importers, text with placeholders. written: Vec, } impl Sources { /// A literal part (list values count element by element). fn literal(&mut self, value: &JsonValue) { match value { JsonValue::Array(items) => self.written.extend(items.iter().cloned()), other => self.written.push(other.clone()), } } /// A part from `importer`. fn import(&mut self, importer: &import::Importer) { let described = importer.describe(); if importer.is_interpolation() { self.templates.push(described.clone()); } else { self.imports.push(described.clone()); } self.written.push(JsonValue::String(described)); } fn origin(self, provenance: Provenance, unset: bool, kind: Kind) -> Origin { let written = (!self.templates.is_empty()).then(|| match kind { Kind::List => JsonValue::Array(self.written), _ => self.written.into_iter().last().unwrap_or(JsonValue::Null), }); Origin { provenance, unset, imports: self.imports, templates: self.templates, written, } } } /// A value from a file, and where its parts came from. fn resolve_file_value( spec: &OptionSpec, value: JsonValue, dir: &Path, ) -> Result<(JsonValue, Sources)> { let mut sources = Sources::default(); let value = match value { JsonValue::Object(table) => match import::parse_table(&table, dir)? { Some(importer) => { sources.import(&importer); from_import(spec.kind, &importer)? } None => bail!("a table here must be an importer (env, file or command)"), }, JsonValue::String(text) => match import::interpolated(&text, Some(dir)) { Some(importer) => { sources.import(&importer); from_import(spec.kind, &importer)? } None => JsonValue::String(text), }, JsonValue::Array(items) if spec.kind == Kind::List => { let mut values = Vec::new(); for item in items { match item { JsonValue::Object(table) => { let importer = import::parse_table(&table, dir)?.ok_or_else(|| { anyhow!("a table here must be an importer (env, file or command)") })?; match from_import(Kind::List, &importer)? { JsonValue::Array(more) => values.extend(more), other => values.push(other), } sources.import(&importer); } JsonValue::String(text) => match import::interpolated(&text, Some(dir)) { Some(importer) => { values.push(from_import(Kind::Text, &importer)?); sources.import(&importer); } None => { let value = JsonValue::String(text); sources.literal(&value); values.push(value); } }, other => { sources.literal(&other); values.push(other); } } } JsonValue::Array(values) } other => other, }; Ok((value, sources)) } /// The first option whose value alone doesn't fit (serde doesn't name it). fn culprit(merged: &Layer) -> Option<(&String, &Entry)> { merged.iter().find(|(name, entry)| { if entry.value.is_null() { return false; } let mut alone = serde_json::Map::new(); alone.insert("htpasswd".to_owned(), JsonValue::String(String::new())); alone.insert("node-key-file".to_owned(), JsonValue::String(String::new())); alone.insert((*name).clone(), entry.value.clone()); serde_json::from_value::(JsonValue::Object(alone)).is_err() }) } // --- command line -------------------------------------------------------------- const CONFIG: &str = "config"; const PRINT_CONFIG: &str = "print-config"; const UNSET: &str = "unset"; fn command() -> Command { let mut command = Command::new("continuum") .version(env!("CARGO_PKG_VERSION")) .about("continuum: a replicated key-value store over Mycelium") .after_help( "Every option can also be set in a configuration file (--config); arguments win.\n\ A value can be imported instead of written out: env:NAME, file:PATH or cmd:COMMAND \ (as text), or env+json:, file+json:, cmd+json: (parsed as JSON). \ literal:VALUE passes a value that starts with one of these prefixes.", ) .arg( Arg::new(CONFIG) .long(CONFIG) .value_name("PATH") .action(ArgAction::Append) .help("Configuration file (.toml or .json); repeatable, later files win."), ) .arg( Arg::new(UNSET) .long(UNSET) .value_name("OPTION") .action(ArgAction::Append) .help("Unset an option set by a configuration file (back to its default); repeatable."), ) .arg( Arg::new(PRINT_CONFIG) .long(PRINT_CONFIG) .action(ArgAction::SetTrue) .help("Print the effective configuration as JSON (with each value's source) and exit."), ); for spec in OPTIONS { let help = match spec.default { Some(default) => format!("{} [default: {default}]", spec.help), None => spec.help.to_owned(), }; let arg = Arg::new(spec.name) .long(spec.name) .help(help) .hide(spec.hidden); command = command.arg(match spec.kind { Kind::Flag => arg.action(ArgAction::SetTrue), Kind::List => arg.value_name(spec.value_name).action(ArgAction::Append), Kind::Text | Kind::Integer | Kind::Duration | Kind::Size => { arg.value_name(spec.value_name).action(ArgAction::Set) } }); } command } /// The values given as arguments (unsets first, so a value given too wins). fn arguments(matches: &ArgMatches) -> Result { let mut layer = Layer::new(); for name in matches.get_many::(UNSET).into_iter().flatten() { if spec(name).is_none() { bail!("--unset {name}: no such option"); } layer.insert(name.clone(), unset_entry(Provenance::Argument)); } for spec in OPTIONS { if matches.value_source(spec.name) != Some(ValueSource::CommandLine) { continue; } let mut sources = Sources::default(); let value = if spec.kind == Kind::Flag { JsonValue::Bool(true) } else { let mut values = Vec::new(); for raw in matches.get_many::(spec.name).into_iter().flatten() { let value = match import::parse_argument(raw) { Raw::Literal(text) => from_text(spec.kind, text).inspect(|value| { sources.literal(value); }), Raw::Import(importer) => { sources.import(&importer); from_import(spec.kind, &importer) } } .with_context(|| format!("--{}", spec.name))?; values.push(value); } if spec.kind == Kind::List { JsonValue::Array( values .into_iter() .flat_map(|value| match value { JsonValue::Array(items) => items, other => vec![other], }) .collect(), ) } else { values.pop().unwrap_or(JsonValue::Null) } }; layer.insert( spec.name.to_owned(), Entry { value, origin: sources.origin(Provenance::Argument, false, spec.kind), }, ); } Ok(layer) } /// The configuration, and where each value set came from (absent: default). #[derive(Debug, Clone)] pub(crate) struct Loaded { pub(crate) settings: Settings, pub(crate) origins: BTreeMap, pub(crate) files: Vec, /// `--print-config`: print [`Loaded::report`] and exit. pub(crate) print_only: bool, } /// How `GET /admin/config` shows text with placeholders (`?values=`). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) enum Values { /// As it ends up. #[default] Interpolated, /// As written, placeholders and all. Original, } /// The configuration reports, made once at start for `GET /admin/config`. #[derive(Debug, Clone, Default)] pub(crate) struct Reports { interpolated: JsonValue, original: JsonValue, } impl Reports { pub(crate) fn get(&self, values: Values) -> &JsonValue { match values { Values::Interpolated => &self.interpolated, Values::Original => &self.original, } } } impl Loaded { /// The effective configuration with each value's source, for `--print-config` and /// `GET /admin/config`. Imported values are left out (they may be secret); their /// importers are listed instead. Secrets are left out however they were given. /// Text with placeholders is shown as `values` says, the text as written listed. pub(crate) fn report(&self, values: Values) -> JsonValue { let settings = match serde_json::to_value(&self.settings) { Ok(JsonValue::Object(settings)) => settings, _ => serde_json::Map::new(), }; let options: serde_json::Map = OPTIONS .iter() .filter(|spec| !spec.hidden || self.origins.contains_key(spec.name)) .map(|spec| { let mut entry = serde_json::Map::new(); let origin = self.origins.get(spec.name); match origin { None => { entry.insert("source".into(), "default".into()); } Some(origin) => { match &origin.provenance { Provenance::File(path) => { entry.insert("source".into(), "file".into()); entry.insert("file".into(), path.display().to_string().into()); } Provenance::Argument => { entry.insert("source".into(), "argument".into()); } } if origin.is_imported() { entry.insert("imported".into(), origin.imports.clone().into()); } // Of a secret, the text may hold some of it. if !origin.templates.is_empty() && !spec.secret { entry.insert("templates".into(), origin.templates.clone().into()); } if origin.unset { entry.insert("unset".into(), true.into()); } } } let given = origin.is_some_and(|origin| !origin.unset); if spec.secret && given { entry.insert("secret".into(), true.into()); } else if !origin.is_some_and(Origin::is_imported) { let written = origin.and_then(|origin| origin.written.clone()); let value = match (values, written) { (Values::Original, Some(written)) => written, _ => settings.get(spec.name).cloned().unwrap_or(JsonValue::Null), }; entry.insert("value".into(), value); } (spec.name.to_owned(), JsonValue::Object(entry)) }) .collect(); serde_json::json!({ "files": self.files.iter().map(|path| path.display().to_string()).collect::>(), "options": options, }) } pub(crate) fn reports(&self) -> Reports { Reports { interpolated: self.report(Values::Interpolated), original: self.report(Values::Original), } } } /// Reads the command line and the configuration files it names. Exits on `--help`, /// `--version` and command-line syntax errors, like any command. pub(crate) fn load() -> Result { let matches = command().get_matches(); let print_only = matches.get_flag(PRINT_CONFIG); if print_only { // Logs go to stdout too; the report says where each value came from anyway. let _ = crate::logging::set_filter("off"); } let files: Vec = matches .get_many::(CONFIG) .into_iter() .flatten() .map(PathBuf::from) .collect(); let mut merged = Layer::new(); for path in &files { for (name, entry) in file::load(path)? { overlay(&mut merged, name, entry, Overlay::FileOverFile); } } for (name, entry) in arguments(&matches)? { overlay(&mut merged, name, entry, Overlay::ArgumentOverFile); } let mut loaded = resolve(merged, files)?; loaded.print_only = print_only; Ok(loaded) } fn resolve(merged: Layer, files: Vec) -> Result { // An override is for the start it was given for; in a file it would silently apply // to every start, e.g. replacing the document file with a backup each time. if let Some(entry) = merged.get("force").filter(|entry| { !entry.value.is_null() && matches!(entry.origin.provenance, Provenance::File(_)) }) { bail!( "force can only be given on the command line: it overrides checks for one start, and in a file ({}) it would apply to every start", entry.origin ); } // Unset options take their defaults. let object: serde_json::Map = merged .iter() .filter(|(_, entry)| !entry.value.is_null()) .map(|(name, entry)| (name.clone(), entry.value.clone())) .collect(); let mut settings: Settings = serde_json::from_value(JsonValue::Object(object)).map_err(|error| { match culprit(&merged) { Some((name, entry)) => anyhow!( "invalid configuration: option {name} (set in {}): {error}", entry.origin ), None => anyhow!("invalid configuration: {error}"), } })?; settings.alias = settings.alias.map(|alias| alias.trim().to_owned()); settings.check().context("invalid configuration")?; Ok(Loaded { settings, origins: merged .into_iter() .map(|(name, entry)| (name, entry.origin)) .collect(), files, print_only: false, }) } #[cfg(test)] mod tests { use super::*; use serde_json::json; fn layer(values: JsonValue) -> Layer { values .as_object() .unwrap() .iter() .map(|(name, value)| { ( name.clone(), Entry { value: value.clone(), origin: Origin { provenance: Provenance::Argument, unset: false, imports: Vec::new(), templates: Vec::new(), written: None, }, }, ) }) .collect() } #[test] fn force_only_on_the_command_line() { let from = |provenance| Entry { value: json!(["swarmId"]), origin: Origin { provenance, unset: false, imports: Vec::new(), templates: Vec::new(), written: None, }, }; let mut merged = layer(json!({ "htpasswd": "u", "node-key-file": "node.key" })); merged.insert( "force".to_owned(), from(Provenance::File(PathBuf::from("continuum.toml"))), ); assert!(resolve(merged.clone(), Vec::new()).is_err()); merged.insert("force".to_owned(), from(Provenance::Argument)); assert!(resolve(merged, Vec::new()).is_ok()); } #[test] fn options_and_settings_agree() { // Every option is a Settings field, and every field an option. let loaded = resolve( layer(json!({ "htpasswd": "users", "node-key-file": "node.key" })), Vec::new(), ) .unwrap(); let JsonValue::Object(fields) = serde_json::to_value(&loaded.settings).unwrap() else { panic!() }; let mut names: Vec<&str> = fields.keys().map(String::as_str).collect(); let mut options: Vec<&str> = OPTIONS.iter().map(|spec| spec.name).collect(); names.sort_unstable(); options.sort_unstable(); assert_eq!(names, options); // Help defaults match the real ones. for spec in OPTIONS { if let Some(default) = spec.default { assert_eq!( from_text(spec.kind, default).unwrap(), fields[spec.name], "default of {}", spec.name ); } } command().debug_assert(); } #[test] fn later_sources_override_conflicts_and_unset() { let from = |provenance: Provenance, value: JsonValue| Entry { value, origin: Origin { provenance, unset: false, imports: Vec::new(), templates: Vec::new(), written: None, }, }; let file = |value| from(Provenance::File("/etc/continuum.toml".into()), value); let argument = |value| from(Provenance::Argument, value); let url = json!("http://[::1]:8989"); let mut merged = Layer::new(); overlay( &mut merged, "htpasswd".into(), file(json!("u")), Overlay::FileOverFile, ); overlay( &mut merged, "node-key-file".into(), file(json!("node.key")), Overlay::FileOverFile, ); overlay( &mut merged, "mycelium-key-file".into(), file(json!("k")), Overlay::FileOverFile, ); overlay( &mut merged, "alias".into(), file(json!("a")), Overlay::FileOverFile, ); // Both in one source: an error. let mut same = merged.clone(); overlay( &mut same, "mycelium-rest-url".into(), file(url.clone()), Overlay::FileOverFile, ); assert!(resolve(same, Vec::new()).is_err()); // The command line wins over the conflicting file value, and unsets. overlay( &mut merged, "mycelium-rest-url".into(), argument(url), Overlay::ArgumentOverFile, ); overlay( &mut merged, "alias".into(), unset_entry(Provenance::Argument), Overlay::ArgumentOverFile, ); let loaded = resolve(merged, Vec::new()).unwrap(); assert!(loaded.settings.mycelium_key_file.is_none()); assert!(loaded.settings.mycelium_rest_url.is_some()); assert!(loaded.settings.alias.is_none()); assert_eq!( loaded.report(Values::Interpolated)["options"]["alias"], json!({ "source": "argument", "unset": true, "value": null }) ); } #[test] fn report_hides_imported_values() { let mut values = layer( json!({ "htpasswd": "users", "node-key-file": "node.key", "alias": "secret-ish" }), ); values.get_mut("alias").unwrap().origin.imports = vec!["env:HOSTNAME".into()]; let report = resolve(values, Vec::new()) .unwrap() .report(Values::Interpolated); let options = &report["options"]; assert_eq!( options["htpasswd"], json!({ "source": "argument", "value": "users" }) ); assert_eq!( options["alias"], json!({ "source": "argument", "imported": ["env:HOSTNAME"] }) ); assert_eq!( options["neighbours"], json!({ "source": "default", "value": 12 }) ); assert!(options.get("debug-epoch").is_none()); } #[test] fn text_values() { assert_eq!(from_text(Kind::Integer, " 30 ").unwrap(), json!(30)); assert!(from_text(Kind::Integer, "-1").is_err()); assert_eq!( from_text(Kind::List, "a, b\nc,").unwrap(), json!(["a", "b", "c"]) ); assert_eq!(from_text(Kind::Flag, "yes").unwrap(), json!(true)); } /// `Settings::check` and the override rule name the same supervised-only options. #[test] fn supervised_only_options_are_the_conflicts() { let settings = resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key" })), Vec::new(), ) .unwrap() .settings; let names: Vec<&str> = settings .supervised_only() .iter() .map(|(name, _)| *name) .collect(); let (rest_url, supervised) = CONFLICTS[0]; assert_eq!(rest_url, ["mycelium-rest-url"]); assert_eq!(names, supervised); } #[test] fn checks() { assert!(resolve(layer(json!({})), Vec::new()).is_err()); // htpasswd is required assert!( resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "neighbours": 1 })), Vec::new() ) .is_err() ); assert!( resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "debug-epoch": 3 })), Vec::new() ) .is_err() ); assert!( resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "mycelium-rest-url": "http://[::1]:8989", "mycelium-key-file": "k", })), Vec::new() ) .is_err() ); assert!( resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "alias": " " })), Vec::new() ) .is_err() ); assert!( resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "keepalive": "x" })), Vec::new() ) .is_err() ); let transports = |names: JsonValue| { resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "transports": names })), Vec::new(), ) }; assert!(transports(json!([])).is_err()); assert!(transports(json!(["pigeon"])).is_err()); assert!(transports(json!(["mycelium", "mycelium"])).is_err()); assert!(transports(json!(["mycelium"])).is_ok()); assert_eq!( transports(json!(["libp2p", "mycelium"])).is_ok(), cfg!(feature = "libp2p") ); } #[test] fn arguments_with_importers() { // SAFETY: tests in this module don't read this variable concurrently. unsafe { std::env::set_var("CONTINUUM_TEST_SEEDS", "[\"a\", \"b\"]") }; let matches = command() .try_get_matches_from([ "continuum", "--seeds", "env+json:CONTINUUM_TEST_SEEDS", "--seeds", "c,d", "--alias", "literal:env:x", "--debug-api", ]) .unwrap(); let layer = arguments(&matches).unwrap(); assert_eq!(layer["seeds"].value, json!(["a", "b", "c", "d"])); assert_eq!( layer["seeds"].origin.imports, ["env+json:CONTINUUM_TEST_SEEDS"] ); assert_eq!(layer["alias"].value, json!("env:x")); assert_eq!(layer["debug-api"].value, json!(true)); assert!(!layer.contains_key("neighbours")); } #[test] fn report_shows_interpolated_values() { // SAFETY: tests in this module don't read this variable concurrently. unsafe { std::env::set_var("CONTINUUM_TEST_SITE", "berlin") }; let matches = command() .try_get_matches_from([ "continuum", "--htpasswd", "users", "--node-key-file", "node.key", "--alias", "node-${env:CONTINUUM_TEST_SITE}", "--seeds", "a,b", "--seeds", "c-${env:CONTINUUM_TEST_SITE}", ]) .unwrap(); let layer = arguments(&matches).unwrap(); assert!(!layer["alias"].origin.is_imported()); assert_eq!(layer["alias"].shown("alias"), "\"node-berlin\""); let loaded = resolve(layer, Vec::new()).unwrap(); let report = loaded.report(Values::Interpolated); assert_eq!( report["options"]["alias"], json!({ "source": "argument", "templates": ["node-${env:CONTINUUM_TEST_SITE}"], "value": "node-berlin", }) ); assert_eq!( report["options"]["seeds"]["value"], json!(["a", "b", "c-berlin"]) ); let report = loaded.report(Values::Original); assert_eq!( report["options"]["alias"]["value"], json!("node-${env:CONTINUUM_TEST_SITE}") ); assert_eq!( report["options"]["seeds"]["value"], json!(["a", "b", "c-${env:CONTINUUM_TEST_SITE}"]) ); // Without placeholders, both are the same. assert_eq!( report["options"]["neighbours"], json!({ "source": "default", "value": 12 }) ); } #[test] fn secrets_are_never_shown() { let values = layer( json!({ "htpasswd": "users", "node-key-file": "node.key", "mycelium-rest-url": "http://u:pw@h:8989" }), ); assert_eq!( values["mycelium-rest-url"].shown("mycelium-rest-url"), "" ); let loaded = resolve(values, Vec::new()).unwrap(); for mode in [Values::Interpolated, Values::Original] { assert_eq!( loaded.report(mode)["options"]["mycelium-rest-url"], json!({ "source": "argument", "secret": true }) ); } } }