//! Configuration: built-in defaults, then configuration files (`--config`, in order), //! then command-line arguments. Every option can be set in each. //! //! An argument that replaces a value from a file is logged (info), as is a file that //! replaces one from another file (debug; warning between two includes of one file). //! Values can be imported instead of written out (`import`): imported values are never //! logged, only where they come from. //! //! All options are described once, in [`OPTIONS`]: the command line, the file keys and //! the conversion of text values all derive from it. The merged values are then read //! into [`Settings`] and checked. mod file; mod import; use std::collections::BTreeMap; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; use clap::parser::ValueSource; use clap::{Arg, ArgAction, ArgMatches, Command}; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; use tracing::{debug, info, warn}; use import::{Imported, Raw}; /// How an option's value is written as text (command line, text imports). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Kind { /// `true`/`false`; on the command line, present or not. Flag, Text, Integer, /// Seconds, or with units: `500ms`, `30s`, `1h30m` (`units::parse_duration`). Duration, /// Bytes, or with a unit: `64KiB` (`units::parse_size`). Size, /// Comma- or line-separated; repeatable on the command line. List, } #[derive(Debug)] pub(crate) struct OptionSpec { pub(crate) name: &'static str, pub(crate) kind: Kind, pub(crate) value_name: &'static str, /// For `--help` only; the effective defaults are in [`Settings`]. pub(crate) default: Option<&'static str>, pub(crate) help: &'static str, pub(crate) hidden: bool, /// Its value is never shown (logs, `--print-config`, `GET /admin/config`), however /// it was given. pub(crate) secret: bool, } impl OptionSpec { const fn secret(self) -> Self { Self { secret: true, ..self } } } const fn option( name: &'static str, kind: Kind, value_name: &'static str, default: Option<&'static str>, help: &'static str, ) -> OptionSpec { OptionSpec { name, kind, value_name, default, help, hidden: false, secret: false, } } /// What `--force` may override at start (`force`). pub(crate) const FORCE: &[&str] = &["swarmId", "badDocumentFile", "replaceDocumentFile"]; /// Every option. Names are the long option names and the configuration file keys. pub(crate) const OPTIONS: &[OptionSpec] = &[ option( "bind-addr", Kind::Text, "HOSTNAME:PORT", Some("127.0.0.1:3000"), "Address of the HTTP API.", ), option( "htpasswd", Kind::Text, "PATH", None, "Users of the HTTP API (htpasswd file with Argon2 hashes). Required.", ), option( "protocol-writers", Kind::List, "USER,[USER...]", None, "HTTP API users (from htpasswd) who may write protocol data under /store/_continuum (remake records, members, forget records), e.g. to repair a stuck remake. Such writes act on the whole swarm and are logged. Epoch metadata and the swarm ID are never writable.", ), option( "debug-api", Kind::Flag, "BOOL", Some("false"), "Enable the /debug endpoints (tests only).", ), option( "seeds", Kind::List, "STRING,[STRING...]", None, "How to reach a node of the swarm to join: its address (`mycelium:`, as GET /admin/stats lists it), a Mycelium public key, a Mycelium IPv6 address, or a host name with an AAAA record for one. A node keeps trying until it has joined.", ), option( "genesis", Kind::Flag, "BOOL", Some("false"), "Start a new swarm, as its genesis node, instead of joining one through seeds. Every start with it starts another new swarm: drop it once the swarm exists, and give the node seeds instead. Can't be combined with seeds. With a --state-file that remembers a swarm, it starts that swarm again instead (empty; nodes of it still running merge with this one).", ), option( "force", Kind::List, "NAME,[NAME...]", None, "Override checks that otherwise refuse to start (command line only: it is for one start): true for all, or names: swarmId (a document to start on is of another swarm than the state file remembers), badDocumentFile (an unusable document file: moved aside, start without it), replaceDocumentFile (recover-from while the document file holds a document).", ), option( "recover-from", Kind::Text, "PATH", None, "Start the swarm again from a backup (as GET /admin/document saves it): this node loads it, on the swarm and epoch it names, and is ready at once; other nodes join it. Members of the swarm still running merge with it, or it rejoins them if they moved on to a newer epoch (the backup is then dropped).", ), option( "remake", Kind::Text, "KIND[:PATH]", None, "With --genesis: make the new swarm's document as a remake of this kind would, instead of starting it empty: import:PATH (an Automerge document, e.g. a backup, without its history and protocol data) or replace:PATH (a JSON document, the store root); compact changes nothing.", ), option( "document-file", Kind::Text, "PATH", None, "This node's document on disk. Read at start: the node resumes with it, ready at once. Kept up to date while ready: changes are appended every --document-append, and the file is rewritten after a switch, a rejoin or a restart, and when the appended changes outgrew it.", ), option( "document-append", Kind::Duration, "DURATION", Some("5s"), "How often changes are appended to the document file.", ), option( "document-rewrite", Kind::Duration, "DURATION", Some("1h"), "Shortest time between rewrites of the document file because its appended changes outgrew its full save (a new epoch or a restart rewrites it at once).", ), option( "state-file", Kind::Text, "PATH", None, "File where this node remembers its swarm across restarts: the swarm ID and some members to rejoin through (not the document). Written once ready, then rarely. Without it, a restarted node knows only its configured seeds.", ), option( "node-key-file", Kind::Text, "PATH", None, "This node's key (Ed25519, 32 raw bytes): who the node is, whatever transport reaches it. Keep it with the node: a new key is a new node. Required.", ), option( "create-node-key", Kind::Flag, "BOOL", Some("false"), "Create a new node key at node-key-file if there is no file there (a new node). Without it, a missing key file is an error.", ), option( "swarm-key-file", Kind::Text, "PATH", None, "Pre-shared swarm key: only nodes that know it may join (raw bytes, at least 16; e.g. `head -c 32 /dev/urandom`). Without it, any node that can reach a member may join.", ), option( "alias", Kind::Text, "NAME", None, "A name for this node, shown next to its public key (/peers, metrics). Published in the swarm's member list; not unique, never an identity.", ), option( "keepalive", Kind::Duration, "DURATION", Some("1m"), "Keepalive interval; dormancy, resync and probe timings derive from it.", ), option( "neighbours", Kind::Integer, "COUNT", Some("12"), "Neighbours each node keeps (at least 2). Swarms of up to this many + 1 nodes are a full mesh; larger ones sync over several hops.", ), option( "electorate-window", Kind::Duration, "DURATION", Some("7d"), "Members that renewed their alive stamp within this time vote on remakes (a majority of them must agree). Matters on the node that coordinates a remake.", ), option( "blob-chunk", Kind::Size, "SIZE", Some("64KiB"), "Chunk size when serving large transfers to other nodes (e.g. an epoch's snapshot to a rejoining node), 1KiB to 4MiB.", ), option( "transports", Kind::List, "NAME,[NAME...]", Some("mycelium"), "The transports this node uses, in order of preference: mycelium, libp2p (in builds with the libp2p feature). A peer is reached on the first one both have; nodes on several carry the traffic between nodes that share none.", ), option( "libp2p-listen", Kind::List, "MULTIADDR,[MULTIADDR...]", None, "With the libp2p transport: where to listen, e.g. /ip4/0.0.0.0/tcp/9661 and /ip4/0.0.0.0/udp/9661/quic-v1. Without, the node only connects out. The addresses it listens on are published in the member list.", ), option( "libp2p-announce", Kind::List, "MULTIADDR,[MULTIADDR...]", None, "With the libp2p transport: the addresses to publish in the member list instead of those listened on, e.g. a public address forwarded to this node's listen port. Without, the listen addresses are published, loopback and link-local ones left out.", ), option( "libp2p-relay", Kind::Flag, "BOOL", Some("false"), "With the libp2p transport: relay for other nodes of the swarm that can't be dialled (bounded: 64 circuits, 8 per peer, each up to an hour and 1 GiB). Needs a listen address others can dial.", ), option( "libp2p-relays", Kind::List, "MULTIADDR,[MULTIADDR...]", None, "With the libp2p transport: relays to be reachable through (multiaddrs, with or without the relay's peer ID), e.g. members started with libp2p-relay. Only these are used; never public ones.", ), option( "libp2p-key-file", Kind::Text, "PATH", None, "With the libp2p transport: its key (Ed25519, 32 raw bytes), which makes its peer ID; a new one each start if absent (the node ID stays the same either way).", ), option( "mycelium-rest-url", Kind::Text, "SCHEME://HOSTNAME:PORT", None, "Use a Mycelium daemon running elsewhere (its REST API) instead of supervising one. Excludes the other mycelium options except mycelium-peers.", ) // A URL may carry credentials. .secret(), option( "mycelium-key-file", Kind::Text, "PATH", None, "Private key of the supervised Mycelium daemon (its identity); a new one each start if absent.", ), option( "mycelium-network-key-file", Kind::Text, "PATH", None, "Key of a private Mycelium network for the supervised daemon (with mycelium-network-name).", ), option( "mycelium-network-name", Kind::Text, "NAME", None, "Name of that private Mycelium network (with mycelium-network-key-file).", ), option( "mycelium-tcp-listen-port", Kind::Integer, "PORT", None, "TCP port the supervised Mycelium daemon listens on for overlay peers; a free port if absent.", ), option( "mycelium-peers", Kind::List, "URL,[URL...]", Some("tls://127.0.0.1:9651"), "Peers of the supervised Mycelium daemon (overlay links such as `tls://host:9651`), not continuum peers.", ), OptionSpec { name: "debug-epoch", kind: Kind::Integer, value_name: "GENERATION", default: None, help: "Test hook: start on a non-genesis epoch of this generation (with a fixed dummy id). Needs debug-api.", hidden: true, secret: false, }, ]; pub(crate) fn spec(name: &str) -> Option<&'static OptionSpec> { OPTIONS.iter().find(|spec| spec.name == name) } /// The effective configuration. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] pub(crate) struct Settings { #[serde(default = "default_bind_addr")] pub(crate) bind_addr: SocketAddr, pub(crate) htpasswd: String, #[serde(default)] pub(crate) protocol_writers: Vec, #[serde(default)] pub(crate) debug_api: bool, #[serde(default)] pub(crate) seeds: Vec, #[serde(default)] pub(crate) genesis: bool, pub(crate) state_file: Option, pub(crate) document_file: Option, #[serde( default = "default_document_append", with = "crate::units::duration_option" )] pub(crate) document_append: Duration, #[serde( default = "default_document_rewrite", with = "crate::units::duration_option" )] pub(crate) document_rewrite: Duration, pub(crate) recover_from: Option, pub(crate) remake: Option, #[serde(default)] pub(crate) force: Vec, pub(crate) node_key_file: PathBuf, #[serde(default)] pub(crate) create_node_key: bool, pub(crate) swarm_key_file: Option, pub(crate) alias: Option, #[serde(default = "default_keepalive", with = "crate::units::duration_option")] pub(crate) keepalive: Duration, #[serde(default = "default_neighbours")] pub(crate) neighbours: usize, #[serde( default = "default_electorate_window", with = "crate::units::duration_option" )] pub(crate) electorate_window: Duration, /// In bytes. #[serde(default = "default_blob_chunk", with = "crate::units::size_option")] pub(crate) blob_chunk: u64, #[serde(default = "default_transports")] pub(crate) transports: Vec, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_listen: Vec, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_announce: Vec, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_relay: bool, #[serde(default)] #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_relays: Vec, #[cfg_attr(not(feature = "libp2p"), allow(dead_code))] pub(crate) libp2p_key_file: Option, pub(crate) mycelium_rest_url: Option, pub(crate) mycelium_key_file: Option, pub(crate) mycelium_network_key_file: Option, pub(crate) mycelium_network_name: Option, pub(crate) mycelium_tcp_listen_port: Option, #[serde(default = "default_mycelium_peers")] pub(crate) mycelium_peers: Vec, pub(crate) debug_epoch: Option, } fn default_bind_addr() -> SocketAddr { SocketAddr::from(([127, 0, 0, 1], 3000)) } fn default_document_append() -> Duration { Duration::from_secs(5) } fn default_document_rewrite() -> Duration { Duration::from_secs(3600) } fn default_keepalive() -> Duration { Duration::from_secs(60) } fn default_neighbours() -> usize { 12 } fn default_electorate_window() -> Duration { Duration::from_secs(7 * 24 * 3600) } fn default_blob_chunk() -> u64 { 64 * 1024 } fn default_transports() -> Vec { vec!["mycelium".to_owned()] } /// The transports `transports` may name, and whether this build has each. pub(crate) const TRANSPORTS: &[(&str, bool)] = &[("mycelium", true), ("libp2p", cfg!(feature = "libp2p"))]; fn default_mycelium_peers() -> Vec { vec!["tls://127.0.0.1:9651".to_owned()] } impl Settings { /// `mycelium-rest-url`, parsed (checked when loading). pub(crate) fn mycelium_rest_url(&self) -> Option { self.mycelium_rest_url .as_deref() .and_then(|url| url::Url::parse(url).ok()) } /// The options only a supervised Mycelium daemon takes, and whether each is set: the /// other side of `CONFLICTS`' `mycelium-rest-url` (a test keeps them the same). fn supervised_only(&self) -> [(&'static str, bool); 4] { [ ("mycelium-key-file", self.mycelium_key_file.is_some()), ( "mycelium-network-key-file", self.mycelium_network_key_file.is_some(), ), ( "mycelium-network-name", self.mycelium_network_name.is_some(), ), ( "mycelium-tcp-listen-port", self.mycelium_tcp_listen_port.is_some(), ), ] } fn check(&self) -> Result<()> { if let Some(url) = &self.mycelium_rest_url { url::Url::parse(url).with_context(|| format!("mycelium-rest-url {url:?}"))?; } if self.mycelium_rest_url.is_some() && let Some((name, _)) = self.supervised_only().iter().find(|(_, set)| *set) { bail!( "{name} is for a supervised Mycelium daemon and can't be combined with mycelium-rest-url" ); } // A private network has both; one alone would be ignored or invented. if self.mycelium_network_key_file.is_some() != self.mycelium_network_name.is_some() { bail!("mycelium-network-key-file and mycelium-network-name go together"); } crate::force::Force::parse(self.force.iter().map(String::as_str), FORCE) .map_err(|reason| anyhow!("force: {reason}"))?; // Contradictions, not overrides: an error wherever the options come from. if self.genesis && !self.seeds.is_empty() { bail!( "genesis starts a new swarm, so there are no seeds to join through; unset seeds (--unset seeds) or genesis" ); } if self.remake.is_some() && !self.genesis { bail!("remake makes a new swarm's document; it needs genesis"); } if self.genesis && self.recover_from.is_some() { bail!("recover-from starts the swarm from a backup; it can't be combined with genesis"); } if self.transports.is_empty() { bail!("transports: a node needs at least one"); } for (index, name) in self.transports.iter().enumerate() { match TRANSPORTS.iter().find(|(known, _)| known == name) { None => bail!( "transports: unknown transport {name:?} (known: {})", TRANSPORTS .iter() .map(|(known, _)| *known) .collect::>() .join(", ") ), Some((_, false)) => { bail!("transports: this build has no {name} (built without the {name} feature)") } Some(_) if self.transports[..index].contains(name) => { bail!("transports: {name} is named twice") } Some(_) => {} } } if self.debug_epoch.is_some() && !self.debug_api { bail!("debug-epoch needs debug-api"); } if self.neighbours < 2 { bail!("neighbours must be at least 2"); } if self.document_append.is_zero() || self.document_rewrite.is_zero() { bail!("document-append and document-rewrite must be longer than 0"); } if self.keepalive < Duration::from_millis(100) { bail!("keepalive must be at least 100ms"); } if !(1024..=4 << 20).contains(&self.blob_chunk) { bail!("blob-chunk must be between 1KiB and 4MiB"); } if self.electorate_window.is_zero() { bail!("electorate-window must be longer than 0"); } if let Some(alias) = &self.alias { check_alias(alias)?; } Ok(()) } } /// Up to 64 characters, no control characters, not blank. fn check_alias(alias: &str) -> Result<()> { if alias.trim().is_empty() { bail!("alias must not be blank"); } if alias.chars().count() > 64 { bail!("alias must be at most 64 characters"); } if alias.chars().any(char::is_control) { bail!("alias must not contain control characters"); } Ok(()) } // --- values and where they came from ------------------------------------------- #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum Provenance { File(PathBuf), Argument, } #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Origin { pub(crate) provenance: Provenance, /// The option was unset here (back to its default): `--unset`, `unset = [...]` in a /// file, or `null` in a JSON file. pub(crate) unset: bool, /// Importers the value (or some list elements) came from, e.g. `env:HOSTNAME`. The /// value is never shown. pub(crate) imports: Vec, /// Text with placeholders the value (or some list elements) came from, e.g. /// `node-${env:HOSTNAME}`. The value is shown. pub(crate) templates: Vec, /// With placeholders: the value as written, placeholders and all. pub(crate) written: Option, } impl Origin { pub(crate) fn is_imported(&self) -> bool { !self.imports.is_empty() } } impl std::fmt::Display for Origin { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self.provenance { Provenance::File(path) => write!(f, "{}", path.display())?, Provenance::Argument => write!(f, "the command line")?, } if self.unset { write!(f, " (unset)")?; } // Not the text with placeholders: of a secret, it may hold some of it. if self.is_imported() { write!(f, " ({})", self.imports.join(", "))?; } Ok(()) } } #[derive(Debug, Clone)] pub(crate) struct Entry { pub(crate) value: JsonValue, pub(crate) origin: Origin, } impl Entry { /// The value of `option` for logs: imported values and secrets are never shown. pub(crate) fn shown(&self, option: &str) -> String { if self.origin.unset { "".to_owned() } else if spec(option).is_some_and(|spec| spec.secret) { "".to_owned() } else if self.origin.is_imported() { "".to_owned() } else { self.value.to_string() } } } /// Option name to value. A `null` value unsets the option (back to its default). pub(crate) type Layer = BTreeMap; /// An unset of `name` from `provenance`. fn unset_entry(provenance: Provenance) -> Entry { Entry { value: JsonValue::Null, origin: Origin { provenance, unset: true, imports: Vec::new(), templates: Vec::new(), written: None, }, } } /// Options that can't be combined. A source that sets one side drops the other side's /// values from earlier sources (lower precedence), so e.g. `--mycelium-rest-url` on the /// command line overrides `mycelium-key-file` from a file. Both set in one source is an /// error (`Settings::check`). const CONFLICTS: &[(&[&str], &[&str])] = &[( &["mycelium-rest-url"], &[ "mycelium-key-file", "mycelium-network-key-file", "mycelium-network-name", "mycelium-tcp-listen-port", ], )]; /// The options `name` can't be combined with. fn conflicting(name: &str) -> impl Iterator + '_ { CONFLICTS.iter().flat_map(move |(a, b)| { let other: &'static [&'static str] = if a.contains(&name) { b } else if b.contains(&name) { a } else { &[] }; other.iter().copied() }) } /// How a later source relates to the earlier ones it overrides (for logging). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Overlay { /// A later include over an earlier include of the same file: usually a mistake. IncludeOverInclude, /// A file's own values over its includes, or a later `--config` over an earlier. FileOverFile, /// The command line over files. ArgumentOverFile, } /// Puts `entry` for `name` into `layer`, which holds earlier sources, logging what it /// replaces and dropping earlier values it can't be combined with. pub(crate) fn overlay(layer: &mut Layer, name: String, entry: Entry, how: Overlay) { let log = |message: &str, option: &str, previous: &Entry| match how { Overlay::IncludeOverInclude => warn!( option, value = %entry.shown(&name), from = %entry.origin, overrides = %previous.shown(option), overridden_from = %previous.origin, "{message}" ), Overlay::FileOverFile => debug!( option, from = %entry.origin, overrides = %previous.origin, "{message}" ), Overlay::ArgumentOverFile => info!( option, value = %entry.shown(&name), overrides = %previous.shown(option), from = %previous.origin, "{message}" ), }; if !entry.value.is_null() { for other in conflicting(&name) { let earlier = layer .get(other) .is_some_and(|previous| previous.origin.provenance != entry.origin.provenance); if earlier && let Some(previous) = layer.remove(other) { log( &format!("{name} can't be combined with {other}; dropped the earlier value"), other, &previous, ); } } } if let Some(previous) = layer.get(&name) { let message = match how { Overlay::IncludeOverInclude => { "Two included configuration files set the same option; the later one wins" } Overlay::FileOverFile => "Configuration file overrides an earlier one", Overlay::ArgumentOverFile => "Command-line argument overrides the configuration file", }; log(message, &name, previous); } layer.insert(name, entry); } /// The value of a text, by the option's kind. fn from_text(kind: Kind, text: &str) -> Result { Ok(match kind { Kind::Text => JsonValue::String(text.to_owned()), Kind::Integer => { let number: u64 = text .trim() .parse() .map_err(|_| anyhow!("{text:?} is not a non-negative integer"))?; JsonValue::from(number) } Kind::Duration => { crate::units::parse_duration(text).map_err(|reason| anyhow!(reason))?; JsonValue::String(text.trim().to_owned()) } Kind::Size => { crate::units::parse_size(text).map_err(|reason| anyhow!(reason))?; JsonValue::String(text.trim().to_owned()) } Kind::Flag => match text.trim() { "true" | "yes" | "1" => JsonValue::Bool(true), "false" | "no" | "0" => JsonValue::Bool(false), _ => bail!("{text:?} is not true or false"), }, Kind::List => JsonValue::Array( text.split([',', '\n']) .map(str::trim) .filter(|item| !item.is_empty()) .map(|item| JsonValue::String(item.to_owned())) .collect(), ), }) } fn from_import(kind: Kind, importer: &import::Importer) -> Result { match importer.fetch()? { Imported::Text(text) => from_text(kind, &text), Imported::Json(value) => Ok(value), } } /// A value from a file in `dir`: importer tables are resolved, also as list elements. /// Collects where the parts of a value came from, for its [`Origin`]. #[derive(Default)] struct Sources { imports: Vec, templates: Vec, /// The parts as written: literal values, importers, text with placeholders. written: Vec, } impl Sources { /// A literal part (list values count element by element). fn literal(&mut self, value: &JsonValue) { match value { JsonValue::Array(items) => self.written.extend(items.iter().cloned()), other => self.written.push(other.clone()), } } /// A part from `importer`. fn import(&mut self, importer: &import::Importer) { let described = importer.describe(); if importer.is_interpolation() { self.templates.push(described.clone()); } else { self.imports.push(described.clone()); } self.written.push(JsonValue::String(described)); } fn origin(self, provenance: Provenance, unset: bool, kind: Kind) -> Origin { let written = (!self.templates.is_empty()).then(|| match kind { Kind::List => JsonValue::Array(self.written), _ => self.written.into_iter().last().unwrap_or(JsonValue::Null), }); Origin { provenance, unset, imports: self.imports, templates: self.templates, written, } } } /// A value from a file, and where its parts came from. fn resolve_file_value( spec: &OptionSpec, value: JsonValue, dir: &Path, ) -> Result<(JsonValue, Sources)> { let mut sources = Sources::default(); let value = match value { JsonValue::Object(table) => match import::parse_table(&table, dir)? { Some(importer) => { sources.import(&importer); from_import(spec.kind, &importer)? } None => bail!("a table here must be an importer (env, file or command)"), }, JsonValue::String(text) => match import::interpolated(&text, Some(dir)) { Some(importer) => { sources.import(&importer); from_import(spec.kind, &importer)? } None => JsonValue::String(text), }, JsonValue::Array(items) if spec.kind == Kind::List => { let mut values = Vec::new(); for item in items { match item { JsonValue::Object(table) => { let importer = import::parse_table(&table, dir)?.ok_or_else(|| { anyhow!("a table here must be an importer (env, file or command)") })?; match from_import(Kind::List, &importer)? { JsonValue::Array(more) => values.extend(more), other => values.push(other), } sources.import(&importer); } JsonValue::String(text) => match import::interpolated(&text, Some(dir)) { Some(importer) => { values.push(from_import(Kind::Text, &importer)?); sources.import(&importer); } None => { let value = JsonValue::String(text); sources.literal(&value); values.push(value); } }, other => { sources.literal(&other); values.push(other); } } } JsonValue::Array(values) } other => other, }; Ok((value, sources)) } /// The first option whose value alone doesn't fit (serde doesn't name it). fn culprit(merged: &Layer) -> Option<(&String, &Entry)> { merged.iter().find(|(name, entry)| { if entry.value.is_null() { return false; } let mut alone = serde_json::Map::new(); alone.insert("htpasswd".to_owned(), JsonValue::String(String::new())); alone.insert("node-key-file".to_owned(), JsonValue::String(String::new())); alone.insert((*name).clone(), entry.value.clone()); serde_json::from_value::(JsonValue::Object(alone)).is_err() }) } // --- command line -------------------------------------------------------------- const CONFIG: &str = "config"; const PRINT_CONFIG: &str = "print-config"; const UNSET: &str = "unset"; fn command() -> Command { let mut command = Command::new("continuum") .version(env!("CARGO_PKG_VERSION")) .about("continuum: a replicated key-value store over Mycelium") .after_help( "Every option can also be set in a configuration file (--config); arguments win.\n\ A value can be imported instead of written out: env:NAME, file:PATH or cmd:COMMAND \ (as text), or env+json:, file+json:, cmd+json: (parsed as JSON). \ literal:VALUE passes a value that starts with one of these prefixes.", ) .arg( Arg::new(CONFIG) .long(CONFIG) .value_name("PATH") .action(ArgAction::Append) .help("Configuration file (.toml or .json); repeatable, later files win."), ) .arg( Arg::new(UNSET) .long(UNSET) .value_name("OPTION") .action(ArgAction::Append) .help("Unset an option set by a configuration file (back to its default); repeatable."), ) .arg( Arg::new(PRINT_CONFIG) .long(PRINT_CONFIG) .action(ArgAction::SetTrue) .help("Print the effective configuration as JSON (with each value's source) and exit."), ); for spec in OPTIONS { let help = match spec.default { Some(default) => format!("{} [default: {default}]", spec.help), None => spec.help.to_owned(), }; let arg = Arg::new(spec.name) .long(spec.name) .help(help) .hide(spec.hidden); command = command.arg(match spec.kind { Kind::Flag => arg.action(ArgAction::SetTrue), Kind::List => arg.value_name(spec.value_name).action(ArgAction::Append), Kind::Text | Kind::Integer | Kind::Duration | Kind::Size => { arg.value_name(spec.value_name).action(ArgAction::Set) } }); } command } /// The values given as arguments (unsets first, so a value given too wins). fn arguments(matches: &ArgMatches) -> Result { let mut layer = Layer::new(); for name in matches.get_many::(UNSET).into_iter().flatten() { if spec(name).is_none() { bail!("--unset {name}: no such option"); } layer.insert(name.clone(), unset_entry(Provenance::Argument)); } for spec in OPTIONS { if matches.value_source(spec.name) != Some(ValueSource::CommandLine) { continue; } let mut sources = Sources::default(); let value = if spec.kind == Kind::Flag { JsonValue::Bool(true) } else { let mut values = Vec::new(); for raw in matches.get_many::(spec.name).into_iter().flatten() { let value = match import::parse_argument(raw) { Raw::Literal(text) => from_text(spec.kind, text).inspect(|value| { sources.literal(value); }), Raw::Import(importer) => { sources.import(&importer); from_import(spec.kind, &importer) } } .with_context(|| format!("--{}", spec.name))?; values.push(value); } if spec.kind == Kind::List { JsonValue::Array( values .into_iter() .flat_map(|value| match value { JsonValue::Array(items) => items, other => vec![other], }) .collect(), ) } else { values.pop().unwrap_or(JsonValue::Null) } }; layer.insert( spec.name.to_owned(), Entry { value, origin: sources.origin(Provenance::Argument, false, spec.kind), }, ); } Ok(layer) } /// The configuration, and where each value set came from (absent: default). #[derive(Debug, Clone)] pub(crate) struct Loaded { pub(crate) settings: Settings, pub(crate) origins: BTreeMap, pub(crate) files: Vec, /// `--print-config`: print [`Loaded::report`] and exit. pub(crate) print_only: bool, } /// How `GET /admin/config` shows text with placeholders (`?values=`). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) enum Values { /// As it ends up. #[default] Interpolated, /// As written, placeholders and all. Original, } /// The configuration reports, made once at start for `GET /admin/config`. #[derive(Debug, Clone, Default)] pub(crate) struct Reports { interpolated: JsonValue, original: JsonValue, } impl Reports { pub(crate) fn get(&self, values: Values) -> &JsonValue { match values { Values::Interpolated => &self.interpolated, Values::Original => &self.original, } } } impl Loaded { /// The effective configuration with each value's source, for `--print-config` and /// `GET /admin/config`. Imported values are left out (they may be secret); their /// importers are listed instead. Secrets are left out however they were given. /// Text with placeholders is shown as `values` says, the text as written listed. pub(crate) fn report(&self, values: Values) -> JsonValue { let settings = match serde_json::to_value(&self.settings) { Ok(JsonValue::Object(settings)) => settings, _ => serde_json::Map::new(), }; let options: serde_json::Map = OPTIONS .iter() .filter(|spec| !spec.hidden || self.origins.contains_key(spec.name)) .map(|spec| { let mut entry = serde_json::Map::new(); let origin = self.origins.get(spec.name); match origin { None => { entry.insert("source".into(), "default".into()); } Some(origin) => { match &origin.provenance { Provenance::File(path) => { entry.insert("source".into(), "file".into()); entry.insert("file".into(), path.display().to_string().into()); } Provenance::Argument => { entry.insert("source".into(), "argument".into()); } } if origin.is_imported() { entry.insert("imported".into(), origin.imports.clone().into()); } // Of a secret, the text may hold some of it. if !origin.templates.is_empty() && !spec.secret { entry.insert("templates".into(), origin.templates.clone().into()); } if origin.unset { entry.insert("unset".into(), true.into()); } } } let given = origin.is_some_and(|origin| !origin.unset); if spec.secret && given { entry.insert("secret".into(), true.into()); } else if !origin.is_some_and(Origin::is_imported) { let written = origin.and_then(|origin| origin.written.clone()); let value = match (values, written) { (Values::Original, Some(written)) => written, _ => settings.get(spec.name).cloned().unwrap_or(JsonValue::Null), }; entry.insert("value".into(), value); } (spec.name.to_owned(), JsonValue::Object(entry)) }) .collect(); serde_json::json!({ "files": self.files.iter().map(|path| path.display().to_string()).collect::>(), "options": options, }) } pub(crate) fn reports(&self) -> Reports { Reports { interpolated: self.report(Values::Interpolated), original: self.report(Values::Original), } } } /// Reads the command line and the configuration files it names. Exits on `--help`, /// `--version` and command-line syntax errors, like any command. pub(crate) fn load() -> Result { let matches = command().get_matches(); let print_only = matches.get_flag(PRINT_CONFIG); if print_only { // Logs go to stdout too; the report says where each value came from anyway. let _ = crate::logging::set_filter("off"); } let files: Vec = matches .get_many::(CONFIG) .into_iter() .flatten() .map(PathBuf::from) .collect(); let mut merged = Layer::new(); for path in &files { for (name, entry) in file::load(path)? { overlay(&mut merged, name, entry, Overlay::FileOverFile); } } for (name, entry) in arguments(&matches)? { overlay(&mut merged, name, entry, Overlay::ArgumentOverFile); } let mut loaded = resolve(merged, files)?; loaded.print_only = print_only; Ok(loaded) } fn resolve(merged: Layer, files: Vec) -> Result { // An override is for the start it was given for; in a file it would silently apply // to every start, e.g. replacing the document file with a backup each time. if let Some(entry) = merged.get("force").filter(|entry| { !entry.value.is_null() && matches!(entry.origin.provenance, Provenance::File(_)) }) { bail!( "force can only be given on the command line: it overrides checks for one start, and in a file ({}) it would apply to every start", entry.origin ); } // Unset options take their defaults. let object: serde_json::Map = merged .iter() .filter(|(_, entry)| !entry.value.is_null()) .map(|(name, entry)| (name.clone(), entry.value.clone())) .collect(); let mut settings: Settings = serde_json::from_value(JsonValue::Object(object)).map_err(|error| { match culprit(&merged) { Some((name, entry)) => anyhow!( "invalid configuration: option {name} (set in {}): {error}", entry.origin ), None => anyhow!("invalid configuration: {error}"), } })?; settings.alias = settings.alias.map(|alias| alias.trim().to_owned()); settings.check().context("invalid configuration")?; Ok(Loaded { settings, origins: merged .into_iter() .map(|(name, entry)| (name, entry.origin)) .collect(), files, print_only: false, }) } #[cfg(test)] mod tests { use super::*; use serde_json::json; fn layer(values: JsonValue) -> Layer { values .as_object() .unwrap() .iter() .map(|(name, value)| { ( name.clone(), Entry { value: value.clone(), origin: Origin { provenance: Provenance::Argument, unset: false, imports: Vec::new(), templates: Vec::new(), written: None, }, }, ) }) .collect() } #[test] fn force_only_on_the_command_line() { let from = |provenance| Entry { value: json!(["swarmId"]), origin: Origin { provenance, unset: false, imports: Vec::new(), templates: Vec::new(), written: None, }, }; let mut merged = layer(json!({ "htpasswd": "u", "node-key-file": "node.key" })); merged.insert( "force".to_owned(), from(Provenance::File(PathBuf::from("continuum.toml"))), ); assert!(resolve(merged.clone(), Vec::new()).is_err()); merged.insert("force".to_owned(), from(Provenance::Argument)); assert!(resolve(merged, Vec::new()).is_ok()); } #[test] fn options_and_settings_agree() { // Every option is a Settings field, and every field an option. let loaded = resolve( layer(json!({ "htpasswd": "users", "node-key-file": "node.key" })), Vec::new(), ) .unwrap(); let JsonValue::Object(fields) = serde_json::to_value(&loaded.settings).unwrap() else { panic!() }; let mut names: Vec<&str> = fields.keys().map(String::as_str).collect(); let mut options: Vec<&str> = OPTIONS.iter().map(|spec| spec.name).collect(); names.sort_unstable(); options.sort_unstable(); assert_eq!(names, options); // Help defaults match the real ones. for spec in OPTIONS { if let Some(default) = spec.default { assert_eq!( from_text(spec.kind, default).unwrap(), fields[spec.name], "default of {}", spec.name ); } } command().debug_assert(); } #[test] fn later_sources_override_conflicts_and_unset() { let from = |provenance: Provenance, value: JsonValue| Entry { value, origin: Origin { provenance, unset: false, imports: Vec::new(), templates: Vec::new(), written: None, }, }; let file = |value| from(Provenance::File("/etc/continuum.toml".into()), value); let argument = |value| from(Provenance::Argument, value); let url = json!("http://[::1]:8989"); let mut merged = Layer::new(); overlay( &mut merged, "htpasswd".into(), file(json!("u")), Overlay::FileOverFile, ); overlay( &mut merged, "node-key-file".into(), file(json!("node.key")), Overlay::FileOverFile, ); overlay( &mut merged, "mycelium-key-file".into(), file(json!("k")), Overlay::FileOverFile, ); overlay( &mut merged, "alias".into(), file(json!("a")), Overlay::FileOverFile, ); // Both in one source: an error. let mut same = merged.clone(); overlay( &mut same, "mycelium-rest-url".into(), file(url.clone()), Overlay::FileOverFile, ); assert!(resolve(same, Vec::new()).is_err()); // The command line wins over the conflicting file value, and unsets. overlay( &mut merged, "mycelium-rest-url".into(), argument(url), Overlay::ArgumentOverFile, ); overlay( &mut merged, "alias".into(), unset_entry(Provenance::Argument), Overlay::ArgumentOverFile, ); let loaded = resolve(merged, Vec::new()).unwrap(); assert!(loaded.settings.mycelium_key_file.is_none()); assert!(loaded.settings.mycelium_rest_url.is_some()); assert!(loaded.settings.alias.is_none()); assert_eq!( loaded.report(Values::Interpolated)["options"]["alias"], json!({ "source": "argument", "unset": true, "value": null }) ); } #[test] fn report_hides_imported_values() { let mut values = layer( json!({ "htpasswd": "users", "node-key-file": "node.key", "alias": "secret-ish" }), ); values.get_mut("alias").unwrap().origin.imports = vec!["env:HOSTNAME".into()]; let report = resolve(values, Vec::new()) .unwrap() .report(Values::Interpolated); let options = &report["options"]; assert_eq!( options["htpasswd"], json!({ "source": "argument", "value": "users" }) ); assert_eq!( options["alias"], json!({ "source": "argument", "imported": ["env:HOSTNAME"] }) ); assert_eq!( options["neighbours"], json!({ "source": "default", "value": 12 }) ); assert!(options.get("debug-epoch").is_none()); } #[test] fn text_values() { assert_eq!(from_text(Kind::Integer, " 30 ").unwrap(), json!(30)); assert!(from_text(Kind::Integer, "-1").is_err()); assert_eq!( from_text(Kind::List, "a, b\nc,").unwrap(), json!(["a", "b", "c"]) ); assert_eq!(from_text(Kind::Flag, "yes").unwrap(), json!(true)); } /// `Settings::check` and the override rule name the same supervised-only options. #[test] fn supervised_only_options_are_the_conflicts() { let settings = resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key" })), Vec::new(), ) .unwrap() .settings; let names: Vec<&str> = settings .supervised_only() .iter() .map(|(name, _)| *name) .collect(); let (rest_url, supervised) = CONFLICTS[0]; assert_eq!(rest_url, ["mycelium-rest-url"]); assert_eq!(names, supervised); } #[test] fn checks() { assert!(resolve(layer(json!({})), Vec::new()).is_err()); // htpasswd is required assert!( resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "neighbours": 1 })), Vec::new() ) .is_err() ); assert!( resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "debug-epoch": 3 })), Vec::new() ) .is_err() ); assert!( resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "mycelium-rest-url": "http://[::1]:8989", "mycelium-key-file": "k", })), Vec::new() ) .is_err() ); assert!( resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "alias": " " })), Vec::new() ) .is_err() ); assert!( resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "keepalive": "x" })), Vec::new() ) .is_err() ); let transports = |names: JsonValue| { resolve( layer(json!({ "htpasswd": "u", "node-key-file": "node.key", "transports": names })), Vec::new(), ) }; assert!(transports(json!([])).is_err()); assert!(transports(json!(["pigeon"])).is_err()); assert!(transports(json!(["mycelium", "mycelium"])).is_err()); assert!(transports(json!(["mycelium"])).is_ok()); assert_eq!( transports(json!(["libp2p", "mycelium"])).is_ok(), cfg!(feature = "libp2p") ); } #[test] fn arguments_with_importers() { // SAFETY: tests in this module don't read this variable concurrently. unsafe { std::env::set_var("CONTINUUM_TEST_SEEDS", "[\"a\", \"b\"]") }; let matches = command() .try_get_matches_from([ "continuum", "--seeds", "env+json:CONTINUUM_TEST_SEEDS", "--seeds", "c,d", "--alias", "literal:env:x", "--debug-api", ]) .unwrap(); let layer = arguments(&matches).unwrap(); assert_eq!(layer["seeds"].value, json!(["a", "b", "c", "d"])); assert_eq!( layer["seeds"].origin.imports, ["env+json:CONTINUUM_TEST_SEEDS"] ); assert_eq!(layer["alias"].value, json!("env:x")); assert_eq!(layer["debug-api"].value, json!(true)); assert!(!layer.contains_key("neighbours")); } #[test] fn report_shows_interpolated_values() { // SAFETY: tests in this module don't read this variable concurrently. unsafe { std::env::set_var("CONTINUUM_TEST_SITE", "berlin") }; let matches = command() .try_get_matches_from([ "continuum", "--htpasswd", "users", "--node-key-file", "node.key", "--alias", "node-${env:CONTINUUM_TEST_SITE}", "--seeds", "a,b", "--seeds", "c-${env:CONTINUUM_TEST_SITE}", ]) .unwrap(); let layer = arguments(&matches).unwrap(); assert!(!layer["alias"].origin.is_imported()); assert_eq!(layer["alias"].shown("alias"), "\"node-berlin\""); let loaded = resolve(layer, Vec::new()).unwrap(); let report = loaded.report(Values::Interpolated); assert_eq!( report["options"]["alias"], json!({ "source": "argument", "templates": ["node-${env:CONTINUUM_TEST_SITE}"], "value": "node-berlin", }) ); assert_eq!( report["options"]["seeds"]["value"], json!(["a", "b", "c-berlin"]) ); let report = loaded.report(Values::Original); assert_eq!( report["options"]["alias"]["value"], json!("node-${env:CONTINUUM_TEST_SITE}") ); assert_eq!( report["options"]["seeds"]["value"], json!(["a", "b", "c-${env:CONTINUUM_TEST_SITE}"]) ); // Without placeholders, both are the same. assert_eq!( report["options"]["neighbours"], json!({ "source": "default", "value": 12 }) ); } #[test] fn secrets_are_never_shown() { let values = layer( json!({ "htpasswd": "users", "node-key-file": "node.key", "mycelium-rest-url": "http://u:pw@h:8989" }), ); assert_eq!( values["mycelium-rest-url"].shown("mycelium-rest-url"), "" ); let loaded = resolve(values, Vec::new()).unwrap(); for mode in [Values::Interpolated, Values::Original] { assert_eq!( loaded.report(mode)["options"]["mycelium-rest-url"], json!({ "source": "argument", "secret": true }) ); } } }