diff options
Diffstat (limited to 'src/config/mod.rs')
| -rw-r--r-- | src/config/mod.rs | 728 |
1 files changed, 728 insertions, 0 deletions
diff --git a/src/config/mod.rs b/src/config/mod.rs new file mode 100644 index 0000000..f042fb8 --- /dev/null +++ b/src/config/mod.rs @@ -0,0 +1,728 @@ +//! Configuration: built-in defaults, then configuration files (`--config`, in order), +//! then command-line arguments. Every option can be set in each. +//! +//! An argument that replaces a value from a file is logged (info), as is a file that +//! replaces one from another file (debug; warning between two includes of one file). +//! Values can be imported instead of written out (`import`): imported values are never +//! logged, only where they come from. +//! +//! All options are described once, in [`OPTIONS`]: the command line, the file keys and +//! the conversion of text values all derive from it. The merged values are then read +//! into [`Settings`] and checked. + +mod file; +mod import; + +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, anyhow, bail}; +use clap::parser::ValueSource; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use serde::{Deserialize, Serialize}; +use serde_json::Value as JsonValue; +use tracing::{debug, info}; + +use import::{Imported, Raw}; + +/// How an option's value is written as text (command line, text imports). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Kind { + /// `true`/`false`; on the command line, present or not. + Flag, + Text, + Integer, + /// Comma- or line-separated; repeatable on the command line. + List, +} + +#[derive(Debug)] +pub(crate) struct OptionSpec { + pub(crate) name: &'static str, + pub(crate) kind: Kind, + pub(crate) value_name: &'static str, + /// For `--help` only; the effective defaults are in [`Settings`]. + pub(crate) default: Option<&'static str>, + pub(crate) help: &'static str, + pub(crate) hidden: bool, +} + +const fn option( + name: &'static str, + kind: Kind, + value_name: &'static str, + default: Option<&'static str>, + help: &'static str, +) -> OptionSpec { + OptionSpec { + name, + kind, + value_name, + default, + help, + hidden: false, + } +} + +/// Every option. Names are the long option names and the configuration file keys. +pub(crate) const OPTIONS: &[OptionSpec] = &[ + option( + "bind-addr", + Kind::Text, + "HOSTNAME:PORT", + Some("127.0.0.1:3000"), + "Address of the HTTP API.", + ), + option( + "htpasswd", + Kind::Text, + "PATH", + None, + "Users of the HTTP API (htpasswd file with Argon2 hashes). Required.", + ), + option( + "debug-api", + Kind::Flag, + "BOOL", + Some("false"), + "Enable the /debug endpoints (tests only).", + ), + option( + "seeds", + Kind::List, + "STRING,[STRING...]", + None, + "How to reach a node of the swarm to join: a Mycelium public key, a Mycelium IPv6 address, or a host name with an AAAA record for one. Without seeds, the node starts a new swarm (genesis).", + ), + option( + "swarm-key-file", + Kind::Text, + "PATH", + None, + "Pre-shared swarm key: only nodes that know it may join (raw bytes, at least 16; e.g. `head -c 32 /dev/urandom`). Without it, anyone who knows the swarm ID can.", + ), + option( + "alias", + Kind::Text, + "NAME", + None, + "A name for this node, shown next to its public key (/peers, metrics). Published in the swarm's member list; not unique, never an identity.", + ), + option( + "keepalive-secs", + Kind::Integer, + "SECONDS", + Some("60"), + "Keepalive interval; dormancy, resync and probe timings derive from it.", + ), + option( + "neighbours", + Kind::Integer, + "COUNT", + Some("12"), + "Neighbours each node keeps (at least 2). Swarms of up to this many + 1 nodes are a full mesh; larger ones sync over several hops.", + ), + option( + "electorate-window-days", + Kind::Integer, + "DAYS", + Some("7"), + "Members that renewed their alive stamp within this many days vote on remakes (a majority of them must agree). Matters on the node that coordinates a remake.", + ), + option( + "mycelium-rest-url", + Kind::Text, + "SCHEME://HOSTNAME:PORT", + None, + "Use a Mycelium daemon running elsewhere (its REST API) instead of supervising one. Excludes the other mycelium options except mycelium-peers.", + ), + option( + "mycelium-key-file", + Kind::Text, + "PATH", + None, + "Private key of the supervised Mycelium daemon (its identity); a new one each start if absent.", + ), + option( + "mycelium-network-key-file", + Kind::Text, + "PATH", + None, + "Key of a private Mycelium network for the supervised daemon.", + ), + option( + "mycelium-network-name", + Kind::Text, + "NAME", + None, + "Name of that private Mycelium network.", + ), + option( + "mycelium-tcp-listen-port", + Kind::Integer, + "PORT", + None, + "TCP port the supervised Mycelium daemon listens on for overlay peers; a free port if absent.", + ), + option( + "mycelium-peers", + Kind::List, + "URL,[URL...]", + Some("tls://127.0.0.1:9651"), + "Peers of the supervised Mycelium daemon (overlay links such as `tls://host:9651`), not continuum peers.", + ), + OptionSpec { + name: "debug-epoch", + kind: Kind::Integer, + value_name: "GENERATION", + default: None, + help: "Test hook: start on a non-genesis epoch of this generation (with a fixed dummy id). Needs debug-api.", + hidden: true, + }, +]; + +pub(crate) fn spec(name: &str) -> Option<&'static OptionSpec> { + OPTIONS.iter().find(|spec| spec.name == name) +} + +/// The effective configuration. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub(crate) struct Settings { + #[serde(default = "default_bind_addr")] + pub(crate) bind_addr: SocketAddr, + pub(crate) htpasswd: String, + #[serde(default)] + pub(crate) debug_api: bool, + #[serde(default)] + pub(crate) seeds: Vec<String>, + pub(crate) swarm_key_file: Option<PathBuf>, + pub(crate) alias: Option<String>, + #[serde(default = "default_keepalive_secs")] + pub(crate) keepalive_secs: u64, + #[serde(default = "default_neighbours")] + pub(crate) neighbours: usize, + #[serde(default = "default_electorate_window_days")] + pub(crate) electorate_window_days: u64, + pub(crate) mycelium_rest_url: Option<String>, + pub(crate) mycelium_key_file: Option<PathBuf>, + pub(crate) mycelium_network_key_file: Option<PathBuf>, + pub(crate) mycelium_network_name: Option<String>, + pub(crate) mycelium_tcp_listen_port: Option<u16>, + #[serde(default = "default_mycelium_peers")] + pub(crate) mycelium_peers: Vec<String>, + pub(crate) debug_epoch: Option<u64>, +} + +fn default_bind_addr() -> SocketAddr { + SocketAddr::from(([127, 0, 0, 1], 3000)) +} +fn default_keepalive_secs() -> u64 { + 60 +} +fn default_neighbours() -> usize { + 12 +} +fn default_electorate_window_days() -> u64 { + 7 +} +fn default_mycelium_peers() -> Vec<String> { + vec!["tls://127.0.0.1:9651".to_owned()] +} + +impl Settings { + /// `mycelium-rest-url`, parsed (checked when loading). + pub(crate) fn mycelium_rest_url(&self) -> Option<url::Url> { + self.mycelium_rest_url + .as_deref() + .and_then(|url| url::Url::parse(url).ok()) + } + + fn check(&self) -> Result<()> { + if let Some(url) = &self.mycelium_rest_url { + url::Url::parse(url).with_context(|| format!("mycelium-rest-url {url:?}"))?; + } + if self.mycelium_rest_url.is_some() { + let supervised_only = [ + ("mycelium-key-file", self.mycelium_key_file.is_some()), + ( + "mycelium-network-key-file", + self.mycelium_network_key_file.is_some(), + ), + ( + "mycelium-network-name", + self.mycelium_network_name.is_some(), + ), + ( + "mycelium-tcp-listen-port", + self.mycelium_tcp_listen_port.is_some(), + ), + ]; + if let Some((name, _)) = supervised_only.iter().find(|(_, set)| *set) { + bail!( + "{name} is for a supervised Mycelium daemon and can't be combined with mycelium-rest-url" + ); + } + } + if self.debug_epoch.is_some() && !self.debug_api { + bail!("debug-epoch needs debug-api"); + } + if self.neighbours < 2 { + bail!("neighbours must be at least 2"); + } + if self.keepalive_secs == 0 { + bail!("keepalive-secs must be at least 1"); + } + if self.electorate_window_days == 0 { + bail!("electorate-window-days must be at least 1"); + } + if let Some(alias) = &self.alias { + check_alias(alias)?; + } + Ok(()) + } +} + +/// Up to 64 characters, no control characters, not blank. +fn check_alias(alias: &str) -> Result<()> { + if alias.trim().is_empty() { + bail!("alias must not be blank"); + } + if alias.chars().count() > 64 { + bail!("alias must be at most 64 characters"); + } + if alias.chars().any(char::is_control) { + bail!("alias must not contain control characters"); + } + Ok(()) +} + +// --- values and where they came from ------------------------------------------- + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Provenance { + File(PathBuf), + Argument, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Origin { + pub(crate) provenance: Provenance, + /// Importers the value (or some list elements) came from, e.g. `env:HOSTNAME`. + pub(crate) imports: Vec<String>, +} + +impl Origin { + pub(crate) fn is_imported(&self) -> bool { + !self.imports.is_empty() + } +} + +impl std::fmt::Display for Origin { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.provenance { + Provenance::File(path) => write!(f, "{}", path.display())?, + Provenance::Argument => write!(f, "the command line")?, + } + if self.is_imported() { + write!(f, " ({})", self.imports.join(", "))?; + } + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct Entry { + pub(crate) value: JsonValue, + pub(crate) origin: Origin, +} + +impl Entry { + /// The value for logs: imported values are never shown. + pub(crate) fn shown(&self) -> String { + if self.origin.is_imported() { + "<imported>".to_owned() + } else { + self.value.to_string() + } + } +} + +/// Option name to value. +pub(crate) type Layer = BTreeMap<String, Entry>; + +/// The value of a text, by the option's kind. +fn from_text(kind: Kind, text: &str) -> Result<JsonValue> { + Ok(match kind { + Kind::Text => JsonValue::String(text.to_owned()), + Kind::Integer => { + let number: u64 = text + .trim() + .parse() + .map_err(|_| anyhow!("{text:?} is not a non-negative integer"))?; + JsonValue::from(number) + } + Kind::Flag => match text.trim() { + "true" | "yes" | "1" => JsonValue::Bool(true), + "false" | "no" | "0" => JsonValue::Bool(false), + _ => bail!("{text:?} is not true or false"), + }, + Kind::List => JsonValue::Array( + text.split([',', '\n']) + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(|item| JsonValue::String(item.to_owned())) + .collect(), + ), + }) +} + +fn from_import(kind: Kind, importer: &import::Importer) -> Result<JsonValue> { + match importer.fetch()? { + Imported::Text(text) => from_text(kind, &text), + Imported::Json(value) => Ok(value), + } +} + +/// A value from a file in `dir`: importer tables are resolved, also as list elements. +fn resolve_file_value( + spec: &OptionSpec, + value: JsonValue, + dir: &Path, +) -> Result<(JsonValue, Vec<String>)> { + match value { + JsonValue::Object(table) => match import::parse_table(&table, dir)? { + Some(importer) => Ok(( + from_import(spec.kind, &importer)?, + vec![importer.describe()], + )), + None => bail!("a table here must be an importer (env, file or command)"), + }, + JsonValue::Array(items) if spec.kind == Kind::List => { + let mut imports = Vec::new(); + let mut values = Vec::new(); + for item in items { + match item { + JsonValue::Object(table) => { + let importer = import::parse_table(&table, dir)?.ok_or_else(|| { + anyhow!("a table here must be an importer (env, file or command)") + })?; + match from_import(Kind::List, &importer)? { + JsonValue::Array(more) => values.extend(more), + other => values.push(other), + } + imports.push(importer.describe()); + } + other => values.push(other), + } + } + Ok((JsonValue::Array(values), imports)) + } + other => Ok((other, Vec::new())), + } +} + +/// The first option whose value alone doesn't fit (serde doesn't name it). +fn culprit(merged: &Layer) -> Option<(&String, &Entry)> { + merged.iter().find(|(name, entry)| { + let mut alone = serde_json::Map::new(); + alone.insert("htpasswd".to_owned(), JsonValue::String(String::new())); + alone.insert((*name).clone(), entry.value.clone()); + serde_json::from_value::<Settings>(JsonValue::Object(alone)).is_err() + }) +} + +// --- command line -------------------------------------------------------------- + +const CONFIG: &str = "config"; + +fn command() -> Command { + let mut command = Command::new("continuum") + .version(env!("CARGO_PKG_VERSION")) + .about("continuum: a replicated key-value store over Mycelium") + .after_help( + "Every option can also be set in a configuration file (--config); arguments win.\n\ + A value can be imported instead of written out: env:NAME, file:PATH or cmd:COMMAND \ + (as text), or env+json:, file+json:, cmd+json: (parsed as JSON). \ + literal:VALUE passes a value that starts with one of these prefixes.", + ) + .arg( + Arg::new(CONFIG) + .long(CONFIG) + .value_name("PATH") + .action(ArgAction::Append) + .help("Configuration file (.toml or .json); repeatable, later files win."), + ); + for spec in OPTIONS { + let help = match spec.default { + Some(default) => format!("{} [default: {default}]", spec.help), + None => spec.help.to_owned(), + }; + let arg = Arg::new(spec.name) + .long(spec.name) + .help(help) + .hide(spec.hidden); + command = command.arg(match spec.kind { + Kind::Flag => arg.action(ArgAction::SetTrue), + Kind::List => arg.value_name(spec.value_name).action(ArgAction::Append), + Kind::Text | Kind::Integer => arg.value_name(spec.value_name).action(ArgAction::Set), + }); + } + command +} + +/// The values given as arguments. +fn arguments(matches: &ArgMatches) -> Result<Layer> { + let mut layer = Layer::new(); + for spec in OPTIONS { + if matches.value_source(spec.name) != Some(ValueSource::CommandLine) { + continue; + } + let mut imports = Vec::new(); + let value = if spec.kind == Kind::Flag { + JsonValue::Bool(true) + } else { + let mut values = Vec::new(); + for raw in matches.get_many::<String>(spec.name).into_iter().flatten() { + let value = match import::parse_argument(raw) { + Raw::Literal(text) => from_text(spec.kind, text), + Raw::Import(importer) => { + imports.push(importer.describe()); + from_import(spec.kind, &importer) + } + } + .with_context(|| format!("--{}", spec.name))?; + values.push(value); + } + if spec.kind == Kind::List { + JsonValue::Array( + values + .into_iter() + .flat_map(|value| match value { + JsonValue::Array(items) => items, + other => vec![other], + }) + .collect(), + ) + } else { + values.pop().unwrap_or(JsonValue::Null) + } + }; + layer.insert( + spec.name.to_owned(), + Entry { + value, + origin: Origin { + provenance: Provenance::Argument, + imports, + }, + }, + ); + } + Ok(layer) +} + +/// The configuration, and where each value set came from (absent: default). +#[derive(Debug, Clone)] +pub(crate) struct Loaded { + pub(crate) settings: Settings, + pub(crate) origins: BTreeMap<String, Origin>, + pub(crate) files: Vec<PathBuf>, +} + +/// Reads the command line and the configuration files it names. Exits on `--help`, +/// `--version` and command-line syntax errors, like any command. +pub(crate) fn load() -> Result<Loaded> { + let matches = command().get_matches(); + let files: Vec<PathBuf> = matches + .get_many::<String>(CONFIG) + .into_iter() + .flatten() + .map(PathBuf::from) + .collect(); + + let mut merged = Layer::new(); + for path in &files { + for (name, entry) in file::load(path)? { + if let Some(previous) = merged.get(&name) { + debug!( + option = %name, + from = %entry.origin, + overrides = %previous.origin, + "Later configuration file overrides an earlier one" + ); + } + merged.insert(name, entry); + } + } + for (name, entry) in arguments(&matches)? { + if let Some(previous) = merged.get(&name) { + info!( + option = %name, + value = %entry.shown(), + overrides = %previous.shown(), + from = %previous.origin, + "Command-line argument overrides the configuration file" + ); + } + merged.insert(name, entry); + } + resolve(merged, files) +} + +fn resolve(merged: Layer, files: Vec<PathBuf>) -> Result<Loaded> { + let object: serde_json::Map<String, JsonValue> = merged + .iter() + .map(|(name, entry)| (name.clone(), entry.value.clone())) + .collect(); + let mut settings: Settings = + serde_json::from_value(JsonValue::Object(object)).map_err(|error| { + match culprit(&merged) { + Some((name, entry)) => anyhow!( + "invalid configuration: option {name} (set in {}): {error}", + entry.origin + ), + None => anyhow!("invalid configuration: {error}"), + } + })?; + settings.alias = settings.alias.map(|alias| alias.trim().to_owned()); + settings.check().context("invalid configuration")?; + Ok(Loaded { + settings, + origins: merged + .into_iter() + .map(|(name, entry)| (name, entry.origin)) + .collect(), + files, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn layer(values: JsonValue) -> Layer { + values + .as_object() + .unwrap() + .iter() + .map(|(name, value)| { + ( + name.clone(), + Entry { + value: value.clone(), + origin: Origin { + provenance: Provenance::Argument, + imports: Vec::new(), + }, + }, + ) + }) + .collect() + } + + #[test] + fn options_and_settings_agree() { + // Every option is a Settings field, and every field an option. + let loaded = resolve(layer(json!({ "htpasswd": "users" })), Vec::new()).unwrap(); + let JsonValue::Object(fields) = serde_json::to_value(&loaded.settings).unwrap() else { + panic!() + }; + let mut names: Vec<&str> = fields.keys().map(String::as_str).collect(); + let mut options: Vec<&str> = OPTIONS.iter().map(|spec| spec.name).collect(); + names.sort_unstable(); + options.sort_unstable(); + assert_eq!(names, options); + + // Help defaults match the real ones. + for spec in OPTIONS { + if let Some(default) = spec.default { + assert_eq!( + from_text(spec.kind, default).unwrap(), + fields[spec.name], + "default of {}", + spec.name + ); + } + } + command().debug_assert(); + } + + #[test] + fn text_values() { + assert_eq!(from_text(Kind::Integer, " 30 ").unwrap(), json!(30)); + assert!(from_text(Kind::Integer, "-1").is_err()); + assert_eq!( + from_text(Kind::List, "a, b\nc,").unwrap(), + json!(["a", "b", "c"]) + ); + assert_eq!(from_text(Kind::Flag, "yes").unwrap(), json!(true)); + } + + #[test] + fn checks() { + assert!(resolve(layer(json!({})), Vec::new()).is_err()); // htpasswd is required + assert!( + resolve( + layer(json!({ "htpasswd": "u", "neighbours": 1 })), + Vec::new() + ) + .is_err() + ); + assert!( + resolve( + layer(json!({ "htpasswd": "u", "debug-epoch": 3 })), + Vec::new() + ) + .is_err() + ); + assert!( + resolve( + layer(json!({ + "htpasswd": "u", + "mycelium-rest-url": "http://[::1]:8989", + "mycelium-key-file": "k", + })), + Vec::new() + ) + .is_err() + ); + assert!(resolve(layer(json!({ "htpasswd": "u", "alias": " " })), Vec::new()).is_err()); + assert!( + resolve( + layer(json!({ "htpasswd": "u", "keepalive-secs": "x" })), + Vec::new() + ) + .is_err() + ); + } + + #[test] + fn arguments_with_importers() { + // SAFETY: tests in this module don't read this variable concurrently. + unsafe { std::env::set_var("CONTINUUM_TEST_SEEDS", "[\"a\", \"b\"]") }; + let matches = command() + .try_get_matches_from([ + "continuum", + "--seeds", + "env+json:CONTINUUM_TEST_SEEDS", + "--seeds", + "c,d", + "--alias", + "literal:env:x", + "--debug-api", + ]) + .unwrap(); + let layer = arguments(&matches).unwrap(); + assert_eq!(layer["seeds"].value, json!(["a", "b", "c", "d"])); + assert_eq!( + layer["seeds"].origin.imports, + ["env+json:CONTINUUM_TEST_SEEDS"] + ); + assert_eq!(layer["alias"].value, json!("env:x")); + assert_eq!(layer["debug-api"].value, json!(true)); + assert!(!layer.contains_key("neighbours")); + } +} |