diff options
| author | tv <tv@krebsco.de> | 2026-10-24 15:11:37 +0200 |
|---|---|---|
| committer | tv <tv@krebsco.de> | 2026-10-24 15:11:37 +0200 |
| commit | e5d590f4e927a18fbed334a3d90650015ed75106 (patch) | |
| tree | 81483f982a9f719ded1d60b2dd3caa97e6a53a3a | |
| parent | e66c24101a0a93e49c3959dc6094879d51e823e8 (diff) | |
add configuration files
| -rw-r--r-- | Cargo.toml | 1 | ||||
| -rw-r--r-- | src/config/file.rs | 233 | ||||
| -rw-r--r-- | src/config/import.rs | 289 | ||||
| -rw-r--r-- | src/config/mod.rs | 728 | ||||
| -rw-r--r-- | src/main.rs | 118 |
5 files changed, 1262 insertions, 107 deletions
@@ -32,6 +32,7 @@ serde_json = "1.0" sha2 = "0.11" sysinfo = "0.39" thiserror = "2.0" +toml = "1.1" tokio = { version = "1.53", features = ["full"] } tokio-tungstenite = { version = "0.30", features = ["native-tls"] } tower = "0.5" diff --git a/src/config/file.rs b/src/config/file.rs new file mode 100644 index 0000000..fea601b --- /dev/null +++ b/src/config/file.rs @@ -0,0 +1,233 @@ +//! Configuration files: TOML (`.toml`) for people, JSON (`.json`) for generated +//! configuration, with the same keys: the long option names. Nested tables join their +//! keys with `-`, so `[mycelium] key-file = …` is `mycelium-key-file`. +//! +//! `include = [paths]` and `include-optional = [paths]` load other files first +//! (relative to this one); this file's own values win over theirs, and later includes +//! over earlier ones. A missing optional include is skipped; a cycle is an error. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, anyhow, bail}; +use serde_json::{Map as JsonMap, Value as JsonValue}; +use tracing::{debug, warn}; + +use super::import; +use super::{Entry, Layer, Origin, Provenance, spec}; + +const INCLUDE: &str = "include"; +const INCLUDE_OPTIONAL: &str = "include-optional"; + +/// The settings `path` makes, its includes resolved. +pub(super) fn load(path: &Path) -> Result<Layer> { + load_inner(path, &mut Vec::new()) +} + +fn load_inner(path: &Path, loading: &mut Vec<PathBuf>) -> Result<Layer> { + let canonical = path + .canonicalize() + .with_context(|| format!("configuration file {}", path.display()))?; + if loading.contains(&canonical) { + bail!("{} includes itself (through {:?})", path.display(), loading); + } + let dir = canonical + .parent() + .map(Path::to_owned) + .unwrap_or_else(|| PathBuf::from("/")); + + let mut root = parse(&canonical)?; + let includes = take_paths(&mut root, INCLUDE, &canonical)?; + let optional = take_paths(&mut root, INCLUDE_OPTIONAL, &canonical)?; + + loading.push(canonical.clone()); + let mut layer = Layer::new(); + for (include, required) in includes + .into_iter() + .map(|p| (p, true)) + .chain(optional.into_iter().map(|p| (p, false))) + { + let include = dir.join(include); + if !required && !include.exists() { + debug!(file = %include.display(), "Optional configuration file is missing; skipped"); + continue; + } + let included = load_inner(&include, loading) + .with_context(|| format!("included from {}", canonical.display()))?; + for (name, entry) in included { + if let Some(previous) = layer.get(&name) { + warn!( + option = %name, + value = %entry.shown(), + from = %entry.origin, + overrides = %previous.shown(), + overridden_from = %previous.origin, + "Two included configuration files set the same option; the later one wins" + ); + } + layer.insert(name, entry); + } + } + loading.pop(); + + let mut own = Vec::new(); + flatten(&root, "", &dir, &mut own).with_context(|| format!("in {}", canonical.display()))?; + for (name, value) in own { + let spec = spec(&name) + .ok_or_else(|| anyhow!("unknown option {name:?} in {}", canonical.display()))?; + let (value, imports) = super::resolve_file_value(spec, value, &dir) + .with_context(|| format!("option {name} in {}", canonical.display()))?; + let entry = Entry { + value, + origin: Origin { + provenance: Provenance::File(canonical.clone()), + imports, + }, + }; + if let Some(previous) = layer.get(&name) { + debug!( + option = %name, + from = %entry.origin, + overrides = %previous.origin, + "Configuration file overrides an included one" + ); + } + layer.insert(name, entry); + } + Ok(layer) +} + +fn parse(path: &Path) -> Result<JsonMap<String, JsonValue>> { + let text = + std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; + let value: JsonValue = match path.extension().and_then(|e| e.to_str()) { + Some("toml") => { + let value: toml::Value = + toml::from_str(&text).with_context(|| format!("parsing {}", path.display()))?; + serde_json::to_value(value).with_context(|| format!("converting {}", path.display()))? + } + Some("json") => { + serde_json::from_str(&text).with_context(|| format!("parsing {}", path.display()))? + } + _ => bail!( + "{}: unknown configuration format (use .toml or .json)", + path.display() + ), + }; + match value { + JsonValue::Object(map) => Ok(map), + _ => bail!("{}: the top level must be a table", path.display()), + } +} + +fn take_paths( + root: &mut JsonMap<String, JsonValue>, + key: &str, + path: &Path, +) -> Result<Vec<String>> { + match root.remove(key) { + None => Ok(Vec::new()), + Some(JsonValue::String(single)) => Ok(vec![single]), + Some(JsonValue::Array(items)) => items + .into_iter() + .map(|item| match item { + JsonValue::String(item) => Ok(item), + other => Err(anyhow!( + "{}: {key} entries must be paths, not {other}", + path.display() + )), + }) + .collect(), + Some(other) => bail!( + "{}: {key} must be a list of paths, not {other}", + path.display() + ), + } +} + +/// Option names and values, with nested tables joined by `-`. Importer tables are +/// values, not nesting. +fn flatten( + table: &JsonMap<String, JsonValue>, + prefix: &str, + dir: &Path, + out: &mut Vec<(String, JsonValue)>, +) -> Result<()> { + for (key, value) in table { + let name = if prefix.is_empty() { + key.clone() + } else { + format!("{prefix}-{key}") + }; + match value { + JsonValue::Object(nested) if import::parse_table(nested, dir)?.is_none() => { + flatten(nested, &name, dir, out)? + } + _ => out.push((name, value.clone())), + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn write(dir: &Path, name: &str, text: &str) -> PathBuf { + let path = dir.join(name); + std::fs::write(&path, text).unwrap(); + path + } + + fn scratch(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("continuum-config-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn includes_tables_and_importers() { + let dir = scratch("includes"); + write(&dir, "key-path", "/run/secrets/swarm-key\n"); + write( + &dir, + "public.toml", + "neighbours = 8\nalias = \"public\"\n[mycelium]\npeers = [\"tls://a:9651\"]\n", + ); + write( + &dir, + "secrets.json", + r#"{ "swarm-key-file": { "file": "key-path" } }"#, + ); + let main = write( + &dir, + "main.toml", + "include = [\"public.toml\", \"secrets.json\"]\ninclude-optional = [\"missing.toml\"]\nalias = \"main\"\n", + ); + let layer = load(&main).unwrap(); + assert_eq!(layer["neighbours"].value, json!(8)); + assert_eq!(layer["alias"].value, json!("main")); + assert_eq!(layer["mycelium-peers"].value, json!(["tls://a:9651"])); + assert_eq!( + layer["swarm-key-file"].value, + json!("/run/secrets/swarm-key") + ); + assert!(layer["swarm-key-file"].origin.is_imported()); + assert!(layer["alias"].origin.to_string().ends_with("main.toml")); + } + + #[test] + fn errors() { + let dir = scratch("errors"); + let unknown = write(&dir, "unknown.toml", "nieghbours = 3\n"); + assert!(load(&unknown).is_err()); + let cycle = write(&dir, "cycle.toml", "include = [\"cycle.toml\"]\n"); + assert!(load(&cycle).is_err()); + let missing = write(&dir, "missing.toml", "include = [\"nope.toml\"]\n"); + assert!(load(&missing).is_err()); + let format = write(&dir, "config.yaml", "alias: x\n"); + assert!(load(&format).is_err()); + } +} diff --git a/src/config/import.rs b/src/config/import.rs new file mode 100644 index 0000000..895204a --- /dev/null +++ b/src/config/import.rs @@ -0,0 +1,289 @@ +//! Importers: values taken from elsewhere instead of written out, so a configuration +//! can say where a value comes from without holding it (secrets, host facts). +//! +//! * `env:NAME`: an environment variable; +//! * `file:PATH`: a file's contents; +//! * `cmd:COMMAND`: the standard output of a shell command (`sh -c`). +//! +//! The content is text (trailing line breaks removed), converted like a value written +//! on the command line; with `+json` (`env+json:NAME`, …) it is parsed as JSON instead. +//! +//! On the command line, a value with one of these prefixes is imported; `literal:` +//! passes the rest as is. In configuration files, an importer is a table: +//! `{ env = "NAME" }`, `{ file = "PATH" }`, `{ command = "COMMAND" }` (or an argument +//! list, run without a shell), each with an optional `format = "json"`. Relative paths +//! are relative to the file, and commands run in its directory. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, anyhow, bail}; +use serde_json::{Map as JsonMap, Value as JsonValue}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Format { + Text, + Json, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Source { + Env(String), + File(PathBuf), + Shell { + command: String, + dir: Option<PathBuf>, + }, + Argv { + argv: Vec<String>, + dir: Option<PathBuf>, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Importer { + pub(crate) source: Source, + pub(crate) format: Format, +} + +/// What a command-line value is: imported, or literal. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Raw<'a> { + Import(Importer), + Literal(&'a str), +} + +/// Reads a command-line value. +pub(crate) fn parse_argument(raw: &str) -> Raw<'_> { + if let Some(rest) = raw.strip_prefix("literal:") { + return Raw::Literal(rest); + } + let Some((prefix, rest)) = raw.split_once(':') else { + return Raw::Literal(raw); + }; + let (kind, format) = match prefix.split_once('+') { + None => (prefix, Format::Text), + Some((kind, "json")) => (kind, Format::Json), + Some(_) => return Raw::Literal(raw), + }; + let source = match kind { + "env" => Source::Env(rest.to_owned()), + "file" => Source::File(PathBuf::from(rest)), + "cmd" => Source::Shell { + command: rest.to_owned(), + dir: None, + }, + _ => return Raw::Literal(raw), + }; + Raw::Import(Importer { source, format }) +} + +/// Reads an importer table from a file in `dir`; None if `object` isn't one. +pub(crate) fn parse_table( + object: &JsonMap<String, JsonValue>, + dir: &Path, +) -> Result<Option<Importer>> { + let sources = ["env", "file", "command"] + .into_iter() + .filter(|key| object.contains_key(*key)) + .count(); + if sources == 0 { + return Ok(None); + } + if sources > 1 { + bail!("an importer takes exactly one of env, file and command"); + } + if let Some(unknown) = object + .keys() + .find(|key| !["env", "file", "command", "format"].contains(&key.as_str())) + { + bail!("unknown importer key {unknown:?}"); + } + let format = match object.get("format") { + None => Format::Text, + Some(JsonValue::String(format)) if format == "text" => Format::Text, + Some(JsonValue::String(format)) if format == "json" => Format::Json, + Some(other) => bail!("importer format must be \"text\" or \"json\", not {other}"), + }; + let text = |key: &str| -> Result<String> { + match object.get(key) { + Some(JsonValue::String(value)) => Ok(value.clone()), + _ => Err(anyhow!("importer {key} must be a string")), + } + }; + let source = if object.contains_key("env") { + Source::Env(text("env")?) + } else if object.contains_key("file") { + Source::File(dir.join(text("file")?)) + } else { + match &object["command"] { + JsonValue::String(command) => Source::Shell { + command: command.clone(), + dir: Some(dir.to_owned()), + }, + JsonValue::Array(items) => { + let argv = items + .iter() + .map(|item| item.as_str().map(str::to_owned)) + .collect::<Option<Vec<_>>>() + .filter(|argv| !argv.is_empty()) + .ok_or_else(|| { + anyhow!("importer command must be a string or a non-empty list of strings") + })?; + Source::Argv { + argv, + dir: Some(dir.to_owned()), + } + } + _ => bail!("importer command must be a string or a non-empty list of strings"), + } + }; + Ok(Some(Importer { source, format })) +} + +impl Importer { + /// Shown instead of the value (which may be secret), e.g. `env:HOSTNAME`. + pub(crate) fn describe(&self) -> String { + let format = match self.format { + Format::Text => "", + Format::Json => "+json", + }; + match &self.source { + Source::Env(name) => format!("env{format}:{name}"), + Source::File(path) => format!("file{format}:{}", path.display()), + Source::Shell { command, .. } => format!("cmd{format}:{command}"), + Source::Argv { argv, .. } => format!("cmd{format}:{}", argv.join(" ")), + } + } + + /// The imported content: text, or JSON. + pub(crate) fn fetch(&self) -> Result<Imported> { + let text = match &self.source { + Source::Env(name) => std::env::var(name).with_context(|| { + format!("environment variable {name} is not set (or not UTF-8)") + })?, + Source::File(path) => std::fs::read_to_string(path) + .with_context(|| format!("reading {}", path.display()))?, + Source::Shell { command, dir } => { + let mut process = Command::new("sh"); + process.arg("-c").arg(command); + run(process, dir.as_deref(), command)? + } + Source::Argv { argv, dir } => { + let mut process = Command::new(&argv[0]); + process.args(&argv[1..]); + run(process, dir.as_deref(), &argv.join(" "))? + } + }; + let text = text.trim_end_matches(['\n', '\r']); + Ok(match self.format { + Format::Text => Imported::Text(text.to_owned()), + Format::Json => Imported::Json( + serde_json::from_str(text) + .with_context(|| format!("{} is not valid JSON", self.describe()))?, + ), + }) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum Imported { + Text(String), + Json(JsonValue), +} + +fn run(mut process: Command, dir: Option<&Path>, shown: &str) -> Result<String> { + if let Some(dir) = dir { + process.current_dir(dir); + } + let output = process + .output() + .with_context(|| format!("running {shown}"))?; + if !output.status.success() { + bail!( + "{shown} failed ({}): {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + } + String::from_utf8(output.stdout).with_context(|| format!("output of {shown} is not UTF-8")) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn command_line_values() { + assert_eq!(parse_argument("plain"), Raw::Literal("plain")); + assert_eq!( + parse_argument("tls://host:9651"), + Raw::Literal("tls://host:9651") + ); + assert_eq!(parse_argument("literal:env:X"), Raw::Literal("env:X")); + assert_eq!( + parse_argument("env+json:SEEDS"), + Raw::Import(Importer { + source: Source::Env("SEEDS".into()), + format: Format::Json + }) + ); + assert_eq!( + parse_argument("cmd:hostname -f"), + Raw::Import(Importer { + source: Source::Shell { + command: "hostname -f".into(), + dir: None + }, + format: Format::Text + }) + ); + assert_eq!(parse_argument("env+yaml:X"), Raw::Literal("env+yaml:X")); + } + + #[test] + fn tables() { + let dir = Path::new("/etc/continuum"); + let table = |value: JsonValue| parse_table(value.as_object().unwrap(), dir); + assert_eq!(table(json!({ "x": 1 })).unwrap(), None); + assert_eq!( + table(json!({ "file": "key", "format": "json" })).unwrap(), + Some(Importer { + source: Source::File("/etc/continuum/key".into()), + format: Format::Json + }) + ); + assert!(table(json!({ "env": "A", "file": "b" })).is_err()); + assert!(table(json!({ "env": "A", "extra": 1 })).is_err()); + assert!(table(json!({ "command": [] })).is_err()); + } + + #[test] + fn fetching() { + let echo = Importer { + source: Source::Argv { + argv: vec!["printf".into(), "[1, 2]\\n\\n".into()], + dir: None, + }, + format: Format::Json, + }; + assert_eq!(echo.fetch().unwrap(), Imported::Json(json!([1, 2]))); + let shell = Importer { + source: Source::Shell { + command: "echo hello".into(), + dir: None, + }, + format: Format::Text, + }; + assert_eq!(shell.fetch().unwrap(), Imported::Text("hello".into())); + let failing = Importer { + source: Source::Shell { + command: "exit 3".into(), + dir: None, + }, + format: Format::Text, + }; + assert!(failing.fetch().is_err()); + } +} diff --git a/src/config/mod.rs b/src/config/mod.rs new file mode 100644 index 0000000..f042fb8 --- /dev/null +++ b/src/config/mod.rs @@ -0,0 +1,728 @@ +//! Configuration: built-in defaults, then configuration files (`--config`, in order), +//! then command-line arguments. Every option can be set in each. +//! +//! An argument that replaces a value from a file is logged (info), as is a file that +//! replaces one from another file (debug; warning between two includes of one file). +//! Values can be imported instead of written out (`import`): imported values are never +//! logged, only where they come from. +//! +//! All options are described once, in [`OPTIONS`]: the command line, the file keys and +//! the conversion of text values all derive from it. The merged values are then read +//! into [`Settings`] and checked. + +mod file; +mod import; + +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, anyhow, bail}; +use clap::parser::ValueSource; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use serde::{Deserialize, Serialize}; +use serde_json::Value as JsonValue; +use tracing::{debug, info}; + +use import::{Imported, Raw}; + +/// How an option's value is written as text (command line, text imports). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Kind { + /// `true`/`false`; on the command line, present or not. + Flag, + Text, + Integer, + /// Comma- or line-separated; repeatable on the command line. + List, +} + +#[derive(Debug)] +pub(crate) struct OptionSpec { + pub(crate) name: &'static str, + pub(crate) kind: Kind, + pub(crate) value_name: &'static str, + /// For `--help` only; the effective defaults are in [`Settings`]. + pub(crate) default: Option<&'static str>, + pub(crate) help: &'static str, + pub(crate) hidden: bool, +} + +const fn option( + name: &'static str, + kind: Kind, + value_name: &'static str, + default: Option<&'static str>, + help: &'static str, +) -> OptionSpec { + OptionSpec { + name, + kind, + value_name, + default, + help, + hidden: false, + } +} + +/// Every option. Names are the long option names and the configuration file keys. +pub(crate) const OPTIONS: &[OptionSpec] = &[ + option( + "bind-addr", + Kind::Text, + "HOSTNAME:PORT", + Some("127.0.0.1:3000"), + "Address of the HTTP API.", + ), + option( + "htpasswd", + Kind::Text, + "PATH", + None, + "Users of the HTTP API (htpasswd file with Argon2 hashes). Required.", + ), + option( + "debug-api", + Kind::Flag, + "BOOL", + Some("false"), + "Enable the /debug endpoints (tests only).", + ), + option( + "seeds", + Kind::List, + "STRING,[STRING...]", + None, + "How to reach a node of the swarm to join: a Mycelium public key, a Mycelium IPv6 address, or a host name with an AAAA record for one. Without seeds, the node starts a new swarm (genesis).", + ), + option( + "swarm-key-file", + Kind::Text, + "PATH", + None, + "Pre-shared swarm key: only nodes that know it may join (raw bytes, at least 16; e.g. `head -c 32 /dev/urandom`). Without it, anyone who knows the swarm ID can.", + ), + option( + "alias", + Kind::Text, + "NAME", + None, + "A name for this node, shown next to its public key (/peers, metrics). Published in the swarm's member list; not unique, never an identity.", + ), + option( + "keepalive-secs", + Kind::Integer, + "SECONDS", + Some("60"), + "Keepalive interval; dormancy, resync and probe timings derive from it.", + ), + option( + "neighbours", + Kind::Integer, + "COUNT", + Some("12"), + "Neighbours each node keeps (at least 2). Swarms of up to this many + 1 nodes are a full mesh; larger ones sync over several hops.", + ), + option( + "electorate-window-days", + Kind::Integer, + "DAYS", + Some("7"), + "Members that renewed their alive stamp within this many days vote on remakes (a majority of them must agree). Matters on the node that coordinates a remake.", + ), + option( + "mycelium-rest-url", + Kind::Text, + "SCHEME://HOSTNAME:PORT", + None, + "Use a Mycelium daemon running elsewhere (its REST API) instead of supervising one. Excludes the other mycelium options except mycelium-peers.", + ), + option( + "mycelium-key-file", + Kind::Text, + "PATH", + None, + "Private key of the supervised Mycelium daemon (its identity); a new one each start if absent.", + ), + option( + "mycelium-network-key-file", + Kind::Text, + "PATH", + None, + "Key of a private Mycelium network for the supervised daemon.", + ), + option( + "mycelium-network-name", + Kind::Text, + "NAME", + None, + "Name of that private Mycelium network.", + ), + option( + "mycelium-tcp-listen-port", + Kind::Integer, + "PORT", + None, + "TCP port the supervised Mycelium daemon listens on for overlay peers; a free port if absent.", + ), + option( + "mycelium-peers", + Kind::List, + "URL,[URL...]", + Some("tls://127.0.0.1:9651"), + "Peers of the supervised Mycelium daemon (overlay links such as `tls://host:9651`), not continuum peers.", + ), + OptionSpec { + name: "debug-epoch", + kind: Kind::Integer, + value_name: "GENERATION", + default: None, + help: "Test hook: start on a non-genesis epoch of this generation (with a fixed dummy id). Needs debug-api.", + hidden: true, + }, +]; + +pub(crate) fn spec(name: &str) -> Option<&'static OptionSpec> { + OPTIONS.iter().find(|spec| spec.name == name) +} + +/// The effective configuration. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub(crate) struct Settings { + #[serde(default = "default_bind_addr")] + pub(crate) bind_addr: SocketAddr, + pub(crate) htpasswd: String, + #[serde(default)] + pub(crate) debug_api: bool, + #[serde(default)] + pub(crate) seeds: Vec<String>, + pub(crate) swarm_key_file: Option<PathBuf>, + pub(crate) alias: Option<String>, + #[serde(default = "default_keepalive_secs")] + pub(crate) keepalive_secs: u64, + #[serde(default = "default_neighbours")] + pub(crate) neighbours: usize, + #[serde(default = "default_electorate_window_days")] + pub(crate) electorate_window_days: u64, + pub(crate) mycelium_rest_url: Option<String>, + pub(crate) mycelium_key_file: Option<PathBuf>, + pub(crate) mycelium_network_key_file: Option<PathBuf>, + pub(crate) mycelium_network_name: Option<String>, + pub(crate) mycelium_tcp_listen_port: Option<u16>, + #[serde(default = "default_mycelium_peers")] + pub(crate) mycelium_peers: Vec<String>, + pub(crate) debug_epoch: Option<u64>, +} + +fn default_bind_addr() -> SocketAddr { + SocketAddr::from(([127, 0, 0, 1], 3000)) +} +fn default_keepalive_secs() -> u64 { + 60 +} +fn default_neighbours() -> usize { + 12 +} +fn default_electorate_window_days() -> u64 { + 7 +} +fn default_mycelium_peers() -> Vec<String> { + vec!["tls://127.0.0.1:9651".to_owned()] +} + +impl Settings { + /// `mycelium-rest-url`, parsed (checked when loading). + pub(crate) fn mycelium_rest_url(&self) -> Option<url::Url> { + self.mycelium_rest_url + .as_deref() + .and_then(|url| url::Url::parse(url).ok()) + } + + fn check(&self) -> Result<()> { + if let Some(url) = &self.mycelium_rest_url { + url::Url::parse(url).with_context(|| format!("mycelium-rest-url {url:?}"))?; + } + if self.mycelium_rest_url.is_some() { + let supervised_only = [ + ("mycelium-key-file", self.mycelium_key_file.is_some()), + ( + "mycelium-network-key-file", + self.mycelium_network_key_file.is_some(), + ), + ( + "mycelium-network-name", + self.mycelium_network_name.is_some(), + ), + ( + "mycelium-tcp-listen-port", + self.mycelium_tcp_listen_port.is_some(), + ), + ]; + if let Some((name, _)) = supervised_only.iter().find(|(_, set)| *set) { + bail!( + "{name} is for a supervised Mycelium daemon and can't be combined with mycelium-rest-url" + ); + } + } + if self.debug_epoch.is_some() && !self.debug_api { + bail!("debug-epoch needs debug-api"); + } + if self.neighbours < 2 { + bail!("neighbours must be at least 2"); + } + if self.keepalive_secs == 0 { + bail!("keepalive-secs must be at least 1"); + } + if self.electorate_window_days == 0 { + bail!("electorate-window-days must be at least 1"); + } + if let Some(alias) = &self.alias { + check_alias(alias)?; + } + Ok(()) + } +} + +/// Up to 64 characters, no control characters, not blank. +fn check_alias(alias: &str) -> Result<()> { + if alias.trim().is_empty() { + bail!("alias must not be blank"); + } + if alias.chars().count() > 64 { + bail!("alias must be at most 64 characters"); + } + if alias.chars().any(char::is_control) { + bail!("alias must not contain control characters"); + } + Ok(()) +} + +// --- values and where they came from ------------------------------------------- + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Provenance { + File(PathBuf), + Argument, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Origin { + pub(crate) provenance: Provenance, + /// Importers the value (or some list elements) came from, e.g. `env:HOSTNAME`. + pub(crate) imports: Vec<String>, +} + +impl Origin { + pub(crate) fn is_imported(&self) -> bool { + !self.imports.is_empty() + } +} + +impl std::fmt::Display for Origin { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.provenance { + Provenance::File(path) => write!(f, "{}", path.display())?, + Provenance::Argument => write!(f, "the command line")?, + } + if self.is_imported() { + write!(f, " ({})", self.imports.join(", "))?; + } + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct Entry { + pub(crate) value: JsonValue, + pub(crate) origin: Origin, +} + +impl Entry { + /// The value for logs: imported values are never shown. + pub(crate) fn shown(&self) -> String { + if self.origin.is_imported() { + "<imported>".to_owned() + } else { + self.value.to_string() + } + } +} + +/// Option name to value. +pub(crate) type Layer = BTreeMap<String, Entry>; + +/// The value of a text, by the option's kind. +fn from_text(kind: Kind, text: &str) -> Result<JsonValue> { + Ok(match kind { + Kind::Text => JsonValue::String(text.to_owned()), + Kind::Integer => { + let number: u64 = text + .trim() + .parse() + .map_err(|_| anyhow!("{text:?} is not a non-negative integer"))?; + JsonValue::from(number) + } + Kind::Flag => match text.trim() { + "true" | "yes" | "1" => JsonValue::Bool(true), + "false" | "no" | "0" => JsonValue::Bool(false), + _ => bail!("{text:?} is not true or false"), + }, + Kind::List => JsonValue::Array( + text.split([',', '\n']) + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(|item| JsonValue::String(item.to_owned())) + .collect(), + ), + }) +} + +fn from_import(kind: Kind, importer: &import::Importer) -> Result<JsonValue> { + match importer.fetch()? { + Imported::Text(text) => from_text(kind, &text), + Imported::Json(value) => Ok(value), + } +} + +/// A value from a file in `dir`: importer tables are resolved, also as list elements. +fn resolve_file_value( + spec: &OptionSpec, + value: JsonValue, + dir: &Path, +) -> Result<(JsonValue, Vec<String>)> { + match value { + JsonValue::Object(table) => match import::parse_table(&table, dir)? { + Some(importer) => Ok(( + from_import(spec.kind, &importer)?, + vec![importer.describe()], + )), + None => bail!("a table here must be an importer (env, file or command)"), + }, + JsonValue::Array(items) if spec.kind == Kind::List => { + let mut imports = Vec::new(); + let mut values = Vec::new(); + for item in items { + match item { + JsonValue::Object(table) => { + let importer = import::parse_table(&table, dir)?.ok_or_else(|| { + anyhow!("a table here must be an importer (env, file or command)") + })?; + match from_import(Kind::List, &importer)? { + JsonValue::Array(more) => values.extend(more), + other => values.push(other), + } + imports.push(importer.describe()); + } + other => values.push(other), + } + } + Ok((JsonValue::Array(values), imports)) + } + other => Ok((other, Vec::new())), + } +} + +/// The first option whose value alone doesn't fit (serde doesn't name it). +fn culprit(merged: &Layer) -> Option<(&String, &Entry)> { + merged.iter().find(|(name, entry)| { + let mut alone = serde_json::Map::new(); + alone.insert("htpasswd".to_owned(), JsonValue::String(String::new())); + alone.insert((*name).clone(), entry.value.clone()); + serde_json::from_value::<Settings>(JsonValue::Object(alone)).is_err() + }) +} + +// --- command line -------------------------------------------------------------- + +const CONFIG: &str = "config"; + +fn command() -> Command { + let mut command = Command::new("continuum") + .version(env!("CARGO_PKG_VERSION")) + .about("continuum: a replicated key-value store over Mycelium") + .after_help( + "Every option can also be set in a configuration file (--config); arguments win.\n\ + A value can be imported instead of written out: env:NAME, file:PATH or cmd:COMMAND \ + (as text), or env+json:, file+json:, cmd+json: (parsed as JSON). \ + literal:VALUE passes a value that starts with one of these prefixes.", + ) + .arg( + Arg::new(CONFIG) + .long(CONFIG) + .value_name("PATH") + .action(ArgAction::Append) + .help("Configuration file (.toml or .json); repeatable, later files win."), + ); + for spec in OPTIONS { + let help = match spec.default { + Some(default) => format!("{} [default: {default}]", spec.help), + None => spec.help.to_owned(), + }; + let arg = Arg::new(spec.name) + .long(spec.name) + .help(help) + .hide(spec.hidden); + command = command.arg(match spec.kind { + Kind::Flag => arg.action(ArgAction::SetTrue), + Kind::List => arg.value_name(spec.value_name).action(ArgAction::Append), + Kind::Text | Kind::Integer => arg.value_name(spec.value_name).action(ArgAction::Set), + }); + } + command +} + +/// The values given as arguments. +fn arguments(matches: &ArgMatches) -> Result<Layer> { + let mut layer = Layer::new(); + for spec in OPTIONS { + if matches.value_source(spec.name) != Some(ValueSource::CommandLine) { + continue; + } + let mut imports = Vec::new(); + let value = if spec.kind == Kind::Flag { + JsonValue::Bool(true) + } else { + let mut values = Vec::new(); + for raw in matches.get_many::<String>(spec.name).into_iter().flatten() { + let value = match import::parse_argument(raw) { + Raw::Literal(text) => from_text(spec.kind, text), + Raw::Import(importer) => { + imports.push(importer.describe()); + from_import(spec.kind, &importer) + } + } + .with_context(|| format!("--{}", spec.name))?; + values.push(value); + } + if spec.kind == Kind::List { + JsonValue::Array( + values + .into_iter() + .flat_map(|value| match value { + JsonValue::Array(items) => items, + other => vec![other], + }) + .collect(), + ) + } else { + values.pop().unwrap_or(JsonValue::Null) + } + }; + layer.insert( + spec.name.to_owned(), + Entry { + value, + origin: Origin { + provenance: Provenance::Argument, + imports, + }, + }, + ); + } + Ok(layer) +} + +/// The configuration, and where each value set came from (absent: default). +#[derive(Debug, Clone)] +pub(crate) struct Loaded { + pub(crate) settings: Settings, + pub(crate) origins: BTreeMap<String, Origin>, + pub(crate) files: Vec<PathBuf>, +} + +/// Reads the command line and the configuration files it names. Exits on `--help`, +/// `--version` and command-line syntax errors, like any command. +pub(crate) fn load() -> Result<Loaded> { + let matches = command().get_matches(); + let files: Vec<PathBuf> = matches + .get_many::<String>(CONFIG) + .into_iter() + .flatten() + .map(PathBuf::from) + .collect(); + + let mut merged = Layer::new(); + for path in &files { + for (name, entry) in file::load(path)? { + if let Some(previous) = merged.get(&name) { + debug!( + option = %name, + from = %entry.origin, + overrides = %previous.origin, + "Later configuration file overrides an earlier one" + ); + } + merged.insert(name, entry); + } + } + for (name, entry) in arguments(&matches)? { + if let Some(previous) = merged.get(&name) { + info!( + option = %name, + value = %entry.shown(), + overrides = %previous.shown(), + from = %previous.origin, + "Command-line argument overrides the configuration file" + ); + } + merged.insert(name, entry); + } + resolve(merged, files) +} + +fn resolve(merged: Layer, files: Vec<PathBuf>) -> Result<Loaded> { + let object: serde_json::Map<String, JsonValue> = merged + .iter() + .map(|(name, entry)| (name.clone(), entry.value.clone())) + .collect(); + let mut settings: Settings = + serde_json::from_value(JsonValue::Object(object)).map_err(|error| { + match culprit(&merged) { + Some((name, entry)) => anyhow!( + "invalid configuration: option {name} (set in {}): {error}", + entry.origin + ), + None => anyhow!("invalid configuration: {error}"), + } + })?; + settings.alias = settings.alias.map(|alias| alias.trim().to_owned()); + settings.check().context("invalid configuration")?; + Ok(Loaded { + settings, + origins: merged + .into_iter() + .map(|(name, entry)| (name, entry.origin)) + .collect(), + files, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn layer(values: JsonValue) -> Layer { + values + .as_object() + .unwrap() + .iter() + .map(|(name, value)| { + ( + name.clone(), + Entry { + value: value.clone(), + origin: Origin { + provenance: Provenance::Argument, + imports: Vec::new(), + }, + }, + ) + }) + .collect() + } + + #[test] + fn options_and_settings_agree() { + // Every option is a Settings field, and every field an option. + let loaded = resolve(layer(json!({ "htpasswd": "users" })), Vec::new()).unwrap(); + let JsonValue::Object(fields) = serde_json::to_value(&loaded.settings).unwrap() else { + panic!() + }; + let mut names: Vec<&str> = fields.keys().map(String::as_str).collect(); + let mut options: Vec<&str> = OPTIONS.iter().map(|spec| spec.name).collect(); + names.sort_unstable(); + options.sort_unstable(); + assert_eq!(names, options); + + // Help defaults match the real ones. + for spec in OPTIONS { + if let Some(default) = spec.default { + assert_eq!( + from_text(spec.kind, default).unwrap(), + fields[spec.name], + "default of {}", + spec.name + ); + } + } + command().debug_assert(); + } + + #[test] + fn text_values() { + assert_eq!(from_text(Kind::Integer, " 30 ").unwrap(), json!(30)); + assert!(from_text(Kind::Integer, "-1").is_err()); + assert_eq!( + from_text(Kind::List, "a, b\nc,").unwrap(), + json!(["a", "b", "c"]) + ); + assert_eq!(from_text(Kind::Flag, "yes").unwrap(), json!(true)); + } + + #[test] + fn checks() { + assert!(resolve(layer(json!({})), Vec::new()).is_err()); // htpasswd is required + assert!( + resolve( + layer(json!({ "htpasswd": "u", "neighbours": 1 })), + Vec::new() + ) + .is_err() + ); + assert!( + resolve( + layer(json!({ "htpasswd": "u", "debug-epoch": 3 })), + Vec::new() + ) + .is_err() + ); + assert!( + resolve( + layer(json!({ + "htpasswd": "u", + "mycelium-rest-url": "http://[::1]:8989", + "mycelium-key-file": "k", + })), + Vec::new() + ) + .is_err() + ); + assert!(resolve(layer(json!({ "htpasswd": "u", "alias": " " })), Vec::new()).is_err()); + assert!( + resolve( + layer(json!({ "htpasswd": "u", "keepalive-secs": "x" })), + Vec::new() + ) + .is_err() + ); + } + + #[test] + fn arguments_with_importers() { + // SAFETY: tests in this module don't read this variable concurrently. + unsafe { std::env::set_var("CONTINUUM_TEST_SEEDS", "[\"a\", \"b\"]") }; + let matches = command() + .try_get_matches_from([ + "continuum", + "--seeds", + "env+json:CONTINUUM_TEST_SEEDS", + "--seeds", + "c,d", + "--alias", + "literal:env:x", + "--debug-api", + ]) + .unwrap(); + let layer = arguments(&matches).unwrap(); + assert_eq!(layer["seeds"].value, json!(["a", "b", "c", "d"])); + assert_eq!( + layer["seeds"].origin.imports, + ["env+json:CONTINUUM_TEST_SEEDS"] + ); + assert_eq!(layer["alias"].value, json!("env:x")); + assert_eq!(layer["debug-api"].value, json!(true)); + assert!(!layer.contains_key("neighbours")); + } +} diff --git a/src/main.rs b/src/main.rs index f6c7b5d..f89cd13 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,21 +5,18 @@ use std::collections::HashMap; use std::future::IntoFuture; -use std::net::SocketAddr; -use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::time::Duration; use anyhow::{Context, Result}; use automerge::AutoCommit; -use clap::Parser; use tokio::sync::{RwLock, broadcast, mpsc, watch}; use tracing::{debug, error, info, trace, warn}; -use url::Url; mod api; mod backoff; +mod config; mod document; mod logging; mod remake; @@ -36,105 +33,12 @@ use transport::firewall::{Firewall, FirewallTransport}; use transport::mycelium::{self, MyceliumSetup, MyceliumSpawnConfig}; use transport::traffic::{CountingTransport, Traffic}; -#[derive(Parser, Debug)] -#[command(version, about = "continuum messaging system", long_about = None)] -struct Args { - #[arg(long)] - debug_api: bool, - - #[arg(long, value_name = "HOSTNAME:PORT", default_value = "127.0.0.1:3000")] - bind_addr: SocketAddr, - - #[arg( - long, - value_name = "SCHEME://HOSTNAME:PORT", - default_value = None, - conflicts_with_all = &["mycelium_key_file", "mycelium_network_key_file", "mycelium_network_name", "mycelium_tcp_listen_port"], - )] - mycelium_rest_url: Option<Url>, - - #[arg(long)] - pub mycelium_key_file: Option<PathBuf>, - - #[arg(long)] - pub mycelium_network_key_file: Option<PathBuf>, - - #[arg(long)] - pub mycelium_network_name: Option<String>, - - /// TCP port the supervised Mycelium daemon listens on for overlay peers; a free - /// port if absent. - #[arg(long, value_name = "PORT")] - pub mycelium_tcp_listen_port: Option<u16>, - - #[arg( - long, - use_value_delimiter = true, - default_value = "tls://127.0.0.1:9651" - )] - /// Peers of the supervised Mycelium daemon (overlay links such as - /// `tls://host:9651`), not continuum peers. - pub mycelium_peers: Vec<String>, - - /// Seeds: how to reach a node of the swarm to join, as a Mycelium public key, a - /// Mycelium IPv6 address, or a host name with an AAAA record for one. Without seeds, - /// the node starts a new swarm (genesis). - #[arg(long, value_name = "STRING,[STRING...]", value_delimiter = ',')] - seeds: Option<Vec<String>>, - - #[arg(long, value_name = "PATH")] - htpasswd: String, - - /// Pre-shared swarm key: only nodes that know it may join (raw bytes, at least 16; - /// e.g. `head -c 32 /dev/urandom`). Without it, anyone who knows the swarm ID can. - #[arg(long, value_name = "PATH")] - swarm_key_file: Option<PathBuf>, - - /// Keepalive interval; dormancy, resync and probe timings derive from it. - #[arg(long, value_name = "SECONDS", default_value_t = 60)] - keepalive_secs: u64, - - /// Neighbours each node keeps (at least 2). Swarms of up to this many + 1 nodes are - /// a full mesh; larger ones sync over several hops. - #[arg(long, value_name = "COUNT", default_value_t = 12)] - neighbours: usize, - - /// A name for this node, shown next to its public key (/peers, metrics). - /// Published in the swarm's member list; not unique, never an identity. - #[arg(long, value_name = "NAME", value_parser = parse_alias)] - alias: Option<String>, - - /// Members that renewed their alive stamp within this many days vote on remakes - /// (a majority of them must agree). Matters on the node that coordinates a remake. - #[arg(long, value_name = "DAYS", default_value_t = 7, value_parser = clap::value_parser!(u64).range(1..))] - electorate_window_days: u64, - - /// Test hook: start on a non-genesis epoch of this generation (with a fixed dummy - /// id), to check that nodes on different epochs refuse to sync. - #[arg(long, hide = true, requires = "debug_api", value_name = "GENERATION")] - debug_epoch: Option<u64>, -} - -/// Up to 64 characters, no control characters, not blank. -fn parse_alias(value: &str) -> Result<String, String> { - let alias = value.trim(); - if alias.is_empty() { - return Err("must not be blank".to_owned()); - } - if alias.chars().count() > 64 { - return Err("must be at most 64 characters".to_owned()); - } - if alias.chars().any(char::is_control) { - return Err("must not contain control characters".to_owned()); - } - Ok(alias.to_owned()) -} - #[tokio::main] async fn main() -> Result<()> { logging::init(); - let args = Args::parse(); + let config = config::load()?; + let args = &config.settings; // Read before anything starts, so a bad key file fails right away. let swarm_key = match &args.swarm_key_file { @@ -151,14 +55,14 @@ async fn main() -> Result<()> { let mycelium_setup = { let mut spawn_config = MyceliumSpawnConfig { - rest_url_opt: args.mycelium_rest_url, - key_file_opt: args.mycelium_key_file, - network_key_file: args.mycelium_network_key_file, + rest_url_opt: args.mycelium_rest_url(), + key_file_opt: args.mycelium_key_file.clone(), + network_key_file: args.mycelium_network_key_file.clone(), tcp_port: args.mycelium_tcp_listen_port, - peers: args.mycelium_peers, + peers: args.mycelium_peers.clone(), ..Default::default() }; - if let Some(net_name) = args.mycelium_network_name { + if let Some(net_name) = args.mycelium_network_name.clone() { spawn_config.network_name = net_name; } tokio::select! { @@ -178,7 +82,7 @@ async fn main() -> Result<()> { "Using Mycelium REST API" ); - let seeds: Vec<String> = args.seeds.unwrap_or_default(); + let seeds: Vec<String> = args.seeds.clone(); debug!(seed_count = seeds.len(), "Loaded bootstrap configuration"); let (sync_tx, sync_rx) = mpsc::unbounded_channel(); @@ -228,8 +132,8 @@ async fn main() -> Result<()> { remake: Arc::new(remake::RemakeState::default()), swarm_key, run_id: uuid::Uuid::new_v4().into_bytes(), - neighbours_k: args.neighbours.max(2), - alias: args.alias, + neighbours_k: args.neighbours, + alias: args.alias.clone(), electorate_window: Duration::from_secs(args.electorate_window_days * 24 * 3600), neighbours: Default::default(), pulling: Default::default(), |
