diff options
| -rw-r--r-- | docs/openapi.yaml | 22 | ||||
| -rw-r--r-- | src/api/mod.rs | 16 | ||||
| -rw-r--r-- | src/config/file.rs | 14 | ||||
| -rw-r--r-- | src/config/import.rs | 6 | ||||
| -rw-r--r-- | src/config/mod.rs | 295 | ||||
| -rw-r--r-- | src/main.rs | 15 | ||||
| -rw-r--r-- | src/node.rs | 4 | ||||
| -rw-r--r-- | src/simtest/mod.rs | 2 | ||||
| -rw-r--r-- | src/state/mod.rs | 4 |
9 files changed, 308 insertions, 70 deletions
diff --git a/docs/openapi.yaml b/docs/openapi.yaml index bfb297d..3025146 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -407,7 +407,21 @@ paths: description: | Every option with its value and where it came from (`default`, `file` with the file, or `argument`), as `--print-config` prints it. Imported values (`env:`, - `file:`, `cmd:`) are left out; `imported` lists their importers. + `file:`, `cmd:`) are left out; `imported` lists their importers. Secrets are + left out however they were given (`secret`). Text with placeholders + (`${env:NAME}`) is shown as `values` says; `templates` lists the text as + written. + parameters: + - name: values + in: query + required: false + description: | + Text with placeholders as it ends up (`interpolated`), or as written + (`original`). + schema: + type: string + enum: [interpolated, original] + default: interpolated responses: '200': description: The configuration. @@ -436,6 +450,12 @@ paths: type: array items: type: string + templates: + type: array + items: + type: string + secret: + type: boolean value: {} example: files: [/etc/continuum.toml] diff --git a/src/api/mod.rs b/src/api/mod.rs index 20237e1..45eaec4 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -136,10 +136,20 @@ async fn openapi_spec() -> impl IntoResponse { ) } +#[derive(Debug, Default, serde::Deserialize)] +struct ConfigQuery { + /// Text with placeholders as it ends up, or as written. + #[serde(default)] + values: crate::config::Values, +} + /// `GET /admin/config`: the effective configuration with each value's source; -/// imported values are left out. -async fn get_config(State(state): State<AppState>) -> Json<JsonValue> { - Json((*state.config).clone()) +/// imported values and secrets are left out. +async fn get_config( + State(state): State<AppState>, + axum::extract::Query(query): axum::extract::Query<ConfigQuery>, +) -> Json<JsonValue> { + Json(state.config.get(query.values).clone()) } // --- shared helpers ---------------------------------------------------------- diff --git a/src/config/file.rs b/src/config/file.rs index 50b3d48..b30d147 100644 --- a/src/config/file.rs +++ b/src/config/file.rs @@ -16,7 +16,7 @@ use serde_json::{Map as JsonMap, Value as JsonValue}; use tracing::debug; use super::import; -use super::{Entry, Layer, Origin, Overlay, Provenance, overlay, spec, unset_entry}; +use super::{Entry, Layer, Overlay, Provenance, overlay, spec, unset_entry}; const INCLUDE: &str = "include"; const INCLUDE_OPTIONAL: &str = "include-optional"; @@ -80,14 +80,14 @@ fn load_inner(path: &Path, loading: &mut Vec<PathBuf>) -> Result<Layer> { for (name, value) in own { let spec = spec(&name) .ok_or_else(|| anyhow!("unknown option {name:?} in {}", canonical.display()))?; - let (value, imports) = super::resolve_file_value(spec, value, &dir) + let (value, sources) = super::resolve_file_value(spec, value, &dir) .with_context(|| format!("option {name} in {}", canonical.display()))?; let entry = Entry { - origin: Origin { - provenance: Provenance::File(canonical.clone()), - unset: value.is_null(), - imports, - }, + origin: sources.origin( + Provenance::File(canonical.clone()), + value.is_null(), + spec.kind, + ), value, }; overlay(&mut layer, name, entry, Overlay::FileOverFile); diff --git a/src/config/import.rs b/src/config/import.rs index a5b2a63..9f315f3 100644 --- a/src/config/import.rs +++ b/src/config/import.rs @@ -185,6 +185,12 @@ impl Importer { } } + /// Whether this is text with placeholders (`interpolated`) rather than an importer + /// of the whole value: its result is shown, the importers' are not. + pub(crate) fn is_interpolation(&self) -> bool { + matches!(self.source, Source::Template { .. }) + } + /// The imported content: text, or JSON. pub(crate) fn fetch(&self) -> Result<Imported> { let text = match &self.source { diff --git a/src/config/mod.rs b/src/config/mod.rs index fd8547d..928730e 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -51,6 +51,18 @@ pub(crate) struct OptionSpec { pub(crate) default: Option<&'static str>, pub(crate) help: &'static str, pub(crate) hidden: bool, + /// Its value is never shown (logs, `--print-config`, `GET /admin/config`), however + /// it was given. + pub(crate) secret: bool, +} + +impl OptionSpec { + const fn secret(self) -> Self { + Self { + secret: true, + ..self + } + } } const fn option( @@ -67,6 +79,7 @@ const fn option( default, help, hidden: false, + secret: false, } } @@ -214,7 +227,9 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ "SCHEME://HOSTNAME:PORT", None, "Use a Mycelium daemon running elsewhere (its REST API) instead of supervising one. Excludes the other mycelium options except mycelium-peers.", - ), + ) + // A URL may carry credentials. + .secret(), option( "mycelium-key-file", Kind::Text, @@ -257,6 +272,7 @@ pub(crate) const OPTIONS: &[OptionSpec] = &[ default: None, help: "Test hook: start on a non-genesis epoch of this generation (with a fixed dummy id). Needs debug-api.", hidden: true, + secret: false, }, ]; @@ -454,8 +470,14 @@ pub(crate) struct Origin { /// The option was unset here (back to its default): `--unset`, `unset = [...]` in a /// file, or `null` in a JSON file. pub(crate) unset: bool, - /// Importers the value (or some list elements) came from, e.g. `env:HOSTNAME`. + /// Importers the value (or some list elements) came from, e.g. `env:HOSTNAME`. The + /// value is never shown. pub(crate) imports: Vec<String>, + /// Text with placeholders the value (or some list elements) came from, e.g. + /// `node-${env:HOSTNAME}`. The value is shown. + pub(crate) templates: Vec<String>, + /// With placeholders: the value as written, placeholders and all. + pub(crate) written: Option<JsonValue>, } impl Origin { @@ -473,6 +495,7 @@ impl std::fmt::Display for Origin { if self.unset { write!(f, " (unset)")?; } + // Not the text with placeholders: of a secret, it may hold some of it. if self.is_imported() { write!(f, " ({})", self.imports.join(", "))?; } @@ -487,10 +510,12 @@ pub(crate) struct Entry { } impl Entry { - /// The value for logs: imported values are never shown. - pub(crate) fn shown(&self) -> String { + /// The value of `option` for logs: imported values and secrets are never shown. + pub(crate) fn shown(&self, option: &str) -> String { if self.origin.unset { "<default>".to_owned() + } else if spec(option).is_some_and(|spec| spec.secret) { + "<secret>".to_owned() } else if self.origin.is_imported() { "<imported>".to_owned() } else { @@ -510,6 +535,8 @@ fn unset_entry(provenance: Provenance) -> Entry { provenance, unset: true, imports: Vec::new(), + templates: Vec::new(), + written: None, }, } } @@ -559,9 +586,9 @@ pub(crate) fn overlay(layer: &mut Layer, name: String, entry: Entry, how: Overla let log = |message: &str, option: &str, previous: &Entry| match how { Overlay::IncludeOverInclude => warn!( option, - value = %entry.shown(), + value = %entry.shown(&name), from = %entry.origin, - overrides = %previous.shown(), + overrides = %previous.shown(option), overridden_from = %previous.origin, "{message}" ), @@ -573,8 +600,8 @@ pub(crate) fn overlay(layer: &mut Layer, name: String, entry: Entry, how: Overla ), Overlay::ArgumentOverFile => info!( option, - value = %entry.shown(), - overrides = %previous.shown(), + value = %entry.shown(&name), + overrides = %previous.shown(option), from = %previous.origin, "{message}" ), @@ -649,28 +676,73 @@ fn from_import(kind: Kind, importer: &import::Importer) -> Result<JsonValue> { } /// A value from a file in `dir`: importer tables are resolved, also as list elements. +/// Collects where the parts of a value came from, for its [`Origin`]. +#[derive(Default)] +struct Sources { + imports: Vec<String>, + templates: Vec<String>, + /// The parts as written: literal values, importers, text with placeholders. + written: Vec<JsonValue>, +} + +impl Sources { + /// A literal part (list values count element by element). + fn literal(&mut self, value: &JsonValue) { + match value { + JsonValue::Array(items) => self.written.extend(items.iter().cloned()), + other => self.written.push(other.clone()), + } + } + + /// A part from `importer`. + fn import(&mut self, importer: &import::Importer) { + let described = importer.describe(); + if importer.is_interpolation() { + self.templates.push(described.clone()); + } else { + self.imports.push(described.clone()); + } + self.written.push(JsonValue::String(described)); + } + + fn origin(self, provenance: Provenance, unset: bool, kind: Kind) -> Origin { + let written = (!self.templates.is_empty()).then(|| match kind { + Kind::List => JsonValue::Array(self.written), + _ => self.written.into_iter().last().unwrap_or(JsonValue::Null), + }); + Origin { + provenance, + unset, + imports: self.imports, + templates: self.templates, + written, + } + } +} + +/// A value from a file, and where its parts came from. fn resolve_file_value( spec: &OptionSpec, value: JsonValue, dir: &Path, -) -> Result<(JsonValue, Vec<String>)> { - match value { +) -> Result<(JsonValue, Sources)> { + let mut sources = Sources::default(); + let value = match value { JsonValue::Object(table) => match import::parse_table(&table, dir)? { - Some(importer) => Ok(( - from_import(spec.kind, &importer)?, - vec![importer.describe()], - )), + Some(importer) => { + sources.import(&importer); + from_import(spec.kind, &importer)? + } None => bail!("a table here must be an importer (env, file or command)"), }, JsonValue::String(text) => match import::interpolated(&text, Some(dir)) { - Some(importer) => Ok(( - from_import(spec.kind, &importer)?, - vec![importer.describe()], - )), - None => Ok((JsonValue::String(text), Vec::new())), + Some(importer) => { + sources.import(&importer); + from_import(spec.kind, &importer)? + } + None => JsonValue::String(text), }, JsonValue::Array(items) if spec.kind == Kind::List => { - let mut imports = Vec::new(); let mut values = Vec::new(); for item in items { match item { @@ -682,22 +754,30 @@ fn resolve_file_value( JsonValue::Array(more) => values.extend(more), other => values.push(other), } - imports.push(importer.describe()); + sources.import(&importer); } JsonValue::String(text) => match import::interpolated(&text, Some(dir)) { Some(importer) => { values.push(from_import(Kind::Text, &importer)?); - imports.push(importer.describe()); + sources.import(&importer); + } + None => { + let value = JsonValue::String(text); + sources.literal(&value); + values.push(value); } - None => values.push(JsonValue::String(text)), }, - other => values.push(other), + other => { + sources.literal(&other); + values.push(other); + } } } - Ok((JsonValue::Array(values), imports)) + JsonValue::Array(values) } - other => Ok((other, Vec::new())), - } + other => other, + }; + Ok((value, sources)) } /// The first option whose value alone doesn't fit (serde doesn't name it). @@ -782,16 +862,18 @@ fn arguments(matches: &ArgMatches) -> Result<Layer> { if matches.value_source(spec.name) != Some(ValueSource::CommandLine) { continue; } - let mut imports = Vec::new(); + let mut sources = Sources::default(); let value = if spec.kind == Kind::Flag { JsonValue::Bool(true) } else { let mut values = Vec::new(); for raw in matches.get_many::<String>(spec.name).into_iter().flatten() { let value = match import::parse_argument(raw) { - Raw::Literal(text) => from_text(spec.kind, text), + Raw::Literal(text) => from_text(spec.kind, text).inspect(|value| { + sources.literal(value); + }), Raw::Import(importer) => { - imports.push(importer.describe()); + sources.import(&importer); from_import(spec.kind, &importer) } } @@ -816,11 +898,7 @@ fn arguments(matches: &ArgMatches) -> Result<Layer> { spec.name.to_owned(), Entry { value, - origin: Origin { - provenance: Provenance::Argument, - unset: false, - imports, - }, + origin: sources.origin(Provenance::Argument, false, spec.kind), }, ); } @@ -837,13 +915,41 @@ pub(crate) struct Loaded { pub(crate) print_only: bool, } +/// How `GET /admin/config` shows text with placeholders (`?values=`). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum Values { + /// As it ends up. + #[default] + Interpolated, + /// As written, placeholders and all. + Original, +} + +/// The configuration reports, made once at start for `GET /admin/config`. +#[derive(Debug, Clone, Default)] +pub(crate) struct Reports { + interpolated: JsonValue, + original: JsonValue, +} + +impl Reports { + pub(crate) fn get(&self, values: Values) -> &JsonValue { + match values { + Values::Interpolated => &self.interpolated, + Values::Original => &self.original, + } + } +} + impl Loaded { /// The effective configuration with each value's source, for `--print-config` and /// `GET /admin/config`. Imported values are left out (they may be secret); their - /// importers are listed instead. - pub(crate) fn report(&self) -> JsonValue { - let values = match serde_json::to_value(&self.settings) { - Ok(JsonValue::Object(values)) => values, + /// importers are listed instead. Secrets are left out however they were given. + /// Text with placeholders is shown as `values` says, the text as written listed. + pub(crate) fn report(&self, values: Values) -> JsonValue { + let settings = match serde_json::to_value(&self.settings) { + Ok(JsonValue::Object(settings)) => settings, _ => serde_json::Map::new(), }; let options: serde_json::Map<String, JsonValue> = OPTIONS @@ -851,7 +957,8 @@ impl Loaded { .filter(|spec| !spec.hidden || self.origins.contains_key(spec.name)) .map(|spec| { let mut entry = serde_json::Map::new(); - match self.origins.get(spec.name) { + let origin = self.origins.get(spec.name); + match origin { None => { entry.insert("source".into(), "default".into()); } @@ -868,17 +975,25 @@ impl Loaded { if origin.is_imported() { entry.insert("imported".into(), origin.imports.clone().into()); } + // Of a secret, the text may hold some of it. + if !origin.templates.is_empty() && !spec.secret { + entry.insert("templates".into(), origin.templates.clone().into()); + } if origin.unset { entry.insert("unset".into(), true.into()); } } } - let imported = self.origins.get(spec.name).is_some_and(Origin::is_imported); - if !imported { - entry.insert( - "value".into(), - values.get(spec.name).cloned().unwrap_or(JsonValue::Null), - ); + let given = origin.is_some_and(|origin| !origin.unset); + if spec.secret && given { + entry.insert("secret".into(), true.into()); + } else if !origin.is_some_and(Origin::is_imported) { + let written = origin.and_then(|origin| origin.written.clone()); + let value = match (values, written) { + (Values::Original, Some(written)) => written, + _ => settings.get(spec.name).cloned().unwrap_or(JsonValue::Null), + }; + entry.insert("value".into(), value); } (spec.name.to_owned(), JsonValue::Object(entry)) }) @@ -888,6 +1003,13 @@ impl Loaded { "options": options, }) } + + pub(crate) fn reports(&self) -> Reports { + Reports { + interpolated: self.report(Values::Interpolated), + original: self.report(Values::Original), + } + } } /// Reads the command line and the configuration files it names. Exits on `--help`, @@ -979,6 +1101,8 @@ mod tests { provenance: Provenance::Argument, unset: false, imports: Vec::new(), + templates: Vec::new(), + written: None, }, }, ) @@ -994,6 +1118,8 @@ mod tests { provenance, unset: false, imports: Vec::new(), + templates: Vec::new(), + written: None, }, }; let mut merged = layer(json!({ "htpasswd": "u" })); @@ -1041,6 +1167,8 @@ mod tests { provenance, unset: false, imports: Vec::new(), + templates: Vec::new(), + written: None, }, }; let file = |value| from(Provenance::File("/etc/continuum.toml".into()), value); @@ -1095,7 +1223,7 @@ mod tests { assert!(loaded.settings.mycelium_rest_url.is_some()); assert!(loaded.settings.alias.is_none()); assert_eq!( - loaded.report()["options"]["alias"], + loaded.report(Values::Interpolated)["options"]["alias"], json!({ "source": "argument", "unset": true, "value": null }) ); } @@ -1104,7 +1232,9 @@ mod tests { fn report_hides_imported_values() { let mut values = layer(json!({ "htpasswd": "users", "alias": "secret-ish" })); values.get_mut("alias").unwrap().origin.imports = vec!["env:HOSTNAME".into()]; - let report = resolve(values, Vec::new()).unwrap().report(); + let report = resolve(values, Vec::new()) + .unwrap() + .report(Values::Interpolated); let options = &report["options"]; assert_eq!( options["htpasswd"], @@ -1212,4 +1342,73 @@ mod tests { assert_eq!(layer["debug-api"].value, json!(true)); assert!(!layer.contains_key("neighbours")); } + + #[test] + fn report_shows_interpolated_values() { + // SAFETY: tests in this module don't read this variable concurrently. + unsafe { std::env::set_var("CONTINUUM_TEST_SITE", "berlin") }; + let matches = command() + .try_get_matches_from([ + "continuum", + "--htpasswd", + "users", + "--alias", + "node-${env:CONTINUUM_TEST_SITE}", + "--seeds", + "a,b", + "--seeds", + "c-${env:CONTINUUM_TEST_SITE}", + ]) + .unwrap(); + let layer = arguments(&matches).unwrap(); + assert!(!layer["alias"].origin.is_imported()); + assert_eq!(layer["alias"].shown("alias"), "\"node-berlin\""); + let loaded = resolve(layer, Vec::new()).unwrap(); + + let report = loaded.report(Values::Interpolated); + assert_eq!( + report["options"]["alias"], + json!({ + "source": "argument", + "templates": ["node-${env:CONTINUUM_TEST_SITE}"], + "value": "node-berlin", + }) + ); + assert_eq!( + report["options"]["seeds"]["value"], + json!(["a", "b", "c-berlin"]) + ); + + let report = loaded.report(Values::Original); + assert_eq!( + report["options"]["alias"]["value"], + json!("node-${env:CONTINUUM_TEST_SITE}") + ); + assert_eq!( + report["options"]["seeds"]["value"], + json!(["a", "b", "c-${env:CONTINUUM_TEST_SITE}"]) + ); + // Without placeholders, both are the same. + assert_eq!( + report["options"]["neighbours"], + json!({ "source": "default", "value": 12 }) + ); + } + + #[test] + fn secrets_are_never_shown() { + let values = + layer(json!({ "htpasswd": "users", "mycelium-rest-url": "http://u:pw@h:8989" })); + assert_eq!( + values["mycelium-rest-url"].shown("mycelium-rest-url"), + "<secret>" + ); + let loaded = resolve(values, Vec::new()).unwrap(); + for mode in [Values::Interpolated, Values::Original] { + assert_eq!( + loaded.report(mode)["options"]["mycelium-rest-url"], + json!({ "source": "argument", "secret": true }) + ); + } + } } diff --git a/src/main.rs b/src/main.rs index 0cda7ca..76086ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -43,7 +43,10 @@ async fn main() -> Result<()> { let config = config::load()?; if config.print_only { - println!("{}", serde_json::to_string_pretty(&config.report())?); + println!( + "{}", + serde_json::to_string_pretty(&config.report(config::Values::Interpolated))? + ); return Ok(()); } let args = &config.settings; @@ -84,10 +87,10 @@ async fn main() -> Result<()> { bind_addr = args.bind_addr.to_string(), "Starting replication service" ); - debug!( - url = mycelium_rest_url.to_string(), - "Using Mycelium REST API" - ); + // Without a password it may carry (the option is secret). + let mut shown_url = mycelium_rest_url.clone(); + let _ = shown_url.set_password(None); + debug!(url = shown_url.to_string(), "Using Mycelium REST API"); let force = force::Force::parse(args.force.iter().map(String::as_str), config::FORCE) .map_err(anyhow::Error::msg)?; @@ -181,7 +184,7 @@ async fn main() -> Result<()> { files, state_file, document_file, - config_report: config.report(), + config_reports: config.reports(), clock: Default::default(), }); let state = node.state.clone(); diff --git a/src/node.rs b/src/node.rs index 49e4194..945f5e7 100644 --- a/src/node.rs +++ b/src/node.rs @@ -46,7 +46,7 @@ pub(crate) struct NodeConfig { pub(crate) state_file: Option<Arc<StateFile>>, pub(crate) document_file: Option<Arc<DocumentFile>>, /// For `GET /admin/config`. - pub(crate) config_report: serde_json::Value, + pub(crate) config_reports: crate::config::Reports, /// The node's wall clock; `Clock::default()` outside tests. pub(crate) clock: Arc<Clock>, } @@ -107,7 +107,7 @@ impl Node { state_file: config.state_file, document_file: config.document_file, files: config.files, - config: Arc::new(config.config_report), + config: Arc::new(config.config_reports), neighbours: Default::default(), topology_changed: Default::default(), pulling: Default::default(), diff --git a/src/simtest/mod.rs b/src/simtest/mod.rs index 5e9cc2f..eb5a741 100644 --- a/src/simtest/mod.rs +++ b/src/simtest/mod.rs @@ -91,7 +91,7 @@ impl Sim { files: Arc::new(Files::default()), state_file: None, document_file: None, - config_report: JsonValue::Null, + config_reports: Default::default(), clock: Default::default(), }; adjust(&mut config); diff --git a/src/state/mod.rs b/src/state/mod.rs index d4928a8..c8243a2 100644 --- a/src/state/mod.rs +++ b/src/state/mod.rs @@ -120,9 +120,9 @@ pub(crate) struct AppState { pub(crate) document_file: Option<Arc<crate::document::file::DocumentFile>>, /// What this node remembers across restarts (`--state-file`), if configured. pub(crate) state_file: Option<Arc<crate::swarm::state_file::StateFile>>, - /// The effective configuration with sources, imported values left out + /// The effective configuration with sources, imported values and secrets left out /// (`config::Loaded::report`), for `GET /admin/config`. - pub(crate) config: Arc<serde_json::Value>, + pub(crate) config: Arc<crate::config::Reports>, /// Our own connectivity record and stability tier (`swarm::stability`). pub(crate) stability: Arc<std::sync::Mutex<crate::swarm::stability::Stability>>, /// Round-trip times to peers (`swarm::rtt`). |