//! Importers: values taken from elsewhere instead of written out, so a configuration //! can say where a value comes from without holding it (secrets, host facts). //! //! * `env:NAME`: an environment variable; //! * `file:PATH`: a file's contents; //! * `cmd:COMMAND`: the standard output of a shell command (`sh -c`). //! //! The content is text (trailing line breaks removed), converted like a value written //! on the command line; with `+json` (`env+json:NAME`, …) it is parsed as JSON instead. //! //! On the command line, a value with one of these prefixes is imported; `literal:` //! passes the rest as is. In configuration files, an importer is a table: //! `{ env = "NAME" }`, `{ file = "PATH" }`, `{ command = "COMMAND" }` (or an argument //! list, run without a shell), each with an optional `format = "json"`. //! //! Any other text value may hold placeholders, `${env:NAME}`, `${file:PATH}` or //! `${cmd:COMMAND}`, each replaced by its content as text (`$$` is a `$`; a placeholder //! ends at the first `}`), e.g. `node-${env:HOSTNAME}`. Relative paths are relative to //! the file, and commands run in its directory, in placeholders too. use std::path::{Path, PathBuf}; use std::process::Command; use anyhow::{Context, Result, anyhow, bail}; use serde_json::{Map as JsonMap, Value as JsonValue}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Format { Text, Json, } #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum Source { Env(String), File(PathBuf), Shell { command: String, dir: Option, }, Argv { argv: Vec, dir: Option, }, /// Text with placeholders (`interpolated`). Template { template: String, dir: Option, }, } #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Importer { pub(crate) source: Source, pub(crate) format: Format, } /// What a command-line value is: imported, or literal. #[derive(Debug, PartialEq, Eq)] pub(crate) enum Raw<'a> { Import(Importer), Literal(&'a str), } /// Reads a command-line value. pub(crate) fn parse_argument(raw: &str) -> Raw<'_> { if let Some(rest) = raw.strip_prefix("literal:") { return Raw::Literal(rest); } let Some((prefix, rest)) = raw.split_once(':') else { return literal_or_interpolated(raw); }; let (kind, format) = match prefix.split_once('+') { None => (prefix, Format::Text), Some((kind, "json")) => (kind, Format::Json), Some(_) => return literal_or_interpolated(raw), }; let source = match kind { "env" => Source::Env(rest.to_owned()), "file" => Source::File(PathBuf::from(rest)), "cmd" => Source::Shell { command: rest.to_owned(), dir: None, }, _ => return literal_or_interpolated(raw), }; Raw::Import(Importer { source, format }) } fn literal_or_interpolated(raw: &str) -> Raw<'_> { match interpolated(raw, None) { Some(importer) => Raw::Import(importer), None => Raw::Literal(raw), } } /// `text` as an importer if it holds a `$` (a placeholder, `$$`, or a mistake that /// expanding it reports), read in `dir`; None if it is plain text. pub(crate) fn interpolated(text: &str, dir: Option<&Path>) -> Option { text.contains('$').then(|| Importer { source: Source::Template { template: text.to_owned(), dir: dir.map(Path::to_owned), }, format: Format::Text, }) } /// Reads an importer table from a file in `dir`; None if `object` isn't one. pub(crate) fn parse_table( object: &JsonMap, dir: &Path, ) -> Result> { let sources = ["env", "file", "command"] .into_iter() .filter(|key| object.contains_key(*key)) .count(); if sources == 0 { return Ok(None); } if sources > 1 { bail!("an importer takes exactly one of env, file and command"); } if let Some(unknown) = object .keys() .find(|key| !["env", "file", "command", "format"].contains(&key.as_str())) { bail!("unknown importer key {unknown:?}"); } let format = match object.get("format") { None => Format::Text, Some(JsonValue::String(format)) if format == "text" => Format::Text, Some(JsonValue::String(format)) if format == "json" => Format::Json, Some(other) => bail!("importer format must be \"text\" or \"json\", not {other}"), }; let text = |key: &str| -> Result { match object.get(key) { Some(JsonValue::String(value)) => Ok(value.clone()), _ => Err(anyhow!("importer {key} must be a string")), } }; let source = if object.contains_key("env") { Source::Env(text("env")?) } else if object.contains_key("file") { Source::File(dir.join(text("file")?)) } else { match &object["command"] { JsonValue::String(command) => Source::Shell { command: command.clone(), dir: Some(dir.to_owned()), }, JsonValue::Array(items) => { let argv = items .iter() .map(|item| item.as_str().map(str::to_owned)) .collect::>>() .filter(|argv| !argv.is_empty()) .ok_or_else(|| { anyhow!("importer command must be a string or a non-empty list of strings") })?; Source::Argv { argv, dir: Some(dir.to_owned()), } } _ => bail!("importer command must be a string or a non-empty list of strings"), } }; Ok(Some(Importer { source, format })) } impl Importer { /// Shown instead of the value (which may be secret), e.g. `env:HOSTNAME`. pub(crate) fn describe(&self) -> String { let format = match self.format { Format::Text => "", Format::Json => "+json", }; match &self.source { Source::Env(name) => format!("env{format}:{name}"), Source::File(path) => format!("file{format}:{}", path.display()), Source::Shell { command, .. } => format!("cmd{format}:{command}"), Source::Argv { argv, .. } => format!("cmd{format}:{}", argv.join(" ")), Source::Template { template, .. } => template.clone(), } } /// Whether this is text with placeholders (`interpolated`) rather than an importer /// of the whole value: its result is shown, the importers' are not. pub(crate) fn is_interpolation(&self) -> bool { matches!(self.source, Source::Template { .. }) } /// The imported content: text, or JSON. pub(crate) fn fetch(&self) -> Result { let text = match &self.source { Source::Env(name) => std::env::var(name).with_context(|| { format!("environment variable {name} is not set (or not UTF-8)") })?, Source::File(path) => std::fs::read_to_string(path) .with_context(|| format!("reading {}", path.display()))?, Source::Shell { command, dir } => { let mut process = Command::new("sh"); process.arg("-c").arg(command); run(process, dir.as_deref(), command)? } Source::Argv { argv, dir } => { let mut process = Command::new(&argv[0]); process.args(&argv[1..]); run(process, dir.as_deref(), &argv.join(" "))? } Source::Template { template, dir } => expand(template, dir.as_deref())?, }; let text = text.trim_end_matches(['\n', '\r']); Ok(match self.format { Format::Text => Imported::Text(text.to_owned()), Format::Json => Imported::Json( serde_json::from_str(text) .with_context(|| format!("{} is not valid JSON", self.describe()))?, ), }) } } #[derive(Debug, Clone, PartialEq)] pub(crate) enum Imported { Text(String), Json(JsonValue), } /// Where the placeholder that `text` starts (after its `${`) ends: at the `}` that /// matches its `{`. Braces inside balance, so a command may hold `{...}` (a jq object, a /// shell group); in a command (`cmd:`), braces in shell quotes (`'...'`, `"..."`) or /// escaped with a backslash don't count either. fn placeholder_end(text: &str) -> Option { let shell = text.starts_with("cmd:") || text.starts_with("cmd+"); let mut depth = 0usize; let mut quote: Option = None; let mut escaped = false; for (at, char) in text.char_indices() { if shell { if escaped { escaped = false; continue; } match (quote, char) { // Nothing escapes in single quotes. (Some('\''), '\'') => quote = None, (Some('\''), _) => {} (_, '\\') => escaped = true, (Some('"'), '"') => quote = None, (Some('"'), _) => {} (None, '\'' | '"') => quote = Some(char), _ => {} } if quote.is_some() || char == '\'' || char == '"' || char == '\\' { continue; } } match char { '{' => depth += 1, '}' if depth == 0 => return Some(at), '}' => depth -= 1, _ => {} } } None } /// Text with its placeholders replaced by their content. fn expand(template: &str, dir: Option<&Path>) -> Result { let mut out = String::new(); let mut rest = template; while let Some(at) = rest.find('$') { out.push_str(&rest[..at]); let after = &rest[at + 1..]; if let Some(after) = after.strip_prefix('$') { out.push('$'); rest = after; } else if let Some(after) = after.strip_prefix('{') { let end = placeholder_end(after) .ok_or_else(|| anyhow!("unterminated placeholder in template {template:?}"))?; let placeholder = &after[..end]; let Raw::Import(mut importer) = parse_argument(placeholder) else { bail!( "placeholder {placeholder:?} in template {template:?} isn't env:, file: or cmd:" ); }; match &mut importer.source { Source::File(path) => { if let Some(dir) = dir { *path = dir.join(&*path); } } Source::Shell { dir: run_in, .. } => *run_in = dir.map(Path::to_owned), Source::Template { .. } => bail!("placeholders don't nest: {placeholder:?}"), Source::Env(_) | Source::Argv { .. } => {} } if importer.format != Format::Text { bail!("placeholder {placeholder:?} must be text (+json isn't for placeholders)"); } match importer.fetch()? { Imported::Text(text) => out.push_str(&text), Imported::Json(_) => unreachable!("text importer"), } rest = &after[end + 1..]; } else { // Most likely a shell-style `$NAME` that would silently stay as it is. bail!( "a `$` in {template:?} that starts no placeholder: write `$$` for a `$`, or \ e.g. `${{env:NAME}}` for a variable" ); } } out.push_str(rest); Ok(out) } fn run(mut process: Command, dir: Option<&Path>, shown: &str) -> Result { if let Some(dir) = dir { process.current_dir(dir); } let output = process .output() .with_context(|| format!("running {shown}"))?; if !output.status.success() { bail!( "{shown} failed ({}): {}", output.status, String::from_utf8_lossy(&output.stderr).trim() ); } String::from_utf8(output.stdout).with_context(|| format!("output of {shown} is not UTF-8")) } #[cfg(test)] mod tests { use super::*; use serde_json::json; #[test] fn command_line_values() { assert_eq!(parse_argument("plain"), Raw::Literal("plain")); assert_eq!( parse_argument("tls://host:9651"), Raw::Literal("tls://host:9651") ); assert_eq!(parse_argument("literal:env:X"), Raw::Literal("env:X")); assert_eq!( parse_argument("env+json:SEEDS"), Raw::Import(Importer { source: Source::Env("SEEDS".into()), format: Format::Json }) ); assert_eq!( parse_argument("cmd:hostname -f"), Raw::Import(Importer { source: Source::Shell { command: "hostname -f".into(), dir: None }, format: Format::Text }) ); assert_eq!(parse_argument("env+yaml:X"), Raw::Literal("env+yaml:X")); // A `$` makes it interpolated; expanding it then says what's wrong. assert!(matches!(parse_argument("a $5 price"), Raw::Import(_))); assert_eq!(parse_argument("literal:a $5"), Raw::Literal("a $5")); assert_eq!( parse_argument("node-${env:HOST}"), Raw::Import(Importer { source: Source::Template { template: "node-${env:HOST}".into(), dir: None }, format: Format::Text }) ); // A placeholder in a value that looks like a prefix is interpolated too. assert!(matches!( parse_argument("tls://${env:HOST}:9651"), Raw::Import(_) )); assert_eq!( parse_argument("literal:${env:HOST}"), Raw::Literal("${env:HOST}") ); } #[test] fn templates() { let expanded = |template: &str| expand(template, None); assert_eq!(expanded("a-${cmd:echo b}-c $$5").unwrap(), "a-b-c $5"); assert!(expanded("$HOME/x").is_err()); assert!(expanded("a $5 price").is_err()); assert_eq!( expanded("${cmd:printf one}${cmd:printf two}").unwrap(), "onetwo" ); assert!(expanded("${cmd:echo b").is_err()); assert!(expanded("${plain}").is_err()); assert!(expanded("${env+json:X}").is_err()); } #[test] fn placeholders_end_at_their_matching_brace() { fn end(text: &str) -> Option<&str> { placeholder_end(text).map(|at| &text[..at]) } assert_eq!(end("env:HOST}-x"), Some("env:HOST")); // Balanced braces, as a jq object has them. assert_eq!(end("cmd:jq -r '{a: .b}'}/tcp"), Some("cmd:jq -r '{a: .b}'")); assert_eq!(end("cmd:sh -c { true; }}"), Some("cmd:sh -c { true; }")); // In a command, braces in quotes or escaped don't count. assert_eq!(end("cmd:printf '}'}x"), Some("cmd:printf '}'")); assert_eq!(end(r#"cmd:printf "}\""}x"#), Some(r#"cmd:printf "}\"""#)); assert_eq!(end(r"cmd:printf \}}x"), Some(r"cmd:printf \}")); assert_eq!(end("cmd:printf '}"), None); assert_eq!(end("env:X"), None); // The kind of command this is for: quotes of both kinds, and `$` inside. let command = r#"cmd:ip -j addr show wireguard | jq -r '.[].addr_info[]|select(.family=="inet6" and .scope=="global").local'"#; assert_eq!(end(&format!("{command}}}/tcp/9661")), Some(command)); } #[test] fn commands_with_braces_and_quotes() { let expanded = |template: &str| expand(template, None); assert_eq!( expanded(r#"/ip6/${cmd:printf '%s' "$(echo '::1')" | tr -d '{}'}/tcp/9661"#).unwrap(), "/ip6/::1/tcp/9661" ); assert_eq!(expanded("x${cmd:printf '{%s}' a}y").unwrap(), "x{a}y"); } #[test] fn tables() { let dir = Path::new("/etc/continuum"); let table = |value: JsonValue| parse_table(value.as_object().unwrap(), dir); assert_eq!(table(json!({ "x": 1 })).unwrap(), None); assert_eq!( table(json!({ "file": "key", "format": "json" })).unwrap(), Some(Importer { source: Source::File("/etc/continuum/key".into()), format: Format::Json }) ); assert!(table(json!({ "env": "A", "file": "b" })).is_err()); assert!(table(json!({ "env": "A", "extra": 1 })).is_err()); assert!(table(json!({ "command": [] })).is_err()); } #[test] fn fetching() { let echo = Importer { source: Source::Argv { argv: vec!["printf".into(), "[1, 2]\\n\\n".into()], dir: None, }, format: Format::Json, }; assert_eq!(echo.fetch().unwrap(), Imported::Json(json!([1, 2]))); let shell = Importer { source: Source::Shell { command: "echo hello".into(), dir: None, }, format: Format::Text, }; assert_eq!(shell.fetch().unwrap(), Imported::Text("hello".into())); let failing = Importer { source: Source::Shell { command: "exit 3".into(), dir: None, }, format: Format::Text, }; assert!(failing.fetch().is_err()); } }