aboutsummaryrefslogtreecommitdiffstats
path: root/pkgs/populate
diff options
context:
space:
mode:
Diffstat (limited to 'pkgs/populate')
-rw-r--r--pkgs/populate/Cargo.lock7
-rw-r--r--pkgs/populate/Cargo.toml1
-rw-r--r--pkgs/populate/docs/design.md21
-rw-r--r--pkgs/populate/rust.nix10
-rw-r--r--pkgs/populate/src/main.rs31
-rw-r--r--pkgs/populate/src/populate.rs216
-rw-r--r--pkgs/populate/src/rsync.rs322
-rw-r--r--pkgs/populate/src/run.rs348
-rw-r--r--pkgs/populate/src/sh.rs485
-rw-r--r--pkgs/populate/src/spec.rs11
10 files changed, 1435 insertions, 17 deletions
diff --git a/pkgs/populate/Cargo.lock b/pkgs/populate/Cargo.lock
index 6690d33..ea97ade 100644
--- a/pkgs/populate/Cargo.lock
+++ b/pkgs/populate/Cargo.lock
@@ -9,6 +9,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
[[package]]
+name = "libc"
+version = "0.2.191"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
+
+[[package]]
name = "memchr"
version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -18,6 +24,7 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
name = "populate"
version = "2.0.0-dev"
dependencies = [
+ "libc",
"serde",
"serde_json",
]
diff --git a/pkgs/populate/Cargo.toml b/pkgs/populate/Cargo.toml
index 9532d9d..4a4face 100644
--- a/pkgs/populate/Cargo.toml
+++ b/pkgs/populate/Cargo.toml
@@ -9,3 +9,4 @@ publish = false
[dependencies]
serde = { version = "1", features = ["derive"] }
serde_json = "1"
+libc = "0.2"
diff --git a/pkgs/populate/docs/design.md b/pkgs/populate/docs/design.md
index a312510..a824f45 100644
--- a/pkgs/populate/docs/design.md
+++ b/pkgs/populate/docs/design.md
@@ -27,9 +27,9 @@ command delivering them. `implementation` selects how:
- `"shell"`, the default: `shell.nix`.
- `"rust"`: `rust.nix`, which checks the arguments with `checks.nix`, turns
them into a spec, and returns a script running the program with the spec
- on its standard input. The script sets `PATH` to the tools populate
- runs, and depends on the store paths in the spec, so building it builds
- them. `rustProgram` is the program, by default built by `program.nix`.
+ on file descriptor 3, so the commands populate runs keep standard input.
+ The script puts the tools populate runs in front of `PATH`, and depends
+ on the store paths in the spec, so building it builds them. `rustProgram` is the program, by default built by `program.nix`.
`writeDeploy` and the other functions of `pkgs.krops` use `pkgs.populate`,
so their arguments stay as they are.
@@ -72,6 +72,7 @@ The spec is a JSON object:
| `path` | string | | The target directory. |
| `sudo` | boolean | false | Deliver as root, via sudo. |
| `extraOptions` | array of strings | [] | More arguments for ssh. |
+| `local` | boolean or null | null | Whether the target is local. |
Each source is an object with the field `type`, the source's type, and a
field named after the type, holding the source's settings:
@@ -148,9 +149,11 @@ places populate must not touch, check this after every phase.
## Execution
-Whether a target is local gets decided when the program runs: it is, if
-its user is the user running populate and its host is `localhost` or the
-host's name. For a local target, populate runs commands directly.
+Whether a target is local is what the target's `local` says, or, if
+that's null, gets decided when the program runs: it is, if its user is
+`LOGNAME` and its host is `localhost` or the host's name, from `HOSTNAME`
+or `/etc/hostname`. `rust.nix` decides during evaluation, like the other
+functions of krops. For a local target, populate runs commands directly.
For a remote target, populate runs commands over ssh, sharing one master
connection per target. The commands are fixed POSIX shell snippets, with
@@ -173,5 +176,7 @@ Populate runs rsync, Git, ssh, and Nix as programs, found in `PATH`.
## Status
-The program reads the spec, checks it, and fails, saying that delivering
-sources isn't implemented yet.
+The program delivers `derivation`, `file`, `git`, `pass`, `passage`,
+`pipe`, and `symlink` sources, makes backups, and prunes. Given `flake`
+or `flake-input` sources, or sources drawing inputs from others, it fails
+before changing anything, saying that these aren't implemented yet.
diff --git a/pkgs/populate/rust.nix b/pkgs/populate/rust.nix
index cbf6fad..f6ece25 100644
--- a/pkgs/populate/rust.nix
+++ b/pkgs/populate/rust.nix
@@ -26,18 +26,22 @@ let
port = if target.port or null == null then null else toString target.port;
sudo = target.sudo or false;
extraOptions = target.extraOptions or [];
+ # Decided during evaluation, like the other functions of krops do.
+ local = isLocalTarget target;
};
};
in
writers.writeDash "populate.${target.host}" /* sh */ ''
set -efu
echo + "$0" >&2
+ # The tools populate depends on, in front of others of the same name.
PATH=${makeBinPath [
- coreutils findutils git jq nix openssh pass passage rsync
+ rsync openssh git nix pass passage jq findutils coreutils
]}''${PATH:+:$PATH}
export PATH
- # The JSON is a single line, which can't be the delimiter.
- exec ${program}/bin/populate - <<'EOF'
+ # On a file descriptor of its own, so the commands populate runs keep
+ # standard input. The JSON is a single line, which can't be the delimiter.
+ exec ${program}/bin/populate /dev/fd/3 3<<'EOF'
${spec}
EOF
''
diff --git a/pkgs/populate/src/main.rs b/pkgs/populate/src/main.rs
index a095b74..dc0c511 100644
--- a/pkgs/populate/src/main.rs
+++ b/pkgs/populate/src/main.rs
@@ -2,6 +2,10 @@
//! JSON file or - for standard input, says. See docs/design.md.
mod check;
+mod populate;
+mod rsync;
+mod run;
+mod sh;
mod spec;
use std::io::Read;
@@ -50,6 +54,29 @@ fn main() -> ExitCode {
eprintln!("{message}");
return ExitCode::FAILURE;
}
- eprintln!("populate: delivering sources isn't implemented yet");
- ExitCode::FAILURE
+ run::trap_signals();
+ let context = match run::Context::new(spec.target.clone(), spec.target.local) {
+ Ok(context) => context,
+ Err(failure) => return exit(failure),
+ };
+ let result = populate::populate(&context, &spec);
+ run::ignore_signals();
+ context.clean_up();
+ match result {
+ Ok(()) => ExitCode::SUCCESS,
+ Err(failure) => exit(failure),
+ }
+}
+
+/// Says why populate fails, unless a command did, and returns the exit
+/// status for that.
+fn exit(failure: run::Failure) -> ExitCode {
+ match failure {
+ run::Failure::Message(message) => {
+ eprintln!("{message}");
+ ExitCode::FAILURE
+ }
+ run::Failure::Command => ExitCode::FAILURE,
+ run::Failure::Signal(signal) => ExitCode::from(128 + signal as u8),
+ }
}
diff --git a/pkgs/populate/src/populate.rs b/pkgs/populate/src/populate.rs
new file mode 100644
index 0000000..d3567d3
--- /dev/null
+++ b/pkgs/populate/src/populate.rs
@@ -0,0 +1,216 @@
+//! Delivering the sources of a spec.
+
+use crate::rsync::{self, Config};
+use crate::run::{self, Context, Failure, Result};
+use crate::sh;
+use crate::spec::{Source, Spec};
+use serde_json::{Map, Value};
+use std::fs;
+use std::path::PathBuf;
+use std::process::Command;
+
+/// Checks the sentinel, makes a backup if asked, delivers each source, and
+/// prunes if asked.
+pub fn populate(context: &Context, spec: &Spec) -> Result<()> {
+ let dir = &spec.target.path;
+ unsupported(context, spec)?;
+ let sentinel = format!("{dir}/.populate");
+ let force = if spec.force { "1" } else { "" };
+ context.run(
+ sh::CHECK,
+ &[&sentinel, &context.name(&sentinel), dir, force],
+ )?;
+ if spec.backup {
+ context.run(&sh::backup(), &[dir, &context.name(&sentinel)])?;
+ }
+ for (name, source) in &spec.source {
+ let path = format!("{dir}/{name}");
+ eprintln!("populating '{}'...", context.name(&path));
+ let workdir = context.tmpdir.join("workdirs").join(name);
+ fs::create_dir_all(&workdir).map_err(|e| {
+ Failure::Message(format!("error: can't create {}: {e}", workdir.display()))
+ })?;
+ deliver(context, source, &path, workdir)?;
+ }
+ if spec.prune {
+ let flakes = spec
+ .source
+ .iter()
+ .filter(|(_, source)| matches!(source.kind.as_str(), "flake" | "flake-input"))
+ .map(|(name, _)| name.as_str());
+ let mut args = vec![dir.as_str()];
+ let log = context.name(dir);
+ args.push(&log);
+ args.extend(spec.source.keys().map(String::as_str));
+ args.push("--");
+ args.extend(flakes);
+ context.run(&sh::prune(), &args)?;
+ }
+ Ok(())
+}
+
+/// Fails, before changing anything, if the spec needs what this program
+/// doesn't do yet.
+fn unsupported(context: &Context, spec: &Spec) -> Result<()> {
+ for (name, source) in &spec.source {
+ let copying = matches!(source.kind.as_str(), "flake" | "flake-input")
+ && source
+ .settings
+ .get("fetchOnTarget")
+ .and_then(Value::as_bool)
+ != Some(true);
+ if copying && context.target.sudo && !context.local {
+ return Err(Failure::Message(format!(
+ "krops: source {}/{name}: copying flakes to targets with sudo is not supported",
+ spec.target.path
+ )));
+ }
+ }
+ for source in spec.source.values() {
+ let what = match source.kind.as_str() {
+ "derivation" | "file" | "git" | "pass" | "passage" | "pipe" | "symlink" => None,
+ kind => Some(format!("{kind} sources")),
+ };
+ let what = what.or_else(|| {
+ (!source.follows().is_empty()).then(|| "drawing inputs from sources".to_string())
+ });
+ if let Some(what) = what {
+ return Err(Failure::Message(format!(
+ "populate: {what} aren't implemented yet, so it delivers nothing to {}",
+ context.name(&spec.target.path)
+ )));
+ }
+ }
+ Ok(())
+}
+
+fn string<'a>(settings: &'a Map<String, Value>, key: &str) -> Result<&'a str> {
+ settings
+ .get(key)
+ .and_then(Value::as_str)
+ .ok_or_else(|| Failure::Message(format!("populate: the source has no {key}")))
+}
+
+/// Delivers a source to path on the target, using workdir for files it
+/// prepares on the deploying machine.
+fn deliver(context: &Context, source: &Source, path: &str, workdir: PathBuf) -> Result<()> {
+ let settings = &source.settings;
+ match source.kind.as_str() {
+ "derivation" => context.run(
+ &sh::derivation(),
+ &[path, &context.name(path), string(settings, "text")?],
+ ),
+ "file" => {
+ let source_path = string(settings, "path")?;
+ // Store paths have the same time, so only checksums tell
+ // different contents apart.
+ let use_checksum = settings
+ .get("useChecksum")
+ .and_then(Value::as_bool)
+ .unwrap_or(run::is_store_path(source_path));
+ let config = Config {
+ use_checksum,
+ ..Config::from_settings(settings)
+ };
+ rsync::transfer(context, &config, source_path, path)
+ }
+ "pipe" => {
+ let out = workdir.join("out");
+ let file = fs::File::create(&out).map_err(|e| {
+ Failure::Message(format!("error: can't create {}: {e}", out.display()))
+ })?;
+ run::status(
+ Command::new(string(settings, "command")?)
+ .current_dir(&workdir)
+ .stdout(file),
+ )?;
+ run::status(
+ Command::new("touch")
+ .arg("-t")
+ .arg("300001010000")
+ .arg(&out),
+ )?;
+ // With the fixed modification time, only checksums tell outputs
+ // of the same size apart.
+ let config = Config {
+ use_checksum: true,
+ ..Config::default()
+ };
+ rsync::transfer(context, &config, &out.to_string_lossy(), path)
+ }
+ "git" => {
+ let flag = |key| match settings.get(key).and_then(Value::as_bool) {
+ Some(true) => "1",
+ _ => "",
+ };
+ let excludes = settings
+ .get("clean")
+ .and_then(|clean| clean.get("exclude"))
+ .and_then(Value::as_array)
+ .into_iter()
+ .flatten()
+ .filter_map(Value::as_str);
+ let name = context.name(path);
+ let mut args = vec![
+ path,
+ &name,
+ string(settings, "url")?,
+ string(settings, "ref")?,
+ flag("shallow"),
+ flag("fetchAlways"),
+ flag("reclone"),
+ ];
+ args.extend(excludes);
+ context.run(&sh::git(), &args)
+ }
+ "pass" => {
+ let dir = string(settings, "dir")?;
+ let name = string(settings, "name")?;
+ let info = run::output(&mut run::local_shell(
+ sh::local::PASS_INFO,
+ &[dir, name],
+ &workdir,
+ ))?;
+ let info = info.trim_end_matches('\n');
+ if !info.is_empty() {
+ let delivered = context.output(sh::PASS_INFO, &[path])?;
+ if delivered.trim_end_matches('\n') == info {
+ return Ok(());
+ }
+ }
+ run::status(&mut run::local_shell(
+ sh::local::PASS,
+ &[dir, name, info],
+ &workdir,
+ ))?;
+ rsync::transfer(
+ context,
+ &Config::default(),
+ &workdir.to_string_lossy(),
+ path,
+ )
+ }
+ "passage" => {
+ let host = context.target.host.split('.').next().unwrap_or_default();
+ let mut command = run::local_shell(sh::local::PASSAGE, &[host], &workdir);
+ for (variable, key) in [
+ ("PASSAGE_AGE", "age"),
+ ("PASSAGE_DIR", "dir"),
+ ("PASSAGE_IDENTITIES_FILE", "identities_file"),
+ ] {
+ command.env(variable, string(settings, key)?);
+ }
+ run::status(&mut command)?;
+ rsync::transfer(
+ context,
+ &Config::default(),
+ &workdir.to_string_lossy(),
+ path,
+ )
+ }
+ "symlink" => context.run(sh::SYMLINK, &[path, string(settings, "target")?]),
+ kind => Err(Failure::Message(format!(
+ "populate: {kind} sources aren't implemented yet"
+ ))),
+ }
+}
diff --git a/pkgs/populate/src/rsync.rs b/pkgs/populate/src/rsync.rs
new file mode 100644
index 0000000..dd61b03
--- /dev/null
+++ b/pkgs/populate/src/rsync.rs
@@ -0,0 +1,322 @@
+//! Transferring with rsync, from the deploying machine to the target.
+
+use crate::run::{self, Context, Failure, Result};
+use crate::sh;
+use serde_json::{Map, Value};
+use std::path::Path;
+use std::process::Command;
+
+/// What a transfer excludes and filters, and how it compares files.
+pub struct Config {
+ pub use_checksum: bool,
+ pub exclude: Vec<String>,
+ /// Filter rules, as rsync's option without the dashes and its pattern.
+ pub filters: Vec<(String, String)>,
+ pub delete_excluded: bool,
+ /// What to do about .rsync-filter files rsync can't read: fail if None,
+ /// make rsync read them anyway if true, sync without them if false.
+ pub insecure_links: Option<bool>,
+ /// Whether .rsync-filter files apply.
+ pub filter_files: bool,
+}
+
+impl Default for Config {
+ fn default() -> Config {
+ Config {
+ use_checksum: false,
+ exclude: Vec::new(),
+ filters: Vec::new(),
+ delete_excluded: true,
+ insecure_links: None,
+ filter_files: true,
+ }
+ }
+}
+
+impl Config {
+ /// The defaults, with what a file source's settings say instead.
+ pub fn from_settings(settings: &Map<String, Value>) -> Config {
+ let mut config = Config::default();
+ if let Some(Value::Bool(b)) = settings.get("useChecksum") {
+ config.use_checksum = *b;
+ }
+ if let Some(Value::Array(patterns)) = settings.get("exclude") {
+ config.exclude = patterns
+ .iter()
+ .filter_map(|p| p.as_str().map(String::from))
+ .collect();
+ }
+ if let Some(Value::Array(filters)) = settings.get("filters") {
+ config.filters = filters
+ .iter()
+ .filter_map(|f| {
+ let kind = f.get("type")?.as_str()?;
+ let pattern = f.get("pattern")?.as_str()?;
+ Some((kind.to_string(), pattern.to_string()))
+ })
+ .collect();
+ }
+ if let Some(Value::Bool(b)) = settings.get("deleteExcluded") {
+ config.delete_excluded = *b;
+ }
+ if let Some(Value::Bool(b)) = settings.get("insecureLinks") {
+ config.insecure_links = Some(*b);
+ }
+ config
+ }
+
+ /// rsync's arguments for what the transfer excludes and filters,
+ /// followed by -F, which applies .rsync-filter files, unless the config
+ /// turns that off.
+ fn filter_args(&self) -> Vec<String> {
+ let mut args = Vec::new();
+ for pattern in &self.exclude {
+ args.extend(["--exclude".to_string(), pattern.clone()]);
+ }
+ for (kind, pattern) in &self.filters {
+ args.extend([format!("--{kind}"), pattern.clone()]);
+ }
+ if self.filter_files {
+ args.push("-F".to_string());
+ }
+ args
+ }
+}
+
+/// Transfers source_path, a directory's contents or a file, to path on the
+/// target.
+pub fn transfer(context: &Context, config: &Config, source_path: &str, path: &str) -> Result<()> {
+ context.run(&sh::break_hard_links(), &[path, &context.name(path)])?;
+ let mut source = source_path.to_string();
+ if Path::new(source_path).is_dir() {
+ source.push('/');
+ }
+ let insecure_links = if config.filter_files {
+ insecure_links(context, config, &source)?
+ } else {
+ false
+ };
+ let mut command = Command::new("rsync");
+ if config.use_checksum {
+ command.arg("--checksum");
+ }
+ if context.target.sudo {
+ command.arg("--rsync-path=sudo rsync");
+ }
+ command.arg("-e").arg(&context.ssh).arg("--protect-args");
+ if insecure_links {
+ command.arg("--insecure-links");
+ }
+ // Each item gets logged with what changed about it, and its size, as
+ // transferred and in total.
+ command
+ .arg("--itemize-changes")
+ .arg("--out-format=%i %n%L %''b/%''l")
+ .arg("--info=backup,del,copy,misc,mount,name,nonreg,remove,skip,symsafe")
+ // Nothing relies on the times of directories and symlinks, and
+ // keeping them would log a line for each directory changed.
+ .arg("--omit-dir-times")
+ .arg("--omit-link-times")
+ .args(config.filter_args())
+ .arg("-rlptD");
+ if config.delete_excluded {
+ command.arg("--delete-excluded");
+ }
+ command
+ .arg(&source)
+ .arg(context.rsync_destination(path))
+ .stdout(run::stderr());
+ run::status(&mut command)
+}
+
+/// rsync 3.5.0 and later open .rsync-filter files with a stricter walk than
+/// the files they transfer, and treat the ones that walk can't open as
+/// missing, without saying so. It fails, e.g., for a directory on the way
+/// that the user may enter but not read, like a home directory with mode
+/// 711, or a symlink that neither root nor the user owns. So rsync, when
+/// asked, tells which .rsync-filter files it reads for source: the ones
+/// therein, and the ones in the directories above it, which apply as well.
+/// If it reports ones as missing that exist and that the user may read, the
+/// config's insecure_links decides: populate fails with an error if it's
+/// None, syncs without them if it's false, and makes rsync open them like
+/// the files it transfers with --insecure-links if it's true, both with a
+/// warning. Returns whether rsync needs --insecure-links.
+fn insecure_links(context: &Context, config: &Config, source: &str) -> Result<bool> {
+ let version_line = run::output(Command::new("rsync").arg("--version"))?;
+ let version = version_line
+ .lines()
+ .next()
+ .and_then(|line| line.split_whitespace().nth(2))
+ .unwrap_or("")
+ .to_string();
+ if !version_at_least(&version, "3.5.0") || !filtered(source) {
+ return Ok(false);
+ }
+ // A dry run of the same transfer, as far as filtering goes.
+ let log = run::output(
+ Command::new("sh")
+ .arg("-c")
+ .arg(r#"rsync "$@" 2>&1 || :"#)
+ .arg("sh")
+ .args(["--dry-run", "--debug=FILTER2"])
+ .args(config.filter_args())
+ .arg("-rlptD")
+ .arg(source)
+ .arg(context.tmpdir.join("rsync-filter-check")),
+ )?;
+ let mut parsed = false;
+ let mut ignored = None;
+ for line in log.lines() {
+ let Some(rest) = line.strip_prefix("[sender] parse_filter_file(") else {
+ continue;
+ };
+ parsed = true;
+ let Some(rest) = rest.strip_suffix(") [not found]") else {
+ continue;
+ };
+ // What precedes the last two comma-separated fields.
+ let Some(file) = rest.rsplitn(3, ',').nth(2) else {
+ continue;
+ };
+ let file = unescape(file);
+ let shown = file.display().to_string();
+ let readable = std::fs::File::open(&file).is_ok();
+ if Path::new(&file).exists() && readable {
+ ignored = Some(shown);
+ break;
+ }
+ }
+ if !parsed {
+ eprintln!(
+ "warning: rsync {version} doesn't tell which .rsync-filter files it reads, so it may sync {source} without some of them unnoticed."
+ );
+ }
+ let Some(ignored) = ignored else {
+ return Ok(false);
+ };
+ match config.insecure_links {
+ None => Err(Failure::Message(format!(
+ "error: rsync {version} can't read {ignored}, so it would sync {source} without it. This happens, e.g., if a directory on the way isn't readable, or if a symlink on the way belongs to neither root nor the user running rsync. Set the source's insecureLinks to true to make rsync read it anyway, or to false to sync without it."
+ ))),
+ Some(true) => {
+ eprintln!(
+ "warning: rsync {version} can't read {ignored}; it reads it with --insecure-links anyway, because insecureLinks is true."
+ );
+ Ok(true)
+ }
+ Some(false) => {
+ eprintln!(
+ "warning: rsync {version} can't read {ignored}, so it syncs {source} without it, because insecureLinks is false."
+ );
+ Ok(false)
+ }
+ }
+}
+
+/// Whether a .rsync-filter file applies to source: one therein, or in a
+/// directory above it.
+fn filtered(source: &str) -> bool {
+ let found = run::output(Command::new("find").arg(source).args([
+ "-name",
+ ".rsync-filter",
+ "-print",
+ "-quit",
+ ]))
+ .map(|out| !out.is_empty())
+ .unwrap_or(false);
+ if found {
+ return true;
+ }
+ let mut above = parent(source);
+ loop {
+ if Path::new(&above).join(".rsync-filter").exists() {
+ return true;
+ }
+ if above == "/" || above == "." {
+ return false;
+ }
+ above = parent(&above);
+ }
+}
+
+/// Like dirname.
+fn parent(path: &str) -> String {
+ let trimmed = path.trim_end_matches('/');
+ if trimmed.is_empty() {
+ return "/".to_string();
+ }
+ match trimmed.rfind('/') {
+ None => ".".to_string(),
+ Some(i) => {
+ let parent = trimmed[..i].trim_end_matches('/');
+ if parent.is_empty() { "/" } else { parent }.to_string()
+ }
+ }
+}
+
+/// Like sort -V: whether version is at least minimum.
+fn version_at_least(version: &str, minimum: &str) -> bool {
+ let parts = |v: &str| -> Vec<u64> {
+ v.split('.')
+ .map(|p| {
+ p.chars()
+ .take_while(char::is_ascii_digit)
+ .collect::<String>()
+ })
+ .map(|p| p.parse().unwrap_or(0))
+ .collect()
+ };
+ parts(version) >= parts(minimum)
+}
+
+/// rsync prints bytes it takes for unprintable, which depends on the
+/// locale, as \#ooo.
+fn unescape(escaped: &str) -> std::path::PathBuf {
+ use std::os::unix::ffi::OsStringExt;
+ let bytes = escaped.as_bytes();
+ let mut out = Vec::with_capacity(bytes.len());
+ let mut i = 0;
+ while i < bytes.len() {
+ let octal = bytes
+ .get(i + 2..i + 5)
+ .filter(|d| d.iter().all(|b| (b'0'..=b'7').contains(b)));
+ let escape = bytes[i] == b'\\' && bytes.get(i + 1) == Some(&b'#');
+ if let (true, Some(digits)) = (escape, octal) {
+ let value = digits
+ .iter()
+ .fold(0u32, |acc, d| acc * 8 + u32::from(d - b'0'));
+ out.push(value as u8);
+ i += 5;
+ } else {
+ out.push(bytes[i]);
+ i += 1;
+ }
+ }
+ std::ffi::OsString::from_vec(out).into()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn versions() {
+ assert!(version_at_least("3.5.0", "3.5.0"));
+ assert!(version_at_least("3.10.1", "3.5.0"));
+ assert!(!version_at_least("3.4.1", "3.5.0"));
+ }
+
+ #[test]
+ fn parents() {
+ assert_eq!(parent("/a/b/"), "/a");
+ assert_eq!(parent("/a"), "/");
+ assert_eq!(parent("a"), ".");
+ assert_eq!(parent("/"), "/");
+ }
+
+ #[test]
+ fn unescaping() {
+ assert_eq!(unescape(r"a\#012b"), std::path::PathBuf::from("a\nb"));
+ assert_eq!(unescape(r"a\#9b"), std::path::PathBuf::from(r"a\#9b"));
+ }
+}
diff --git a/pkgs/populate/src/run.rs b/pkgs/populate/src/run.rs
new file mode 100644
index 0000000..9ea05b3
--- /dev/null
+++ b/pkgs/populate/src/run.rs
@@ -0,0 +1,348 @@
+//! Running commands: on the deploying machine, and on the target, directly
+//! or over ssh, sharing one master connection.
+
+use crate::spec::Target;
+use std::fmt;
+use std::fs;
+use std::io::Write;
+use std::os::unix::fs::PermissionsExt;
+use std::path::{Path, PathBuf};
+use std::process::{Command, ExitStatus, Stdio};
+use std::sync::atomic::{AtomicI32, Ordering};
+
+/// Why populate stops before delivering everything.
+#[derive(Debug)]
+pub enum Failure {
+ /// A message to print.
+ Message(String),
+ /// A command failed, after printing why.
+ Command,
+ /// A signal arrived: SIGHUP, SIGINT, or SIGTERM.
+ Signal(i32),
+}
+
+impl fmt::Display for Failure {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ match self {
+ Failure::Message(message) => f.write_str(message),
+ Failure::Command => f.write_str("a command failed"),
+ Failure::Signal(signal) => write!(f, "got signal {signal}"),
+ }
+ }
+}
+
+pub type Result<T> = std::result::Result<T, Failure>;
+
+/// The signal that arrived last, or 0.
+static SIGNAL: AtomicI32 = AtomicI32::new(0);
+
+extern "C" fn record_signal(signal: libc::c_int) {
+ SIGNAL.store(signal, Ordering::SeqCst);
+}
+
+/// Makes SIGHUP, SIGINT, and SIGTERM get recorded instead of ending
+/// populate, so it can clean up. A signal to the process group, like the
+/// one of an interrupt key, ends the command populate waits for, too, after
+/// which populate stops. A signal to populate alone takes effect once that
+/// command ends.
+pub fn trap_signals() {
+ for signal in [libc::SIGHUP, libc::SIGINT, libc::SIGTERM] {
+ // SAFETY: the handler only stores into an atomic.
+ unsafe {
+ let mut action: libc::sigaction = std::mem::zeroed();
+ action.sa_sigaction = record_signal as extern "C" fn(libc::c_int) as usize;
+ action.sa_flags = libc::SA_RESTART;
+ libc::sigemptyset(&mut action.sa_mask);
+ libc::sigaction(signal, &action, std::ptr::null_mut());
+ }
+ }
+}
+
+/// Ignores further signals, so they don't end cleaning up halfway.
+pub fn ignore_signals() {
+ for signal in [libc::SIGHUP, libc::SIGINT, libc::SIGTERM] {
+ // SAFETY: setting a disposition has no other effect.
+ unsafe {
+ libc::signal(signal, libc::SIG_IGN);
+ }
+ }
+}
+
+/// Fails if a signal arrived.
+pub fn check_signal() -> Result<()> {
+ match SIGNAL.load(Ordering::SeqCst) {
+ 0 => Ok(()),
+ signal => Err(Failure::Signal(signal)),
+ }
+}
+
+/// Quotes a word for POSIX shells.
+pub fn quote(word: &str) -> String {
+ let safe = |c: char| c.is_ascii_alphanumeric() || "-+./:=_@%,".contains(c);
+ if !word.is_empty() && word.chars().all(safe) {
+ word.to_string()
+ } else {
+ format!("'{}'", word.replace('\'', r"'\''"))
+ }
+}
+
+/// Where populate delivers to, and how it gets there.
+pub struct Context {
+ pub target: Target,
+ /// Whether the target is the machine populate runs on, as the user
+ /// running it, so commands run directly instead of over ssh.
+ pub local: bool,
+ /// Populate's private temporary directory, which is $TMPDIR for the
+ /// commands it runs.
+ pub tmpdir: PathBuf,
+ /// The ssh command, with the options for the target and for sharing a
+ /// master connection, as a script in tmpdir, for rsync's -e as well.
+ pub ssh: PathBuf,
+}
+
+impl Context {
+ /// Creates the temporary directory, and the ssh script in it.
+ pub fn new(target: Target, local: Option<bool>) -> Result<Context> {
+ let local = local.unwrap_or_else(|| is_local(&target));
+ let output = Command::new("mktemp")
+ .args(["-dt", "populate.XXXXXXXX"])
+ .stderr(Stdio::inherit())
+ .output()
+ .map_err(|e| Failure::Message(format!("error: can't run mktemp: {e}")))?;
+ if !output.status.success() {
+ return Err(Failure::Command);
+ }
+ let tmpdir = PathBuf::from(String::from_utf8_lossy(&output.stdout).trim_end());
+ let ssh = tmpdir.join("ssh");
+ let context = Context {
+ target,
+ local,
+ tmpdir,
+ ssh,
+ };
+ let mut script = String::from("#!/bin/sh\nexec ssh");
+ for word in context.ssh_options() {
+ script.push(' ');
+ script.push_str(&quote(&word));
+ }
+ script.push_str(" \"$@\"\n");
+ let write = || -> std::io::Result<()> {
+ let mut file = fs::File::create(&context.ssh)?;
+ file.write_all(script.as_bytes())?;
+ fs::set_permissions(&context.ssh, fs::Permissions::from_mode(0o700))
+ };
+ write().map_err(|e| {
+ Failure::Message(format!("error: can't write {}: {e}", context.ssh.display()))
+ })?;
+ Ok(context)
+ }
+
+ fn ssh_options(&self) -> Vec<String> {
+ let mut options = Vec::new();
+ if let Some(user) = &self.target.user {
+ options.extend(["-l".to_string(), user.clone()]);
+ }
+ if let Some(port) = &self.target.port {
+ options.extend(["-p".to_string(), port.clone()]);
+ }
+ let control_path = format!("ControlPath={}/ssh-%r@%h:%p.sock", self.tmpdir.display());
+ for option in ["ControlMaster=auto", &control_path, "ControlPersist=1h"] {
+ options.extend(["-o".to_string(), option.to_string()]);
+ }
+ options.push("-T".to_string());
+ options.extend(self.target.extra_options.iter().cloned());
+ options
+ }
+
+ /// The path on the target, as populate names it in what it prints:
+ /// prefixed by the host if the target isn't local, and by the user as
+ /// well if that isn't root.
+ pub fn name(&self, path: &str) -> String {
+ if self.local {
+ return path.to_string();
+ }
+ let user = match self.target.user.as_deref() {
+ None | Some("") | Some("root") => String::new(),
+ Some(user) => format!("{user}@"),
+ };
+ format!("{user}{}:{path}", self.target.host)
+ }
+
+ /// The path on the target, as rsync names it.
+ pub fn rsync_destination(&self, path: &str) -> String {
+ if self.local {
+ return path.to_string();
+ }
+ let user = match self.target.user.as_deref() {
+ None | Some("") => String::new(),
+ Some(user) => format!("{user}@"),
+ };
+ format!("{user}{}:{path}", self.target.host)
+ }
+
+ /// A command running the shell code script on the target, with args as
+ /// its positional parameters. Over ssh, the remote shell gets it as
+ /// one quoted command, run via sudo if the target says so.
+ pub fn shell(&self, script: &str, args: &[&str]) -> Command {
+ let script = format!("set -efu\n{script}");
+ if self.local {
+ let mut command = Command::new("sh");
+ command.arg("-c").arg(script).arg("sh").args(args);
+ command
+ } else {
+ let mut words = vec![
+ if self.target.sudo {
+ "sudo sh -c"
+ } else {
+ "sh -c"
+ }
+ .to_string(),
+ quote(&script),
+ "sh".to_string(),
+ ];
+ words.extend(args.iter().map(|arg| quote(arg)));
+ let mut command = Command::new(&self.ssh);
+ command.arg(&self.target.host).arg(words.join(" "));
+ command
+ }
+ }
+
+ /// Runs shell code on the target, with its output going where
+ /// populate's errors go.
+ pub fn run(&self, script: &str, args: &[&str]) -> Result<()> {
+ let mut command = self.shell(script, args);
+ command.stdout(stderr());
+ status(&mut command)
+ }
+
+ /// Runs shell code on the target, returning its output.
+ pub fn output(&self, script: &str, args: &[&str]) -> Result<String> {
+ output(&mut self.shell(script, args))
+ }
+
+ /// Stops the ssh master, if any command started one, and removes the
+ /// temporary directory.
+ pub fn clean_up(&self) {
+ if !self.local {
+ let check = Command::new(&self.ssh)
+ .args(["-q", "-O", "check", &self.target.host])
+ .stdin(Stdio::null())
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status();
+ if matches!(check, Ok(status) if status.success()) {
+ let _ = Command::new(&self.ssh)
+ .args(["-q", "-O", "stop", &self.target.host])
+ .stdin(Stdio::null())
+ .status();
+ }
+ }
+ let _ = Command::new("rm").arg("-R").arg(&self.tmpdir).status();
+ }
+}
+
+/// A command running shell code on the deploying machine, in dir, with
+/// args as its positional parameters.
+pub fn local_shell(script: &str, args: &[&str], dir: &Path) -> Command {
+ let mut command = Command::new("sh");
+ command
+ .arg("-c")
+ .arg(format!("set -efu\n{script}"))
+ .arg("sh")
+ .args(args)
+ .current_dir(dir);
+ command
+}
+
+/// Where populate's errors go, for commands whose output goes there too.
+pub fn stderr() -> Stdio {
+ use std::os::fd::AsFd;
+ match std::io::stderr().as_fd().try_clone_to_owned() {
+ Ok(fd) => Stdio::from(fd),
+ Err(_) => Stdio::inherit(),
+ }
+}
+
+fn spawn_error(command: &Command, e: std::io::Error) -> Failure {
+ Failure::Message(format!(
+ "error: can't run {}: {e}",
+ command.get_program().to_string_lossy()
+ ))
+}
+
+/// Runs a command, failing if it fails, or if a signal arrived meanwhile.
+pub fn status(command: &mut Command) -> Result<()> {
+ let result: std::io::Result<ExitStatus> = command.status();
+ check_signal()?;
+ match result {
+ Ok(status) if status.success() => Ok(()),
+ Ok(_) => Err(Failure::Command),
+ Err(e) => Err(spawn_error(command, e)),
+ }
+}
+
+/// Runs a command, returning its output, failing like status.
+pub fn output(command: &mut Command) -> Result<String> {
+ command.stderr(Stdio::inherit());
+ let result = command.output();
+ check_signal()?;
+ match result {
+ Ok(output) if output.status.success() => {
+ Ok(String::from_utf8_lossy(&output.stdout).into_owned())
+ }
+ Ok(_) => Err(Failure::Command),
+ Err(e) => Err(spawn_error(command, e)),
+ }
+}
+
+/// Whether the target is the machine populate runs on, as the user running
+/// it: its user is that user, and its host is localhost or the machine's
+/// name, like krops' lib's isLocalTarget decides during evaluation.
+fn is_local(target: &Target) -> bool {
+ let user = std::env::var("LOGNAME").ok();
+ let host = std::env::var("HOSTNAME").ok().or_else(host_name);
+ target.user == user && (target.host == "localhost" || Some(&target.host) == host.as_ref())
+}
+
+/// The machine's name, from /etc/hostname, as krops' lib reads it.
+fn host_name() -> Option<String> {
+ let text = fs::read_to_string("/etc/hostname").ok()?;
+ let label = |line: &&str| {
+ !line.is_empty()
+ && line.len() <= 63
+ && line.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')
+ && !line.starts_with('-')
+ && !line.ends_with('-')
+ };
+ let names: Vec<&str> = text.lines().filter(label).collect();
+ match names.as_slice() {
+ [name] => Some(name.to_string()),
+ _ => None,
+ }
+}
+
+/// Tells whether path is a path in the store itself, not one below it.
+pub fn is_store_path(path: &str) -> bool {
+ Path::new(path).parent() == Some(Path::new("/nix/store")) && path.len() > "/nix/store/".len()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn quoting() {
+ assert_eq!(quote("abc/d.e"), "abc/d.e");
+ assert_eq!(quote(""), "''");
+ assert_eq!(quote("a b"), "'a b'");
+ assert_eq!(quote("it's"), r"'it'\''s'");
+ }
+
+ #[test]
+ fn store_paths() {
+ assert!(is_store_path("/nix/store/abc-x"));
+ assert!(!is_store_path("/nix/store/abc-x/y"));
+ assert!(!is_store_path("/nix/store"));
+ assert!(!is_store_path("/etc"));
+ }
+}
diff --git a/pkgs/populate/src/sh.rs b/pkgs/populate/src/sh.rs
new file mode 100644
index 0000000..5d6bf3e
--- /dev/null
+++ b/pkgs/populate/src/sh.rs
@@ -0,0 +1,485 @@
+//! Shell code that runs on the target, where populate can't rely on more
+//! than a POSIX shell and the usual tools. Values come as positional
+//! parameters, never quoted into the code.
+
+/// The directories below the sentinel, once that is a directory, which have
+/// a directory for each source that needs one. Besides, there is backup,
+/// which holds copies of the sources.
+pub const STATE_KINDS: &str = "derived flake-inputs flake-uploads flakes patches";
+
+/// Defines use_state SENTINEL NAME..., which prepares keeping state below
+/// the sentinel for the sources named NAME. The sentinel only needs to
+/// exist, so it may be a file, which becomes a directory. Populate changes
+/// and removes what's in its state directories, and a symlink in place of
+/// one would make it do that to whatever the link points to, so it refuses
+/// to use those.
+pub fn use_state() -> String {
+ format!(
+ r#"
+refuse_link() {{
+ if test -L "$1"; then
+ >&2 printf 'error: %s is a symlink, but populate needs a directory of its own there\n' "$1"
+ exit 1
+ fi
+}}
+use_state() {{
+ sentinel=$1
+ shift
+ refuse_link "$sentinel"
+ refuse_link "$sentinel"/backup
+ for state_kind in {STATE_KINDS}; do
+ refuse_link "$sentinel"/"$state_kind"
+ for state_name; do
+ refuse_link "$sentinel"/"$state_kind"/"$state_name"
+ done
+ done
+ if test -f "$sentinel"; then
+ rm "$sentinel"
+ mkdir "$sentinel"
+ fi
+}}
+"#
+ )
+}
+
+/// Defines remove_tree PATH, which removes what's at PATH, with everything
+/// below it, also what isn't writable. A symlink gets removed itself,
+/// instead of followed. Only directories get made writable, which is what
+/// removing what's in them needs: a file may be a hard link to one
+/// elsewhere, which shares its mode.
+pub const REMOVE_TREE: &str = r#"
+remove_tree() {
+ if test -L "$1"; then
+ rm "$1"
+ elif test -e "$1"; then
+ find "$1" -type d -exec chmod u+w {} +
+ rm -fR "$1"
+ fi
+}
+"#;
+
+/// Defines remove_link PATH NAME, which removes a symlink at PATH, saying
+/// so, with NAME for PATH, because the link may be there on purpose, but
+/// populate never follows it: what a source delivers would change or remove
+/// what the link points to. remove_foreign_link PATH NAME does so unless
+/// the link points into the store, like the links populate puts there
+/// itself, and make_room_for_link PATH NAME makes room at PATH for a link
+/// into the store, which Nix only puts in place of another link into the
+/// store.
+pub const LINKS: &str = r#"
+remove_link() {
+ if test -L "$1"; then
+ >&2 printf "replacing symlink '%s', which points to '%s'...\n" \
+ "$2" "$(readlink "$1")"
+ rm "$1"
+ fi
+}
+remove_foreign_link() {
+ if test -L "$1"; then
+ # Resolved, so a link like /nix/store/../.. doesn't count.
+ case $(readlink -f "$1") in
+ /nix/store/*) ;;
+ *) remove_link "$1" "$2" ;;
+ esac
+ fi
+}
+make_room_for_link() {
+ remove_foreign_link "$1" "$2"
+ if ! test -L "$1" && test -e "$1"; then
+ rm -fR "$1"
+ fi
+}
+"#;
+
+/// Defines remove_others DIR LOG NAME..., which removes the entries of DIR
+/// that aren't named NAME. find, unlike a loop reading names, copes with
+/// every name, e.g. ones that end in blanks or contain newlines. With LOG
+/// empty, entries get removed like remove_tree does; for entries of the
+/// target directory, LOG is the directory's name in the line printed for
+/// each entry, and nothing gets made writable first. The names are
+/// portable file names, which mean themselves as patterns of find.
+pub const REMOVE_OTHERS: &str = r#"
+remove_others() {
+ dir=$1
+ log=$2
+ shift 2
+ n=$#
+ while test "$n" -gt 0; do
+ set -- "$@" ! -name "$1"
+ shift
+ n=$((n - 1))
+ done
+ find "$dir" -mindepth 1 -maxdepth 1 "$@" -exec sh -c '
+ log=$1
+ shift
+ for path; do
+ if test -n "$log"; then
+ >&2 printf "pruning '\''%s/%s'\''...\n" "$log" "${path##*/}"
+ rm -fR "$path"
+ elif test -L "$path"; then
+ rm "$path"
+ else
+ find "$path" -type d -exec chmod u+w {\} +
+ rm -fR "$path"
+ fi
+ done
+ ' sh "$log" {} +
+}
+"#;
+
+/// check SENTINEL NAME DIR FORCE: fails unless the sentinel exists, creating
+/// it and the target directory DIR first if FORCE isn't empty. NAME is the
+/// sentinel's name in messages.
+pub const CHECK: &str = r#"
+sentinel=$1 name=$2 dir=$3 force=$4
+# Before anything touches it, which would touch what it points to.
+if test -L "$sentinel"; then
+ >&2 printf 'error: sentinel file is a symlink: %s\n' "$name"
+ exit 1
+fi
+if test -n "$force"; then
+ mkdir -vp "$dir" >&2
+ if ! test -e "$sentinel"; then
+ touch "$sentinel"
+ fi
+fi
+if ! test -e "$sentinel"; then
+ >&2 printf 'error: missing sentinel file: %s\n' "$name"
+ exit 1
+fi
+"#;
+
+/// backup DIR NAME: copies the target directory DIR to the sentinel's
+/// backup directory. NAME is the sentinel's name in messages.
+pub fn backup() -> String {
+ use_state()
+ + r#"
+dir=$1 name=$2
+use_state "$dir"/.populate
+if ! test -d "$dir"/.populate; then
+ >&2 printf 'error: sentinel file is not a directory: %s\n' "$name"
+ exit 1
+fi
+rsync >&2 \
+ -aAXF \
+ --delete \
+ --exclude /.populate \
+ --link-dest="$dir" \
+ "$dir"/ \
+ "$dir"/.populate/backup/
+"#
+}
+
+/// prune DIR LOG NAME...: removes the entries of the target directory DIR
+/// that aren't sources named NAME, logging them with LOG for DIR, and
+/// populate's state that only sources that are gone needed. The names
+/// after -- are the ones of flake and flake-input sources, whose roots for
+/// the garbage collector stay.
+pub fn prune() -> String {
+ use_state()
+ + REMOVE_TREE
+ + REMOVE_OTHERS
+ + r#"
+dir=$1 log=$2
+shift 2
+names=
+while test "$1" != --; do
+ names="$names $1"
+ shift
+done
+shift
+use_state "$dir"/.populate
+cd "$dir"
+# The names are portable file names, so splitting them at blanks is fine.
+# shellcheck disable=SC2086
+remove_others . "$log" .populate $names
+roots=.populate/flake-inputs
+if test -d "$roots"; then
+ remove_others "$roots" "" "$@"
+ if test -z "$(ls -A "$roots")"; then
+ rmdir "$roots"
+ fi
+fi
+remove_tree .populate/flake-uploads
+"#
+}
+
+/// break_hard_links PATH NAME: prepares PATH for rsync. rsync writes
+/// through a symlink at the target path, e.g. one a symlink or flake source
+/// left there, into what it points to. And rsync fixes the mode and time of
+/// a file it takes for unchanged in place, which changes every hard link to
+/// it, e.g. one elsewhere, or a backup's. So each file with other hard
+/// links gets replaced by a copy of its own first.
+pub fn break_hard_links() -> String {
+ LINKS.to_string()
+ + r#"
+remove_link "$1" "$2"
+if test -d "$1"; then
+ find "$1" -type f -links +1 -exec sh -c '
+ for file; do
+ copy=$(mktemp "$file.XXXXXX")
+ cp -p "$file" "$copy"
+ mv -f "$copy" "$file"
+ done
+ ' sh {} +
+fi
+"#
+}
+
+/// derivation PATH NAME TEXT: builds the Nix expression TEXT on the target,
+/// with a link to the result at PATH.
+pub fn derivation() -> String {
+ LINKS.to_string()
+ + r#"
+make_room_for_link "$1" "$2"
+nix-build -E "$3" -o "$1" >&2
+"#
+}
+
+/// symlink PATH TARGET: makes PATH a symlink to TARGET.
+pub const SYMLINK: &str = r#"
+ln -fnsT "$2" "$1"
+"#;
+
+/// Defines git_problems PATH, which prints why populate can't use the .git
+/// directory of the Git work tree at PATH, or nothing. A .git directory
+/// that git clone and populate made has no symlinks, no commondir file,
+/// which names another repository, no linked work trees, and only the
+/// configuration they write: the repository's format, the remote, branches,
+/// and submodules, whose repositories have their work tree in the work
+/// tree.
+pub const GIT_PROBLEMS: &str = r#"
+git_problems() (
+ path=$1
+ git_dir=$1/.git
+ links=$(find "$git_dir" -type l | head -n 3 | paste -s -d ' ' -)
+ if test -n "$links"; then
+ printf '%s contains symlinks: %s' "$git_dir" "$links"
+ exit
+ fi
+ common=$(find "$git_dir" -name commondir | head -n 1)
+ if test -n "$common"; then
+ printf '%s names another repository' "$common"
+ exit
+ fi
+ if test -e "$git_dir"/worktrees; then
+ printf '%s has linked work trees' "$git_dir"
+ exit
+ fi
+ find "$git_dir" -name config -type f | while IFS= read -r config; do
+ command git config --file "$config" --list --name-only |
+ while IFS= read -r key; do
+ case $key in
+ core.repositoryformatversion|core.filemode|core.bare|\
+ core.logallrefupdates|core.symlinks|core.ignorecase|\
+ core.precomposeunicode|\
+ extensions.objectformat|extensions.refstorage|\
+ remote.origin.url|remote.origin.fetch|\
+ branch.*.remote|branch.*.merge|\
+ submodule.active|submodule.*.url|submodule.*.active) ;;
+ core.worktree)
+ # Submodules' repositories have their work tree in the work tree.
+ worktree=$(command git config --file "$config" core.worktree)
+ dir=$(dirname "$config")
+ case $dir/$worktree in
+ "$git_dir"/modules/*)
+ resolved=$(cd "$dir" && cd "$worktree" 2>/dev/null && pwd -P) || :
+ case $resolved in
+ "$(cd "$path" && pwd -P)"/*) continue ;;
+ esac
+ ;;
+ esac
+ printf '%s sets core.worktree to %s' "$config" "$worktree"
+ exit 1
+ ;;
+ *)
+ printf '%s sets %s, which neither git clone nor populate does' "$config" "$key"
+ exit 1
+ ;;
+ esac
+ done || exit 1
+ done || :
+)
+"#;
+
+/// git PATH NAME URL REF SHALLOW FETCH_ALWAYS RECLONE EXCLUDE...: makes
+/// PATH a Git work tree of the repository at URL, with REF checked out,
+/// and everything removed that Git doesn't track, except what matches an
+/// EXCLUDE pattern. SHALLOW, FETCH_ALWAYS, and RECLONE are flags, empty
+/// for false. NAME is PATH's name in messages.
+pub fn git() -> String {
+ LINKS.to_string()
+ + GIT_PROBLEMS
+ + r#"
+path=$1 name=$2 url=$3 ref=$4 shallow=$5 fetch_always=$6 reclone=$7
+shift 7
+# Hooks and commands a repository's configuration names run with Git's
+# commands, so populate's Git doesn't run any. The configuration gets
+# checked as well, but Git also runs a hook it has itself.
+git() {
+ command git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@"
+}
+# A target path that doesn't look like a Git work tree gets removed, e.g.
+# when it had another type before. A work tree that a symlink points to,
+# or whose .git is a symlink or a file naming a repository elsewhere, isn't
+# populate's own: fetching, resetting, and cleaning would change what's
+# elsewhere.
+remove_link "$path" "$name"
+if test -L "$path"/.git || ! test -d "$path"/.git; then
+ rm -fR "$path"
+fi
+# A .git directory can name places elsewhere too, which Git's commands
+# change, or commands, which they run. Populate only uses a work tree
+# whose .git directory has nothing like that, and else replaces it if
+# reclone is set, or fails, telling why.
+if test -e "$path"; then
+ if problems=$(git_problems "$path"); then :; else
+ exit 1
+ fi
+ if test -n "$problems"; then
+ if test -n "$reclone"; then
+ >&2 printf "replacing the Git work tree '%s', because %s...\n" \
+ "$name" "$problems"
+ rm -fR "$path"
+ else
+ >&2 printf 'error: populate doesn'"'"'t use the Git work tree %s, because %s. Fix that, remove the work tree, or set the source'"'"'s reclone to true to replace it by a fresh clone.\n' \
+ "$name" "$problems"
+ exit 1
+ fi
+ fi
+fi
+if ! test -e "$path"; then
+ if test -n "$shallow"; then
+ git init "$path"
+ else
+ git clone --recurse-submodules "$url" "$path"
+ fi
+fi
+cd "$path"
+if ! origin=$(git config remote.origin.url); then
+ git remote add origin "$url"
+elif test "$origin" != "$url"; then
+ git remote set-url origin "$url"
+fi
+# A commit, given by its hash or a prefix thereof, gets fetched only if the
+# work tree lacks it, unless fetchAlways is set. A branch or tag gets
+# fetched on every run, so the work tree follows it: resetting to its name
+# would reset to a local branch, which fetching doesn't move.
+fetch() {
+ if test -n "$shallow"; then
+ git fetch --depth=1 origin "$1"
+ else
+ git fetch origin "$1"
+ fi
+}
+fetch_commit() {
+ if test -n "$shallow"; then
+ fetch "$hash"
+ else
+ git fetch origin
+ fi
+}
+if printf %s "$ref" | grep -Eqx '[0-9a-f]{4,40}'; then
+ hash=$ref
+ if ! test "$(git log --format=%H -1)" = "$hash"; then
+ if test -n "$fetch_always"; then
+ fetch_commit
+ elif ! git log -1 "$hash" >/dev/null 2>&1; then
+ fetch_commit
+ fi
+ fi
+else
+ fetch "$ref"
+ hash=$(git rev-parse --verify 'FETCH_HEAD^{commit}')
+fi
+# Also if HEAD is there already: rewriting a lock file in place, e.g.,
+# changes the work tree, and stages the lock file if Nix created it.
+git reset --hard "$hash" >&2
+git submodule update --init --recursive
+n=$#
+while test "$n" -gt 0; do
+ set -- "$@" -e "$1"
+ shift
+ n=$((n - 1))
+done
+git clean -dfx "$@"
+"#
+}
+
+/// Shell code running on the deploying machine, in the source's working
+/// directory.
+pub mod local {
+ /// pass_info DIR NAME: prints what tells whether the pass source of the
+ /// passwords below NAME in the store DIR changed: the last commits of
+ /// NAME, and of what symlinks therein point to, whose changes the
+ /// commits of NAME miss. Prints nothing if the store isn't a Git
+ /// repository.
+ pub const PASS_INFO: &str = r#"
+dir=$1 name=$2
+if test -e "$dir"/.git; then
+ printf '%s %s\n' "$name" "$(
+ git -C "$dir" log -1 --format=%H "$name"
+ find "$dir"/"$name" -type l -exec realpath {} + |
+ sort |
+ xargs -r -n 1 git -C "$dir" log -1 --format=%H
+ )"
+fi
+"#;
+
+ /// pass DIR NAME INFO: decrypts the passwords below NAME in the store
+ /// DIR into the working directory, with the times of their last
+ /// commits, and INFO, unless empty, in .pass_info.
+ pub const PASS: &str = r#"
+dir=$1 name=$2 info=$3
+umask 0077
+chmod 0700 "$PWD"
+prefix=$dir/$name
+find "$prefix" -type f -follow ! -name .gpg-id |
+while read -r gpg_path; do
+ rel_name=${gpg_path#"$prefix"}
+ rel_name=${rel_name%.gpg}
+ pass_date=$(
+ if test -e "$dir"/.git; then
+ git -C "$dir" log -1 --format=%aI "$gpg_path"
+ fi
+ )
+ pass_name=$name/$rel_name
+ pass_out=$PWD/$rel_name
+ mkdir -p "$(dirname "$pass_out")"
+ PASSWORD_STORE_DIR=$dir pass show "$pass_name" > "$pass_out"
+ if [ -n "$pass_date" ]; then
+ touch -d "$pass_date" "$pass_out"
+ fi
+done
+if test -n "$info"; then
+ printf '%s\n' "$info" > "$PWD"/.pass_info
+fi
+"#;
+
+ /// passage HOST: decrypts the secrets in $PASSAGE_DIR into the working
+ /// directory, with the times of their files. HOST is the first label
+ /// of the target's host, for the command a failure prints.
+ pub const PASSAGE: &str = r#"
+host=$1
+umask 0077
+chmod 0700 "$PWD"
+find "$PASSAGE_DIR" -type f -name \*.age -follow |
+while read -r age_path; do
+ pass_name=${age_path#"$PASSAGE_DIR"/}
+ pass_name=${pass_name%.age}
+ pass_out=$PWD/$pass_name
+ mkdir -p "$(dirname "$pass_out")"
+ if ! passage show "$pass_name" > "$pass_out" 2>&1; then
+ echo ++ secrets "$host" show "$pass_name" >&2
+ jq -Rr 'select(test("^[^[]"))' "$pass_out" >&2
+ exit 1
+ fi
+ touch -r "$age_path" "$pass_out"
+done
+"#;
+}
+
+/// pass_info PATH: prints the .pass_info the pass source at PATH got last.
+pub const PASS_INFO: &str = r#"
+cat "$1"/.pass_info || :
+"#;
diff --git a/pkgs/populate/src/spec.rs b/pkgs/populate/src/spec.rs
index 089d330..8fc74fc 100644
--- a/pkgs/populate/src/spec.rs
+++ b/pkgs/populate/src/spec.rs
@@ -1,8 +1,5 @@
//! The spec: populate's arguments as JSON, as docs/design.md describes them.
-// Delivering sources isn't implemented yet, so most of the spec is unused.
-#![allow(dead_code)]
-
use serde::Deserialize;
use serde_json::{Map, Value};
use std::collections::BTreeMap;
@@ -26,6 +23,8 @@ pub const SOURCE_TYPES: &[&str] = &[
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Spec {
+ /// Checked before the rest, so it's only part of the shape here.
+ #[allow(dead_code)]
pub version: u64,
#[serde(default)]
pub backup: bool,
@@ -37,7 +36,7 @@ pub struct Spec {
pub target: Target,
}
-#[derive(Debug, Deserialize)]
+#[derive(Clone, Debug, Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct Target {
#[serde(default)]
@@ -50,6 +49,10 @@ pub struct Target {
pub sudo: bool,
#[serde(default)]
pub extra_options: Vec<String>,
+ /// Whether the target is the machine populate runs on, or null to
+ /// decide that when it runs.
+ #[serde(default)]
+ pub local: Option<bool>,
}
/// A source: its type, and the settings of that type, under the type's name.
generated by cgit v1.3.1 (git 2.54.0) at 2026-11-14 12:45:33 +0000