populate: deliver sources other than flakes with the Rust program

The program delivers derivation, file, git, pass, passage, pipe, and
symlink sources, makes backups, and prunes, as the shell implementation
does, with the same messages.  Commands on the target are fixed shell
snippets in src/sh.rs, which get values as positional parameters, run
directly for local targets and over ssh, sharing one master connection,
for remote ones.  rsync's transfers and its check for .rsync-filter files
it can't read are in src/rsync.rs.  Given flake or flake-input sources, or
sources drawing inputs from others, the program fails before changing
anything, saying that these aren't implemented yet.

SIGHUP, SIGINT, and SIGTERM make populate stop once the command it waits
for ends, stop the ssh master, remove its temporary directory, and exit
with 128 plus the signal's number.

The spec's target gets local, which rust.nix sets during evaluation, like
the other functions of krops decide, and which the program decides when
it runs if it's null.  The spec comes on file descriptor 3, so the
commands populate runs keep standard input, and the tools populate runs
come first in PATH.

1 files changed, 485 insertions, 0 deletions

diff --git a/pkgs/populate/src/sh.rs b/pkgs/populate/src/sh.rs
new file mode 100644
index 0000000..5d6bf3e
--- /dev/null
+++ b/pkgs/populate/src/sh.rs
@@ -0,0 +1,485 @@
+//! Shell code that runs on the target, where populate can't rely on more
+//! than a POSIX shell and the usual tools.  Values come as positional
+//! parameters, never quoted into the code.
+
+/// The directories below the sentinel, once that is a directory, which have
+/// a directory for each source that needs one.  Besides, there is backup,
+/// which holds copies of the sources.
+pub const STATE_KINDS: &str = "derived flake-inputs flake-uploads flakes patches";
+
+/// Defines use_state SENTINEL NAME..., which prepares keeping state below
+/// the sentinel for the sources named NAME.  The sentinel only needs to
+/// exist, so it may be a file, which becomes a directory.  Populate changes
+/// and removes what's in its state directories, and a symlink in place of
+/// one would make it do that to whatever the link points to, so it refuses
+/// to use those.
+pub fn use_state() -> String {
+    format!(
+        r#"
+refuse_link() {{
+  if test -L "$1"; then
+    >&2 printf 'error: %s is a symlink, but populate needs a directory of its own there\n' "$1"
+    exit 1
+  fi
+}}
+use_state() {{
+  sentinel=$1
+  shift
+  refuse_link "$sentinel"
+  refuse_link "$sentinel"/backup
+  for state_kind in {STATE_KINDS}; do
+    refuse_link "$sentinel"/"$state_kind"
+    for state_name; do
+      refuse_link "$sentinel"/"$state_kind"/"$state_name"
+    done
+  done
+  if test -f "$sentinel"; then
+    rm "$sentinel"
+    mkdir "$sentinel"
+  fi
+}}
+"#
+    )
+}
+
+/// Defines remove_tree PATH, which removes what's at PATH, with everything
+/// below it, also what isn't writable.  A symlink gets removed itself,
+/// instead of followed.  Only directories get made writable, which is what
+/// removing what's in them needs: a file may be a hard link to one
+/// elsewhere, which shares its mode.
+pub const REMOVE_TREE: &str = r#"
+remove_tree() {
+  if test -L "$1"; then
+    rm "$1"
+  elif test -e "$1"; then
+    find "$1" -type d -exec chmod u+w {} +
+    rm -fR "$1"
+  fi
+}
+"#;
+
+/// Defines remove_link PATH NAME, which removes a symlink at PATH, saying
+/// so, with NAME for PATH, because the link may be there on purpose, but
+/// populate never follows it: what a source delivers would change or remove
+/// what the link points to.  remove_foreign_link PATH NAME does so unless
+/// the link points into the store, like the links populate puts there
+/// itself, and make_room_for_link PATH NAME makes room at PATH for a link
+/// into the store, which Nix only puts in place of another link into the
+/// store.
+pub const LINKS: &str = r#"
+remove_link() {
+  if test -L "$1"; then
+    >&2 printf "replacing symlink '%s', which points to '%s'...\n" \
+        "$2" "$(readlink "$1")"
+    rm "$1"
+  fi
+}
+remove_foreign_link() {
+  if test -L "$1"; then
+    # Resolved, so a link like /nix/store/../.. doesn't count.
+    case $(readlink -f "$1") in
+      /nix/store/*) ;;
+      *) remove_link "$1" "$2" ;;
+    esac
+  fi
+}
+make_room_for_link() {
+  remove_foreign_link "$1" "$2"
+  if ! test -L "$1" && test -e "$1"; then
+    rm -fR "$1"
+  fi
+}
+"#;
+
+/// Defines remove_others DIR LOG NAME..., which removes the entries of DIR
+/// that aren't named NAME.  find, unlike a loop reading names, copes with
+/// every name, e.g. ones that end in blanks or contain newlines.  With LOG
+/// empty, entries get removed like remove_tree does; for entries of the
+/// target directory, LOG is the directory's name in the line printed for
+/// each entry, and nothing gets made writable first.  The names are
+/// portable file names, which mean themselves as patterns of find.
+pub const REMOVE_OTHERS: &str = r#"
+remove_others() {
+  dir=$1
+  log=$2
+  shift 2
+  n=$#
+  while test "$n" -gt 0; do
+    set -- "$@" ! -name "$1"
+    shift
+    n=$((n - 1))
+  done
+  find "$dir" -mindepth 1 -maxdepth 1 "$@" -exec sh -c '
+    log=$1
+    shift
+    for path; do
+      if test -n "$log"; then
+        >&2 printf "pruning '\''%s/%s'\''...\n" "$log" "${path##*/}"
+        rm -fR "$path"
+      elif test -L "$path"; then
+        rm "$path"
+      else
+        find "$path" -type d -exec chmod u+w {\} +
+        rm -fR "$path"
+      fi
+    done
+  ' sh "$log" {} +
+}
+"#;
+
+/// check SENTINEL NAME DIR FORCE: fails unless the sentinel exists, creating
+/// it and the target directory DIR first if FORCE isn't empty.  NAME is the
+/// sentinel's name in messages.
+pub const CHECK: &str = r#"
+sentinel=$1 name=$2 dir=$3 force=$4
+# Before anything touches it, which would touch what it points to.
+if test -L "$sentinel"; then
+  >&2 printf 'error: sentinel file is a symlink: %s\n' "$name"
+  exit 1
+fi
+if test -n "$force"; then
+  mkdir -vp "$dir" >&2
+  if ! test -e "$sentinel"; then
+    touch "$sentinel"
+  fi
+fi
+if ! test -e "$sentinel"; then
+  >&2 printf 'error: missing sentinel file: %s\n' "$name"
+  exit 1
+fi
+"#;
+
+/// backup DIR NAME: copies the target directory DIR to the sentinel's
+/// backup directory.  NAME is the sentinel's name in messages.
+pub fn backup() -> String {
+    use_state()
+        + r#"
+dir=$1 name=$2
+use_state "$dir"/.populate
+if ! test -d "$dir"/.populate; then
+  >&2 printf 'error: sentinel file is not a directory: %s\n' "$name"
+  exit 1
+fi
+rsync >&2 \
+    -aAXF \
+    --delete \
+    --exclude /.populate \
+    --link-dest="$dir" \
+    "$dir"/ \
+    "$dir"/.populate/backup/
+"#
+}
+
+/// prune DIR LOG NAME...: removes the entries of the target directory DIR
+/// that aren't sources named NAME, logging them with LOG for DIR, and
+/// populate's state that only sources that are gone needed.  The names
+/// after -- are the ones of flake and flake-input sources, whose roots for
+/// the garbage collector stay.
+pub fn prune() -> String {
+    use_state()
+        + REMOVE_TREE
+        + REMOVE_OTHERS
+        + r#"
+dir=$1 log=$2
+shift 2
+names=
+while test "$1" != --; do
+  names="$names $1"
+  shift
+done
+shift
+use_state "$dir"/.populate
+cd "$dir"
+# The names are portable file names, so splitting them at blanks is fine.
+# shellcheck disable=SC2086
+remove_others . "$log" .populate $names
+roots=.populate/flake-inputs
+if test -d "$roots"; then
+  remove_others "$roots" "" "$@"
+  if test -z "$(ls -A "$roots")"; then
+    rmdir "$roots"
+  fi
+fi
+remove_tree .populate/flake-uploads
+"#
+}
+
+/// break_hard_links PATH NAME: prepares PATH for rsync.  rsync writes
+/// through a symlink at the target path, e.g. one a symlink or flake source
+/// left there, into what it points to.  And rsync fixes the mode and time of
+/// a file it takes for unchanged in place, which changes every hard link to
+/// it, e.g. one elsewhere, or a backup's.  So each file with other hard
+/// links gets replaced by a copy of its own first.
+pub fn break_hard_links() -> String {
+    LINKS.to_string()
+        + r#"
+remove_link "$1" "$2"
+if test -d "$1"; then
+  find "$1" -type f -links +1 -exec sh -c '
+    for file; do
+      copy=$(mktemp "$file.XXXXXX")
+      cp -p "$file" "$copy"
+      mv -f "$copy" "$file"
+    done
+  ' sh {} +
+fi
+"#
+}
+
+/// derivation PATH NAME TEXT: builds the Nix expression TEXT on the target,
+/// with a link to the result at PATH.
+pub fn derivation() -> String {
+    LINKS.to_string()
+        + r#"
+make_room_for_link "$1" "$2"
+nix-build -E "$3" -o "$1" >&2
+"#
+}
+
+/// symlink PATH TARGET: makes PATH a symlink to TARGET.
+pub const SYMLINK: &str = r#"
+ln -fnsT "$2" "$1"
+"#;
+
+/// Defines git_problems PATH, which prints why populate can't use the .git
+/// directory of the Git work tree at PATH, or nothing.  A .git directory
+/// that git clone and populate made has no symlinks, no commondir file,
+/// which names another repository, no linked work trees, and only the
+/// configuration they write: the repository's format, the remote, branches,
+/// and submodules, whose repositories have their work tree in the work
+/// tree.
+pub const GIT_PROBLEMS: &str = r#"
+git_problems() (
+  path=$1
+  git_dir=$1/.git
+  links=$(find "$git_dir" -type l | head -n 3 | paste -s -d ' ' -)
+  if test -n "$links"; then
+    printf '%s contains symlinks: %s' "$git_dir" "$links"
+    exit
+  fi
+  common=$(find "$git_dir" -name commondir | head -n 1)
+  if test -n "$common"; then
+    printf '%s names another repository' "$common"
+    exit
+  fi
+  if test -e "$git_dir"/worktrees; then
+    printf '%s has linked work trees' "$git_dir"
+    exit
+  fi
+  find "$git_dir" -name config -type f | while IFS= read -r config; do
+    command git config --file "$config" --list --name-only |
+    while IFS= read -r key; do
+      case $key in
+        core.repositoryformatversion|core.filemode|core.bare|\
+        core.logallrefupdates|core.symlinks|core.ignorecase|\
+        core.precomposeunicode|\
+        extensions.objectformat|extensions.refstorage|\
+        remote.origin.url|remote.origin.fetch|\
+        branch.*.remote|branch.*.merge|\
+        submodule.active|submodule.*.url|submodule.*.active) ;;
+        core.worktree)
+          # Submodules' repositories have their work tree in the work tree.
+          worktree=$(command git config --file "$config" core.worktree)
+          dir=$(dirname "$config")
+          case $dir/$worktree in
+            "$git_dir"/modules/*)
+              resolved=$(cd "$dir" && cd "$worktree" 2>/dev/null && pwd -P) || :
+              case $resolved in
+                "$(cd "$path" && pwd -P)"/*) continue ;;
+              esac
+              ;;
+          esac
+          printf '%s sets core.worktree to %s' "$config" "$worktree"
+          exit 1
+          ;;
+        *)
+          printf '%s sets %s, which neither git clone nor populate does' "$config" "$key"
+          exit 1
+          ;;
+      esac
+    done || exit 1
+  done || :
+)
+"#;
+
+/// git PATH NAME URL REF SHALLOW FETCH_ALWAYS RECLONE EXCLUDE...: makes
+/// PATH a Git work tree of the repository at URL, with REF checked out,
+/// and everything removed that Git doesn't track, except what matches an
+/// EXCLUDE pattern.  SHALLOW, FETCH_ALWAYS, and RECLONE are flags, empty
+/// for false.  NAME is PATH's name in messages.
+pub fn git() -> String {
+    LINKS.to_string()
+        + GIT_PROBLEMS
+        + r#"
+path=$1 name=$2 url=$3 ref=$4 shallow=$5 fetch_always=$6 reclone=$7
+shift 7
+# Hooks and commands a repository's configuration names run with Git's
+# commands, so populate's Git doesn't run any.  The configuration gets
+# checked as well, but Git also runs a hook it has itself.
+git() {
+  command git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@"
+}
+# A target path that doesn't look like a Git work tree gets removed, e.g.
+# when it had another type before.  A work tree that a symlink points to,
+# or whose .git is a symlink or a file naming a repository elsewhere, isn't
+# populate's own: fetching, resetting, and cleaning would change what's
+# elsewhere.
+remove_link "$path" "$name"
+if test -L "$path"/.git || ! test -d "$path"/.git; then
+  rm -fR "$path"
+fi
+# A .git directory can name places elsewhere too, which Git's commands
+# change, or commands, which they run.  Populate only uses a work tree
+# whose .git directory has nothing like that, and else replaces it if
+# reclone is set, or fails, telling why.
+if test -e "$path"; then
+  if problems=$(git_problems "$path"); then :; else
+    exit 1
+  fi
+  if test -n "$problems"; then
+    if test -n "$reclone"; then
+      >&2 printf "replacing the Git work tree '%s', because %s...\n" \
+          "$name" "$problems"
+      rm -fR "$path"
+    else
+      >&2 printf 'error: populate doesn'"'"'t use the Git work tree %s, because %s.  Fix that, remove the work tree, or set the source'"'"'s reclone to true to replace it by a fresh clone.\n' \
+          "$name" "$problems"
+      exit 1
+    fi
+  fi
+fi
+if ! test -e "$path"; then
+  if test -n "$shallow"; then
+    git init "$path"
+  else
+    git clone --recurse-submodules "$url" "$path"
+  fi
+fi
+cd "$path"
+if ! origin=$(git config remote.origin.url); then
+  git remote add origin "$url"
+elif test "$origin" != "$url"; then
+  git remote set-url origin "$url"
+fi
+# A commit, given by its hash or a prefix thereof, gets fetched only if the
+# work tree lacks it, unless fetchAlways is set.  A branch or tag gets
+# fetched on every run, so the work tree follows it: resetting to its name
+# would reset to a local branch, which fetching doesn't move.
+fetch() {
+  if test -n "$shallow"; then
+    git fetch --depth=1 origin "$1"
+  else
+    git fetch origin "$1"
+  fi
+}
+fetch_commit() {
+  if test -n "$shallow"; then
+    fetch "$hash"
+  else
+    git fetch origin
+  fi
+}
+if printf %s "$ref" | grep -Eqx '[0-9a-f]{4,40}'; then
+  hash=$ref
+  if ! test "$(git log --format=%H -1)" = "$hash"; then
+    if test -n "$fetch_always"; then
+      fetch_commit
+    elif ! git log -1 "$hash" >/dev/null 2>&1; then
+      fetch_commit
+    fi
+  fi
+else
+  fetch "$ref"
+  hash=$(git rev-parse --verify 'FETCH_HEAD^{commit}')
+fi
+# Also if HEAD is there already: rewriting a lock file in place, e.g.,
+# changes the work tree, and stages the lock file if Nix created it.
+git reset --hard "$hash" >&2
+git submodule update --init --recursive
+n=$#
+while test "$n" -gt 0; do
+  set -- "$@" -e "$1"
+  shift
+  n=$((n - 1))
+done
+git clean -dfx "$@"
+"#
+}
+
+/// Shell code running on the deploying machine, in the source's working
+/// directory.
+pub mod local {
+    /// pass_info DIR NAME: prints what tells whether the pass source of the
+    /// passwords below NAME in the store DIR changed: the last commits of
+    /// NAME, and of what symlinks therein point to, whose changes the
+    /// commits of NAME miss.  Prints nothing if the store isn't a Git
+    /// repository.
+    pub const PASS_INFO: &str = r#"
+dir=$1 name=$2
+if test -e "$dir"/.git; then
+  printf '%s %s\n' "$name" "$(
+    git -C "$dir" log -1 --format=%H "$name"
+    find "$dir"/"$name" -type l -exec realpath {} + |
+      sort |
+      xargs -r -n 1 git -C "$dir" log -1 --format=%H
+  )"
+fi
+"#;
+
+    /// pass DIR NAME INFO: decrypts the passwords below NAME in the store
+    /// DIR into the working directory, with the times of their last
+    /// commits, and INFO, unless empty, in .pass_info.
+    pub const PASS: &str = r#"
+dir=$1 name=$2 info=$3
+umask 0077
+chmod 0700 "$PWD"
+prefix=$dir/$name
+find "$prefix" -type f -follow ! -name .gpg-id |
+while read -r gpg_path; do
+  rel_name=${gpg_path#"$prefix"}
+  rel_name=${rel_name%.gpg}
+  pass_date=$(
+    if test -e "$dir"/.git; then
+      git -C "$dir" log -1 --format=%aI "$gpg_path"
+    fi
+  )
+  pass_name=$name/$rel_name
+  pass_out=$PWD/$rel_name
+  mkdir -p "$(dirname "$pass_out")"
+  PASSWORD_STORE_DIR=$dir pass show "$pass_name" > "$pass_out"
+  if [ -n "$pass_date" ]; then
+    touch -d "$pass_date" "$pass_out"
+  fi
+done
+if test -n "$info"; then
+  printf '%s\n' "$info" > "$PWD"/.pass_info
+fi
+"#;
+
+    /// passage HOST: decrypts the secrets in $PASSAGE_DIR into the working
+    /// directory, with the times of their files.  HOST is the first label
+    /// of the target's host, for the command a failure prints.
+    pub const PASSAGE: &str = r#"
+host=$1
+umask 0077
+chmod 0700 "$PWD"
+find "$PASSAGE_DIR" -type f -name \*.age -follow |
+while read -r age_path; do
+  pass_name=${age_path#"$PASSAGE_DIR"/}
+  pass_name=${pass_name%.age}
+  pass_out=$PWD/$pass_name
+  mkdir -p "$(dirname "$pass_out")"
+  if ! passage show "$pass_name" > "$pass_out" 2>&1; then
+    echo ++ secrets "$host" show "$pass_name" >&2
+    jq -Rr 'select(test("^[^[]"))' "$pass_out" >&2
+    exit 1
+  fi
+  touch -r "$age_path" "$pass_out"
+done
+"#;
+}
+
+/// pass_info PATH: prints the .pass_info the pass source at PATH got last.
+pub const PASS_INFO: &str = r#"
+cat "$1"/.pass_info || :
+"#;