populate: deliver flake and flake-input sources natively in Rust

src/flake_input.rs reads what nix flake metadata and nix flake archive
print as JSON itself: it selects the flake or input, decides what the
target fetches or gets copied, and computes the store paths to root,
which are what selected.jq, selfIsRemote.jq, selfLocked.jq, and the Nix
expressions of the shell implementation do there.  On the target, fixed
snippets in sh::flake run Nix, getting values as positional parameters,
and send JSON or store paths back.  The jq test's cases are unit tests of
the Rust equivalents.

1 files changed, 23 insertions, 610 deletions

diff --git a/pkgs/populate/src/flake.rs b/pkgs/populate/src/flake.rs
index 5538288..f03e443 100644
--- a/pkgs/populate/src/flake.rs
+++ b/pkgs/populate/src/flake.rs
@@ -1,4 +1,4 @@
-//! Flake and flake-input sources, and drawing inputs from sources.
+//! Drawing inputs of flakes in sources from other sources.
 //!
 //! This builds the same shell code as shell.nix generates, with every value
 //! quoted, so both implementations behave alike while the code moves to
@@ -10,12 +10,28 @@ use crate::spec::{Follows, Source};
 use serde_json::{Map, Value};
 use std::collections::{BTreeMap, BTreeSet};
 
-pub const LOCK_HAS_INPUT: &str = include_str!("flake/lock-has-input.nix");
-pub const METADATA_SUBDIR: &str = include_str!("flake/metadata-subdir.nix");
-const SUBDIRS: &str = include_str!("flake/subdirs.jq");
-const SELECTED: &str = include_str!("flake/selected.jq");
-const SELF_IS_REMOTE: &str = include_str!("flake/self-is-remote.jq");
-const SELF_LOCKED: &str = include_str!("flake/self-locked.jq");
+/// Nix function returning 1 if the lock file locks has an input at path, a
+/// list of input names, resolving inputs that follow others, and else 0.
+/// Like lockHasInput in flake-jq.nix, for the shell code drawing inputs.
+pub const LOCK_HAS_INPUT: &str = r#"locks: path: let
+  nodes = locks.nodes;
+  input = key: name: (nodes.${key}.inputs or {}).${name} or null;
+  # The node an input refers to, following follows from the root.
+  node = ref:
+    if ref == null then null
+    else if builtins.isList ref then
+      builtins.foldl'
+        (key: name: if key == null then null else node (input key name))
+        locks.root
+        ref
+    else ref;
+  has = key: path:
+    if path == [] then true
+    else let next = node (input key (builtins.head path)); in
+      next != null && has next (builtins.tail path);
+in
+  if has locks.root path then "1" else "0"
+"#;
 
 fn bool_setting(settings: &Map<String, Value>, key: &str, default: bool) -> bool {
     settings
@@ -24,10 +40,6 @@ fn bool_setting(settings: &Map<String, Value>, key: &str, default: bool) -> bool
         .unwrap_or(default)
 }
 
-fn str_setting<'a>(settings: &'a Map<String, Value>, key: &str) -> Option<&'a str> {
-    settings.get(key).and_then(Value::as_str)
-}
-
 fn patches(settings: &Map<String, Value>) -> Vec<&str> {
     settings
         .get("patches")
@@ -57,70 +69,6 @@ pub fn derives_flake(source: &Source) -> bool {
             || (bool_setting(settings, "rewriteLock", true) && !source.follows().is_empty()))
 }
 
-/// Shell code running body on the target: as it is for the local target,
-/// and over ssh, via sudo if the target says so, for remote ones.  The
-/// definitions in sh::prelude come first.
-pub fn run_shell(context: &Context, body: &str) -> String {
-    let command = format!("{}\n{body}", sh::prelude());
-    if context.local {
-        format!("(\n{command}\n)")
-    } else if context.target.sudo {
-        format!(
-            "{} {} {}",
-            q(&context.ssh.to_string_lossy()),
-            q(&context.target.host),
-            q(&format!("sudo sh -c {}", q(&command)))
-        )
-    } else {
-        format!(
-            "{} {} {}",
-            q(&context.ssh.to_string_lossy()),
-            q(&context.target.host),
-            q(&command)
-        )
-    }
-}
-
-/// Shell code transferring source, a shell word, to path on the target, as
-/// rsync::transfer does without .rsync-filter files.
-fn rsync_text(context: &Context, path: &str, source: &str, checksum: bool) -> String {
-    format!(
-        r#"{prepare}
-source_path={source}
-if test -d "$source_path"; then
-  source_path=$source_path/
-fi
-rsync {checksum} {rsync_path} -e {ssh} --protect-args \
-    --itemize-changes \
-    --out-format={format} \
-    --info=backup,del,copy,misc,mount,name,nonreg,remove,skip,symsafe \
-    --omit-dir-times \
-    --omit-link-times \
-    -rlptD \
-    --delete-excluded \
-    "$source_path" {destination} >&2
-"#,
-        prepare = run_shell(
-            context,
-            &format!(
-                "set -- {} {}\n{}",
-                q(path),
-                q(&context.name(path)),
-                sh::break_hard_links()
-            )
-        ),
-        checksum = if checksum { "--checksum" } else { "" },
-        rsync_path = if context.target.sudo {
-            "--rsync-path='sudo rsync'"
-        } else {
-            ""
-        },
-        ssh = q(&context.ssh.to_string_lossy()),
-        format = q("%i %n%L %''b/%''l"),
-        destination = q(&context.rsync_destination(path)),
-    )
-}
-
 fn use_state(sentinel: &str, names: &[&str]) -> String {
     let mut call = format!("use_state {}", q(sentinel));
     for name in names {
@@ -130,541 +78,6 @@ fn use_state(sentinel: &str, names: &[&str]) -> String {
     call
 }
 
-/// Delivers a flake or flake-input source named name to path.  Returns the
-/// shell code doing it on the deploying machine, in the source's working
-/// directory.
-pub fn flake_input(context: &Context, name: &str, path: &str, source: &Source) -> String {
-    let settings = &source.settings;
-    let url = str_setting(settings, "url").unwrap_or_default();
-    let input = if source.kind == "flake" {
-        None
-    } else {
-        str_setting(settings, "name")
-    };
-    let input_path: Vec<&str> = input.map(|i| i.split('/').collect()).unwrap_or_default();
-    let fetch_on_target = bool_setting(settings, "fetchOnTarget", false);
-    let dir = path.rsplit_once('/').map(|(d, _)| d).unwrap_or("");
-    let sentinel = format!("{dir}/.populate");
-    let derived = derives_flake(&Source {
-        kind: "flake".into(),
-        settings: settings.clone(),
-    });
-    let link_path = if derived {
-        format!("{sentinel}/derived/{name}/original")
-    } else {
-        path.to_string()
-    };
-    let patches = patches(settings);
-
-    // Uploads the patches to .populate/patches/<name>, numbered in order.
-    let upload_patches = if patches.is_empty() {
-        String::new()
-    } else {
-        let mut text = String::from("rm -fR \"$PWD\"/patches\nmkdir \"$PWD\"/patches\n");
-        for (i, patch) in patches.iter().enumerate() {
-            let base = patch.rsplit('/').next().unwrap_or(patch);
-            text += &format!(
-                "cp {} \"$PWD\"/patches/{}\n",
-                q(patch),
-                q(&format!("{:04}-{base}", i + 1))
-            );
-        }
-        text += &run_shell(
-            context,
-            &format!(
-                "set -efu\n{}\nmkdir -p {}\n",
-                use_state(&sentinel, &[name]),
-                q(&format!("{sentinel}/patches"))
-            ),
-        );
-        text.push('\n');
-        text += &rsync_text(
-            context,
-            &format!("{sentinel}/patches/{name}"),
-            "\"$PWD\"/patches",
-            true,
-        );
-        text
-    };
-
-    let is_remote_ref = {
-        let scheme: String = url
-            .chars()
-            .take_while(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '+')
-            .collect();
-        !scheme.is_empty()
-            && url[scheme.len()..].starts_with(':')
-            && ![
-                "file",
-                "flake",
-                "git+file",
-                "hg+file",
-                "path",
-                "tarball+file",
-            ]
-            .contains(&scheme.as_str())
-    };
-
-    let no_input = format!(
-        ">&2 printf 'error: flake %s has no input %s\\n' {} {}\nexit 1\n",
-        q(url),
-        q(input.unwrap_or(""))
-    );
-
-    let overridden: Vec<Vec<String>> = Source {
-        kind: "flake".into(),
-        settings: settings.clone(),
-    }
-    .follows()
-    .into_iter()
-    .map(|f| f.path)
-    .collect();
-    let require_inputs = bool_setting(settings, "requireInputs", true);
-
-    // Shell function printing the store path $1, followed by the inputs its
-    // flake in the subdirectory $2 pins in its own lock file.  Without a
-    // lock file, locking would change the flake, so there are no such
-    // inputs.  Inputs that get drawn from other sources are left out, with
-    // all they pin: Nix archives an empty flake in their place, so it
-    // doesn't fetch them.
-    let closure = |nix: &str, mktemp: &str| -> String {
-        let overridden_json = serde_json::to_string(&overridden).expect("serializes");
-        let mut overrides = String::new();
-        if !overridden.is_empty() {
-            overrides += &format!(
-                "closure_empty=$({mktemp} -d)\necho '{{ outputs = _: {{}}; }}' > \"$closure_empty\"/flake.nix\n"
-            );
-            for path in &overridden {
-                overrides += &format!(
-                    "set -- \"$@\" --override-input {} \"path:$closure_empty\"\n",
-                    q(&path.join("/"))
-                );
-            }
-        }
-        let on_failure = if require_inputs {
-            format!(
-                ">&2 printf 'error: source %s: the inputs its own lock file pins can'\"'\"'t be delivered%s\\n' \\\n    {} {}\nexit 1\n",
-                q(name),
-                q(if input.is_some() {
-                    ".  Set its requireInputs to false to deliver it anyway."
-                } else {
-                    ""
-                })
-            )
-        } else {
-            format!(
-                r#"# Without --no-update-lock-file, Nix adds what the lock file
-# lacks.
-if closure_archive=$(
-  {nix} flake archive --json --no-write-lock-file "$@" "$closure_flake"
-); then
-  >&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \
-      {name}
-  closure_paths "$closure_archive"
-else
-  >&2 printf 'warning: source %s: the inputs its own lock file pins can'"'"'t be delivered; delivering it without them\n' \
-      {name}
-fi
-"#,
-                name = q(name)
-            )
-        };
-        format!(
-            r#"
-# Prints the store paths in nix flake archive's output $1.
-closure_paths() {{
-  ARCHIVE=$1 OVERRIDDEN={overridden_json} \
-  {nix} eval --impure --raw --expr '
-    let
-      overridden = builtins.fromJSON (builtins.getEnv "OVERRIDDEN");
-      # Inputs given by relative paths are part of their parent.
-      paths = prefix: node: builtins.concatMap
-        (name: let
-          input = node.inputs.${{name}};
-          path = prefix ++ [ name ];
-        in
-          if builtins.elem path overridden then []
-          else
-            (if input ? path then [ input.path ] else []) ++
-            paths path input)
-        (builtins.attrNames (node.inputs or {{}}));
-    in
-      builtins.concatStringsSep "\n"
-        (paths [] (builtins.fromJSON (builtins.getEnv "ARCHIVE")))
-  '
-  echo
-}}
-closure() {{
-  printf '%s\n' "$1"
-  closure_dir=$1${{2:+/$2}}
-  if test -e "$closure_dir"/flake.nix && test -e "$closure_dir"/flake.lock; then
-    closure_flake=path:$1${{2:+?dir=$2}}
-    set --
-    {overrides}
-    if closure_archive=$(
-      {nix} flake archive --json --no-update-lock-file "$@" "$closure_flake"
-    ); then
-      closure_paths "$closure_archive"
-    else
-      {on_failure}
-    fi
-    {cleanup}
-  fi
-}}
-"#,
-            overridden_json = q(&overridden_json),
-            cleanup = if overridden.is_empty() {
-                ""
-            } else {
-                "rm -R \"$closure_empty\""
-            },
-        )
-    };
-
-    let inputs_roots = format!("{sentinel}/flake-inputs/{name}");
-    let inputs_roots_parent = format!("{sentinel}/flake-inputs");
-    let uploads = format!("{sentinel}/flake-uploads");
-    let upload = format!("{uploads}/{name}");
-    let remove_upload = format!(
-        r#"remove_tree {upload}
-# Sources get populated one after another, so no other source uses the
-# directory meanwhile.
-if test -d {uploads} && test -z "$(ls -A {uploads})"; then
-  rmdir {uploads}
-fi
-"#,
-        upload = q(&upload),
-        uploads = q(&uploads)
-    );
-
-    // Reads the subdirectory containing the flake or input, which may be
-    // empty, followed by store paths from stdin.  Links the first store
-    // path, or the subdirectory therein, at the link path, and registers it
-    // and the remaining ones as garbage collector roots.  Roots get added
-    // before obsolete ones get removed, so the inputs stay rooted.
-    let link = format!(
-        r#"set -efu
-read -r dir
-read -r path
-{use_state}
-{mkdir_derived}
-make_room_for_link {link_path} {link_name}
-nix-store --realise "$path" --add-root {link_path} --indirect >/dev/null
-# The garbage collector follows the link into the store path.
-if test -n "$dir"; then
-  ln -fns "$path/$dir" {link_path}
-fi
-
-# Left behind by an interrupted run, or not needed anymore.
-{remove_upload}
-mkdir -p {roots}
-roots=
-while read -r path; do
-  if test -z "$path"; then
-    continue
-  fi
-  root=$(basename "$path")
-  if test "$(readlink {roots}/"$root")" != "$path"; then
-    nix-store --realise "$path" \
-        --add-root {roots}/"$root" --indirect \
-      >/dev/null
-  fi
-  roots="$roots $root"
-done
-ls -A {roots} | while read -r root; do
-  case " $roots " in
-    *" $root "*) ;;
-    *) rm {roots}/"$root" ;;
-  esac
-done
-# An input without own inputs leaves the directory empty.
-if test -z "$roots"; then
-  rmdir {roots}
-  if test -z "$(ls -A {roots_parent})"; then
-    rmdir {roots_parent}
-  fi
-fi
-"#,
-        use_state = use_state(&sentinel, &[name]),
-        mkdir_derived = if derived {
-            format!(
-                "mkdir -p {}",
-                q(link_path.rsplit_once('/').map(|(d, _)| d).unwrap_or(""))
-            )
-        } else {
-            String::new()
-        },
-        link_path = q(&link_path),
-        link_name = q(&context.name(&link_path)),
-        roots = q(&inputs_roots),
-        roots_parent = q(&inputs_roots_parent),
-    );
-
-    // Reads the selection, as selected produces it, and the store path of
-    // the flake itself from stdin, fetches what's selected, unless it's the
-    // flake itself, which gets added from the upload directory unless the
-    // store has it already, then fetches the inputs its own lock file
-    // pins, and links the results.
-    let fetch_and_link = format!(
-        r#"set -efu
-require_flakes 'fetching flake inputs on the target'
-read -r selected
-read -r self
-read -r expected
-if test -n "$self" && ! nix-store --check-validity "$self" 2>/dev/null; then
-  name=$(basename "$self" | cut -c34-)
-  mv {upload}/tree {upload}/"$name"
-  added=$(nix-store --add-fixed --recursive sha256 {upload}/"$name")
-  {remove_upload}
-  if test "$added" != "$self"; then
-    >&2 printf 'error: flake %s arrived as %s instead of %s\n' \
-        {url} "$added" "$self"
-    exit 1
-  fi
-fi
-# Fetching contacts where the input comes from even if the store has what
-# the lock file pins.
-if test -n "$expected" && ! nix-store --check-validity "$expected" 2>/dev/null; then
-  expected=
-fi
-# The store path and the subdirectory, on separate lines.  Inputs referring
-# to local paths can only be fetched if the target has them as well.
-fetched=$(
-  FLAKE={url} INPUT={input} \
-  SELECTED=$selected SELF=$self EXPECTED=$expected \
-  nix eval --impure --raw --expr '
-    let
-      selected = builtins.fromJSON (builtins.getEnv "SELECTED");
-      expected = builtins.getEnv "EXPECTED";
-      selectedPath =
-        if selected.self || expected != "" then null
-        else localPath selected.locked;
-      path =
-        if selected.self then builtins.getEnv "SELF"
-        else if expected != "" then expected
-        else (builtins.fetchTree selected.locked).outPath;
-      dir = path + (if selected.subdir == "" then "" else "/" + selected.subdir);
-      lock =
-        if builtins.pathExists (dir + "/flake.lock")
-          then builtins.fromJSON (builtins.readFile (dir + "/flake.lock"))
-          else {{ nodes = {{}}; }};
-      localPath = locked:
-        if (locked.type or null) == "path" then locked.path
-        else let
-          match = builtins.match "file:(//)?(.*)" (locked.url or "");
-        in
-          if match == null then null else builtins.elemAt match 1;
-      missing = builtins.filter
-        (key: let
-          p = localPath (lock.nodes.${{key}}.locked or {{}});
-        in
-          p != null && builtins.substring 0 1 p == "/" && !builtins.pathExists p)
-        (builtins.attrNames lock.nodes);
-      key = builtins.head missing;
-    in
-      if selectedPath != null && builtins.substring 0 1 selectedPath == "/" && !builtins.pathExists selectedPath then
-        throw "flake ${{builtins.getEnv "FLAKE"}}: input ${{builtins.getEnv "INPUT"}} refers to ${{selectedPath}}, which does not exist on the target"
-      else if missing != [] then
-        throw "flake ${{builtins.getEnv "FLAKE"}}: input ${{key}} refers to ${{localPath lock.nodes.${{key}}.locked}}, which does not exist on the target"
-      else
-        path + "\n" + selected.subdir
-  '
-)
-path=$(printf '%s\n' "$fetched" | sed -n 1p)
-subdir=$(printf '%s\n' "$fetched" | sed -n 2p)
-{closure}
-# Not in the pipeline, where failing wouldn't fail populate.
-paths=$(closure "$path" "$subdir")
-printf '%s\n%s\n' "$subdir" "$paths" | {{
-  {link}
-}}
-"#,
-        upload = q(&upload),
-        url = q(url),
-        input = q(input.unwrap_or("")),
-        closure = closure("nix", "mktemp"),
-    );
-
-    // Fetches the flake and its inputs, and links the results.
-    let archive_and_link = format!(
-        r#"set -efu
-require_flakes 'fetching flakes on the target'
-metadata=$(nix flake metadata --json --no-update-lock-file {url})
-# The store path and the subdirectory, on separate lines.
-fetched=$(
-  METADATA=$metadata nix eval --impure --raw --expr {expr}
-)
-path=$(printf '%s\n' "$fetched" | sed -n 1p)
-subdir=$(printf '%s\n' "$fetched" | sed -n 2p)
-{closure}
-paths=$(closure "$path" "$subdir")
-printf '%s\n%s\n' "$subdir" "$paths" | {{
-  {link}
-}}
-"#,
-        url = q(url),
-        expr = q(&format!(
-            "let\n  metadata = builtins.fromJSON (builtins.getEnv \"METADATA\");\nin\n  metadata.path + \"\\n\" + ({METADATA_SUBDIR}) metadata\n"
-        )),
-        closure = closure("nix", "mktemp"),
-    );
-
-    let nix_ = "nix --extra-experimental-features 'nix-command flakes'";
-    let selected_jq = format!("{SUBDIRS}{SELECTED}");
-    let self_locked_jq = format!("{SUBDIRS}{SELF_LOCKED}");
-
-    if fetch_on_target && input.is_none() && is_remote_ref {
-        return format!(
-            "{upload_patches}\n{}\n",
-            run_shell(context, &archive_and_link)
-        );
-    }
-
-    let self_text = if fetch_on_target {
-        let upload_self = if context.local {
-            String::new()
-        } else {
-            format!(
-                r#"if ! printf '%s\n' "$self" | {{
-  {check}
-}}; then
-  {prepare}
-  # Without applying .rsync-filter files, which would change the flake.
-  {rsync}
-fi
-"#,
-                check = run_shell(
-                    context,
-                    "read -r self\nnix-store --check-validity \"$self\" 2>/dev/null\n"
-                ),
-                prepare = run_shell(
-                    context,
-                    &format!(
-                        "set -efu\n{}\n{remove_upload}\nmkdir -p {}\n",
-                        use_state(&sentinel, &[name]),
-                        q(&upload)
-                    )
-                ),
-                rsync = rsync_text(context, &format!("{upload}/tree"), "\"$self\"", false),
-            )
-        };
-        format!(
-            r#"# The flake itself has no lock entry, so the target fetches it by its
-# locked reference, if it can, or else receives it.
-if jq -e {self_is_remote} "$cache" >/dev/null; then
-  selected=$(
-    jq -c --argjson selected "$selected" {self_locked} \
-      "$cache"
-  )
-else
-  self=$(jq -r .path "$cache")
-  {upload_self}
-fi
-"#,
-            self_is_remote = q(SELF_IS_REMOTE),
-            self_locked = q(&self_locked_jq),
-        )
-    } else {
-        "self=$(jq -r .path \"$cache\")\n".to_string()
-    };
-
-    let deliver = if fetch_on_target {
-        format!(
-            "printf '%s\\n%s\\n%s\\n' \"$selected\" \"$self\" \"$expected\" | {{\n  {}\n}}\n",
-            run_shell(context, &fetch_and_link)
-        )
-    } else {
-        let copy = if context.local {
-            String::new()
-        } else {
-            format!(
-                "mkdir -p \"$PWD\"/bin\nln -fns {} \"$PWD\"/bin/ssh\nPATH=$PWD/bin:$PATH \\\n{nix_} copy --to {} $paths >&2\n",
-                q(&context.ssh_nix.to_string_lossy()),
-                q(&format!("ssh://{}", context.target.host))
-            )
-        };
-        format!(
-            r#"path=$self
-if test -z "$path"; then
-  if test -n "$expected" &&
-      nix-store --check-validity "$expected" 2>/dev/null; then
-    path=$expected
-  else
-    path=$(
-      SELECTED=$selected {nix_} eval --impure --raw --expr '
-        (builtins.fetchTree
-          (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath
-      '
-    )
-  fi
-fi
-subdir=$(printf '%s\n' "$selected" | jq -r .subdir)
-{closure}
-paths=$(closure "$path" "$subdir")
-
-{copy}
-printf '%s\n%s\n' "$subdir" "$paths" | {{
-  {link}
-}}
-"#,
-            closure = closure(nix_, "mktemp"),
-            link = run_shell(context, &link),
-        )
-    };
-
-    format!(
-        r#"set -efu
-{upload_patches}
-
-cache=$TMPDIR/flake-metadata.$(
-  printf %s {url} |
-  sha256sum |
-  cut -c-32
-).json
-if ! test -e "$cache"; then
-  {{
-    {nix_} flake metadata --json --no-update-lock-file {url}
-  }} > "$cache"
-fi
-
-if ! selected=$(
-  jq -ce --argjson input {input_path} {selected} \
-    "$cache" 2>/dev/null
-); then
-  {no_input}
-fi
-
-self=
-if printf '%s\n' "$selected" | jq -e .self >/dev/null; then
-  {self_text}
-fi
-
-# The store path the lock file pins what's selected to, which is the one of
-# a directory named source with the locked hash.  A store that has it needs
-# no fetching, which contacts where the input comes from even then.
-expected=
-hash=$(printf '%s\n' "$selected" | jq -r '.locked.narHash // empty')
-case $hash in
-  sha256-*)
-    expected=$(
-      nix-store --print-fixed-path --recursive sha256 "$(
-        printf %s "${{hash#sha256-}}" |
-        base64 -d |
-        od -An -v -tx1 |
-        tr -d ' \n'
-      )" source
-    )
-    ;;
-esac
-
-{deliver}
-"#,
-        url = q(url),
-        input_path = q(&serde_json::to_string(&input_path).expect("serializes")),
-        selected = q(&selected_jq),
-    )
-}
-
 /// Draws inputs of flakes in sources from other sources, and derives what
 /// the target paths of flake and flake-input sources that get patched, get
 /// their lock files rewritten, or are materialized get.  Sources get