populate flake-input: fail on failures, reuse the store, leave out replaced inputs

Delivering flakes and flake inputs gets more robust:

- With fetchOnTarget, the inputs get collected before the pipeline that
  links them, so populate fails if the target can't fetch one, instead of
  linking the flake without its inputs and succeeding.
- What the lock file pins gets taken from the store if it's there: the
  store path is the one of a directory named source with the locked hash,
  which nix-store --print-fixed-path computes.  builtins.fetchTree
  contacts where the input comes from even then, so redeploying unchanged
  sources fails without that if a server is unreachable.
- With fetchOnTarget, a delivered input that is itself locked to a local
  path the target lacks fails with populate's error naming the input, like
  its inputs do.
- Inputs that get drawn from other sources are left out, with all they
  pin: Nix archives an empty flake in their place.  So nothing gets
  fetched, copied, and rooted that no evaluation uses, and a pin that is
  unreachable doesn't matter if it gets replaced anyway.
- flake-input sources get the option requireInputs.  By default,
  populating fails if the inputs that the input's own lock file pins
  can't be delivered, e.g. because the lock file lacks an input its
  flake.nix has.  With false, populate warns and delivers the input with
  what Nix locks now, or else without its inputs.

A remote flake that the target fetches entirely on its own collects its
inputs like everything else does.  The Nix function finding its
subdirectory moves next to the jq programs.

New edge cases cover each of these.

2 files changed, 150 insertions, 57 deletions

diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix
index 002326a..2811e27 100644
--- a/pkgs/populate/default.nix
+++ b/pkgs/populate/default.nix
@@ -550,7 +550,8 @@ let
   pop.flake = target: source: pop.flake-input target (source // { name = null; });
 
   pop.flake-input = target: source: let
-    inherit (import ./flake-jq.nix) selected selfIsRemote selfLocked;
+    inherit (import ./flake-jq.nix)
+      metadataSubdir selected selfIsRemote selfLocked;
     nix' = "${nix}/bin/nix --extra-experimental-features ${quote "nix-command flakes"}";
     inputPath = if source.name == null then [] else splitString "/" source.name;
     fetchOnTarget = source.fetchOnTarget or false;
@@ -608,31 +609,81 @@ let
           ${quote source.url} ${quote (toString source.name)}
       exit 1
     '';
+    # The inputs of the flake in this source that get drawn from other
+    # sources, as paths of input names.
+    overridden = map (x: x.path) (flakeFollows { type = "flake"; flake = source; });
+    requireInputs = source.requireInputs or true;
     # Shell function printing the store path $1, followed by the inputs its
-    # flake in the subdirectory $2 pins in its own lock file, using the nix
-    # command nix.  Without a lock file, locking would change the flake, so
-    # there are no such inputs.
-    closure = nix: /* sh */ ''
+    # flake in the subdirectory $2 pins in its own lock file, using the
+    # commands nix and mktemp.  Without a lock file, locking would change the
+    # flake, so there are no such inputs.  Inputs that get drawn from other
+    # sources are left out, with all they pin: Nix archives an empty flake
+    # in their place, so it doesn't fetch them.
+    closure = { nix, mktemp }: /* sh */ ''
+      # Prints the store paths in nix flake archive's output $1.
+      closure_paths() {
+        ARCHIVE=$1 OVERRIDDEN=${quote (toJSON overridden)} \
+        ${nix} eval --impure --raw --expr '
+          let
+            overridden = builtins.fromJSON (builtins.getEnv "OVERRIDDEN");
+            # Inputs given by relative paths are part of their parent.
+            paths = prefix: node: builtins.concatMap
+              (name: let
+                input = node.inputs.''${name};
+                path = prefix ++ [ name ];
+              in
+                if builtins.elem path overridden then []
+                else
+                  (if input ? path then [ input.path ] else []) ++
+                  paths path input)
+              (builtins.attrNames (node.inputs or {}));
+          in
+            builtins.concatStringsSep "\n"
+              (paths [] (builtins.fromJSON (builtins.getEnv "ARCHIVE")))
+        '
+        echo
+      }
       closure() {
         printf '%s\n' "$1"
         closure_dir=$1''${2:+/$2}
         if test -e "$closure_dir"/flake.nix && test -e "$closure_dir"/flake.lock; then
-          ARCHIVE=$(
-            ${nix} flake archive --json --no-update-lock-file \
-                "path:$1''${2:+?dir=$2}"
-          ) \
-          ${nix} eval --impure --raw --expr '
-            let
-              # Inputs given by relative paths are part of their parent.
-              paths = node: builtins.concatMap
-                (input:
-                  (if input ? path then [ input.path ] else []) ++ paths input)
-                (builtins.attrValues (node.inputs or {}));
-            in
-              builtins.concatStringsSep "\n"
-                (paths (builtins.fromJSON (builtins.getEnv "ARCHIVE")))
-          '
-          echo
+          closure_flake=path:$1''${2:+?dir=$2}
+          set --
+          ${optionalString (overridden != []) /* sh */ ''
+            closure_empty=$(${mktemp} -d)
+            echo '{ outputs = _: {}; }' > "$closure_empty"/flake.nix
+            ${concatMapStrings (path: /* sh */ ''
+              set -- "$@" --override-input ${quote (concatStringsSep "/" path)} "path:$closure_empty"
+            '') overridden}
+          ''}
+          if closure_archive=$(
+            ${nix} flake archive --json --no-update-lock-file "$@" "$closure_flake"
+          ); then
+            closure_paths "$closure_archive"
+          else
+            ${if requireInputs then /* sh */ ''
+              >&2 printf 'error: source %s: the inputs its own lock file pins can'"'"'t be delivered%s\n' \
+                  ${quote name} ${quote (optionalString (source.name != null)
+                    ".  Set its requireInputs to false to deliver it anyway.")}
+              exit 1
+            '' else /* sh */ ''
+              # Without --no-update-lock-file, Nix adds what the lock file
+              # lacks.
+              if closure_archive=$(
+                ${nix} flake archive --json --no-write-lock-file "$@" "$closure_flake"
+              ); then
+                >&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \
+                    ${quote name}
+                closure_paths "$closure_archive"
+              else
+                >&2 printf 'warning: source %s: the inputs its own lock file pins can'"'"'t be delivered; delivering it without them\n' \
+                    ${quote name}
+              fi
+            ''}
+          fi
+          ${optionalString (overridden != []) /* sh */ ''
+            rm -R "$closure_empty"
+          ''}
         fi
       }
     '';
@@ -709,6 +760,7 @@ let
       ${requireFlakes "fetching flake inputs on the target"}
       read -r selected
       read -r self
+      read -r expected
       if test -n "$self" && ! nix-store --check-validity "$self" 2>/dev/null; then
         name=$(basename "$self" | cut -c34-)
         mv ${quote upload}/tree ${quote upload}/"$name"
@@ -720,16 +772,27 @@ let
           exit 1
         fi
       fi
+      # Fetching contacts where the input comes from even if the store has
+      # what the lock file pins.
+      if test -n "$expected" && ! nix-store --check-validity "$expected" 2>/dev/null; then
+        expected=
+      fi
       # The store path and the subdirectory, on separate lines.  Inputs
       # referring to local paths can only be fetched if the target has them
       # as well.
       fetched=$(
-        FLAKE=${quote source.url} SELECTED=$selected SELF=$self \
+        FLAKE=${quote source.url} INPUT=${quote (toString source.name)} \
+        SELECTED=$selected SELF=$self EXPECTED=$expected \
         nix eval --impure --raw --expr '
           let
             selected = builtins.fromJSON (builtins.getEnv "SELECTED");
+            expected = builtins.getEnv "EXPECTED";
+            selectedPath =
+              if selected.self || expected != "" then null
+              else localPath selected.locked;
             path =
               if selected.self then builtins.getEnv "SELF"
+              else if expected != "" then expected
               else (builtins.fetchTree selected.locked).outPath;
             dir = path + (if selected.subdir == "" then "" else "/" + selected.subdir);
             lock =
@@ -750,7 +813,9 @@ let
               (builtins.attrNames lock.nodes);
             key = builtins.head missing;
           in
-            if missing != [] then
+            if selectedPath != null && builtins.substring 0 1 selectedPath == "/" && !builtins.pathExists selectedPath then
+              throw "flake ''${builtins.getEnv "FLAKE"}: input ''${builtins.getEnv "INPUT"} refers to ''${selectedPath}, which does not exist on the target"
+            else if missing != [] then
               throw "flake ''${builtins.getEnv "FLAKE"}: input ''${key} refers to ''${localPath lock.nodes.''${key}.locked}, which does not exist on the target"
             else
               path + "\n" + selected.subdir
@@ -758,8 +823,10 @@ let
       )
       path=$(printf '%s\n' "$fetched" | sed -n 1p)
       subdir=$(printf '%s\n' "$fetched" | sed -n 2p)
-      ${closure "nix"}
-      { printf '%s\n' "$subdir"; closure "$path" "$subdir"; } | {
+      ${closure { nix = "nix"; mktemp = "mktemp"; }}
+      # Not in the pipeline, where failing wouldn't fail populate.
+      paths=$(closure "$path" "$subdir")
+      printf '%s\n%s\n' "$subdir" "$paths" | {
         ${link}
       }
     '';
@@ -768,33 +835,20 @@ let
       set -efu
       ${requireFlakes "fetching flakes on the target"}
       metadata=$(nix flake metadata --json --no-update-lock-file ${quote source.url})
-      archive=$(nix flake archive --json --no-update-lock-file ${quote source.url})
-      lines=$(
-        METADATA=$metadata ARCHIVE=$archive \
-        nix eval --impure --raw --expr '
+      # The store path and the subdirectory, on separate lines.
+      fetched=$(
+        METADATA=$metadata nix eval --impure --raw --expr '
           let
-            # Inputs given by relative paths are part of their parent.
-            paths = node:
-              (if node ? path then [ node.path ] else []) ++
-              builtins.concatMap paths (builtins.attrValues (node.inputs or {}));
             metadata = builtins.fromJSON (builtins.getEnv "METADATA");
-            archive = builtins.fromJSON (builtins.getEnv "ARCHIVE");
-            # The subdirectory, as attribute dir or as query parameter dir
-            # of the URL, depending on the version of Nix.
-            subdir = ref:
-              ref.dir or (let
-                match = builtins.match ".*[?&]dir=([^&]*).*" (ref.url or "");
-              in
-                if match == null then "" else builtins.head match);
-            subdirs = builtins.filter (dir: dir != "")
-              (map subdir [ (metadata.locked or {}) (metadata.original or {}) ]);
           in
-            builtins.concatStringsSep "\n"
-              ([ (if subdirs == [] then "" else builtins.head subdirs) ] ++
-               paths archive)
+            metadata.path + "\n" + (${metadataSubdir}) metadata
         '
       )
-      printf '%s\n' "$lines" | {
+      path=$(printf '%s\n' "$fetched" | sed -n 1p)
+      subdir=$(printf '%s\n' "$fetched" | sed -n 2p)
+      ${closure { nix = "nix"; mktemp = "mktemp"; }}
+      paths=$(closure "$path" "$subdir")
+      printf '%s\n%s\n' "$subdir" "$paths" | {
         ${link}
       }
     '';
@@ -855,22 +909,46 @@ let
         ''}
       fi
 
+      # The store path the lock file pins what's selected to, which is the
+      # one of a directory named source with the locked hash.  A store that
+      # has it needs no fetching, which contacts where the input comes from
+      # even then.
+      expected=
+      hash=$(printf '%s\n' "$selected" | ${jq}/bin/jq -r '.locked.narHash // empty')
+      case $hash in
+        sha256-*)
+          expected=$(
+            ${nix}/bin/nix-store --print-fixed-path --recursive sha256 "$(
+              printf %s "''${hash#sha256-}" |
+              ${coreutils}/bin/base64 -d |
+              ${coreutils}/bin/od -An -v -tx1 |
+              ${coreutils}/bin/tr -d ' \n'
+            )" source
+          )
+          ;;
+      esac
+
       ${if fetchOnTarget then /* sh */ ''
-        printf '%s\n%s\n' "$selected" "$self" | {
+        printf '%s\n%s\n%s\n' "$selected" "$self" "$expected" | {
           ${runShell target fetchAndLink}
         }
       '' else /* sh */ ''
         path=$self
         if test -z "$path"; then
-          path=$(
-            SELECTED=$selected ${nix'} eval --impure --raw --expr '
-              (builtins.fetchTree
-                (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath
-            '
-          )
+          if test -n "$expected" &&
+              ${nix}/bin/nix-store --check-validity "$expected" 2>/dev/null; then
+            path=$expected
+          else
+            path=$(
+              SELECTED=$selected ${nix'} eval --impure --raw --expr '
+                (builtins.fetchTree
+                  (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath
+              '
+            )
+          fi
         fi
         subdir=$(printf '%s\n' "$selected" | ${jq}/bin/jq -r .subdir)
-        ${closure nix'}
+        ${closure { nix = nix'; mktemp = "${coreutils}/bin/mktemp"; }}
         paths=$(closure "$path" "$subdir")
 
         ${optionalString (!isLocalTarget target) /* sh */ ''
diff --git a/pkgs/populate/flake-jq.nix b/pkgs/populate/flake-jq.nix
index 41edcc5..63afa9d 100644
--- a/pkgs/populate/flake-jq.nix
+++ b/pkgs/populate/flake-jq.nix
@@ -1,6 +1,21 @@
-# jq programs populate uses for flake and flake-input sources, in a file of
-# their own, so tests can run them.
+# Programs populate uses for flake and flake-input sources, in a file of
+# their own, so tests can run them: jq programs, which run on the deploying
+# machine, and a Nix function for the target, which may lack jq.
 rec {
+  # Nix function returning the subdirectory containing the flake that nix
+  # flake metadata describes, like subdirOf does in jq.
+  metadataSubdir = /* nix */ ''
+    metadata: let
+      subdir = ref:
+        ref.dir or (let
+          match = builtins.match ".*[?&]dir=([^&]*).*" (ref.url or "");
+        in
+          if match == null then "" else builtins.head match);
+      subdirs = builtins.filter (dir: dir != "")
+        (map subdir [ (metadata.locked or {}) (metadata.original or {}) ]);
+    in
+      if subdirs == [] then "" else builtins.head subdirs
+  '';
   # jq definitions for subdirectories containing flakes, which flake
   # references and lock entries name either as attribute dir or as query
   # parameter dir of their URL, depending on the version of Nix.  fetchTree