diff options
| -rw-r--r-- | README.md | 15 | ||||
| -rw-r--r-- | lib/types/populate.nix | 4 | ||||
| -rw-r--r-- | pkgs/populate/default.nix | 188 | ||||
| -rw-r--r-- | pkgs/populate/flake-jq.nix | 19 | ||||
| -rwxr-xr-x | tests/fixtures.sh | 102 | ||||
| -rw-r--r-- | tests/scenarios.nix | 118 |
6 files changed, 387 insertions, 59 deletions
@@ -476,6 +476,15 @@ Supported attributes: e.g. `stockholm/nixpkgs`, and resolved as the flake's lock file resolves them, so an input that follows another one names the other one. +* `requireInputs` (optional) - + boolean that controls whether populating fails if the inputs that the + input's own lock file pins can't be delivered, e.g. because the lock file + pins something unreachable, or lacks an input its `flake.nix` has. With + `false`, populate prints a warning and delivers the input with the inputs + Nix locks when it adds what the lock file lacks, or else without its + inputs. Evaluating the input as a flake then needs network access. + Defaults to `true`. + * `fetchOnTarget` (optional) - boolean that controls whether the target fetches the input itself instead of receiving it from the deploying machine. The target then uses the @@ -669,8 +678,10 @@ there are any), all other sources as `path:`, with symlinks resolved. Lock files get rewritten or generated on every run, so they match the sources. Copies of `flake` and `flake-input` sources get made again only if the delivered store path, the patches, or the sources they draw inputs from -change. Generated flakes and copies get removed once their sources no longer -need them. This requires Nix with flake support (Nix 2.4 or later) on the +change. Inputs that get drawn from other sources don't get delivered with +a `flake` or `flake-input` source, and neither does what they pin. +Generated flakes and copies get removed once their sources no longer need +them. This requires Nix with flake support (Nix 2.4 or later) on the target. Populating fails if a source that draws inputs from other sources contains no `flake.nix`. Sources that inputs get drawn from don't need to be flakes, as long as the inputs drawing from them are declared with diff --git a/lib/types/populate.nix b/lib/types/populate.nix index a278ffd..1a18b62 100644 --- a/lib/types/populate.nix +++ b/lib/types/populate.nix @@ -233,6 +233,10 @@ name = lib.mkOption { type = lib.types.str; }; + requireInputs = lib.mkOption { + default = true; + type = lib.types.bool; + }; }; }; git = lib.types.submodule { diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index 002326a..2811e27 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -550,7 +550,8 @@ let pop.flake = target: source: pop.flake-input target (source // { name = null; }); pop.flake-input = target: source: let - inherit (import ./flake-jq.nix) selected selfIsRemote selfLocked; + inherit (import ./flake-jq.nix) + metadataSubdir selected selfIsRemote selfLocked; nix' = "${nix}/bin/nix --extra-experimental-features ${quote "nix-command flakes"}"; inputPath = if source.name == null then [] else splitString "/" source.name; fetchOnTarget = source.fetchOnTarget or false; @@ -608,31 +609,81 @@ let ${quote source.url} ${quote (toString source.name)} exit 1 ''; + # The inputs of the flake in this source that get drawn from other + # sources, as paths of input names. + overridden = map (x: x.path) (flakeFollows { type = "flake"; flake = source; }); + requireInputs = source.requireInputs or true; # Shell function printing the store path $1, followed by the inputs its - # flake in the subdirectory $2 pins in its own lock file, using the nix - # command nix. Without a lock file, locking would change the flake, so - # there are no such inputs. - closure = nix: /* sh */ '' + # flake in the subdirectory $2 pins in its own lock file, using the + # commands nix and mktemp. Without a lock file, locking would change the + # flake, so there are no such inputs. Inputs that get drawn from other + # sources are left out, with all they pin: Nix archives an empty flake + # in their place, so it doesn't fetch them. + closure = { nix, mktemp }: /* sh */ '' + # Prints the store paths in nix flake archive's output $1. + closure_paths() { + ARCHIVE=$1 OVERRIDDEN=${quote (toJSON overridden)} \ + ${nix} eval --impure --raw --expr ' + let + overridden = builtins.fromJSON (builtins.getEnv "OVERRIDDEN"); + # Inputs given by relative paths are part of their parent. + paths = prefix: node: builtins.concatMap + (name: let + input = node.inputs.''${name}; + path = prefix ++ [ name ]; + in + if builtins.elem path overridden then [] + else + (if input ? path then [ input.path ] else []) ++ + paths path input) + (builtins.attrNames (node.inputs or {})); + in + builtins.concatStringsSep "\n" + (paths [] (builtins.fromJSON (builtins.getEnv "ARCHIVE"))) + ' + echo + } closure() { printf '%s\n' "$1" closure_dir=$1''${2:+/$2} if test -e "$closure_dir"/flake.nix && test -e "$closure_dir"/flake.lock; then - ARCHIVE=$( - ${nix} flake archive --json --no-update-lock-file \ - "path:$1''${2:+?dir=$2}" - ) \ - ${nix} eval --impure --raw --expr ' - let - # Inputs given by relative paths are part of their parent. - paths = node: builtins.concatMap - (input: - (if input ? path then [ input.path ] else []) ++ paths input) - (builtins.attrValues (node.inputs or {})); - in - builtins.concatStringsSep "\n" - (paths (builtins.fromJSON (builtins.getEnv "ARCHIVE"))) - ' - echo + closure_flake=path:$1''${2:+?dir=$2} + set -- + ${optionalString (overridden != []) /* sh */ '' + closure_empty=$(${mktemp} -d) + echo '{ outputs = _: {}; }' > "$closure_empty"/flake.nix + ${concatMapStrings (path: /* sh */ '' + set -- "$@" --override-input ${quote (concatStringsSep "/" path)} "path:$closure_empty" + '') overridden} + ''} + if closure_archive=$( + ${nix} flake archive --json --no-update-lock-file "$@" "$closure_flake" + ); then + closure_paths "$closure_archive" + else + ${if requireInputs then /* sh */ '' + >&2 printf 'error: source %s: the inputs its own lock file pins can'"'"'t be delivered%s\n' \ + ${quote name} ${quote (optionalString (source.name != null) + ". Set its requireInputs to false to deliver it anyway.")} + exit 1 + '' else /* sh */ '' + # Without --no-update-lock-file, Nix adds what the lock file + # lacks. + if closure_archive=$( + ${nix} flake archive --json --no-write-lock-file "$@" "$closure_flake" + ); then + >&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \ + ${quote name} + closure_paths "$closure_archive" + else + >&2 printf 'warning: source %s: the inputs its own lock file pins can'"'"'t be delivered; delivering it without them\n' \ + ${quote name} + fi + ''} + fi + ${optionalString (overridden != []) /* sh */ '' + rm -R "$closure_empty" + ''} fi } ''; @@ -709,6 +760,7 @@ let ${requireFlakes "fetching flake inputs on the target"} read -r selected read -r self + read -r expected if test -n "$self" && ! nix-store --check-validity "$self" 2>/dev/null; then name=$(basename "$self" | cut -c34-) mv ${quote upload}/tree ${quote upload}/"$name" @@ -720,16 +772,27 @@ let exit 1 fi fi + # Fetching contacts where the input comes from even if the store has + # what the lock file pins. + if test -n "$expected" && ! nix-store --check-validity "$expected" 2>/dev/null; then + expected= + fi # The store path and the subdirectory, on separate lines. Inputs # referring to local paths can only be fetched if the target has them # as well. fetched=$( - FLAKE=${quote source.url} SELECTED=$selected SELF=$self \ + FLAKE=${quote source.url} INPUT=${quote (toString source.name)} \ + SELECTED=$selected SELF=$self EXPECTED=$expected \ nix eval --impure --raw --expr ' let selected = builtins.fromJSON (builtins.getEnv "SELECTED"); + expected = builtins.getEnv "EXPECTED"; + selectedPath = + if selected.self || expected != "" then null + else localPath selected.locked; path = if selected.self then builtins.getEnv "SELF" + else if expected != "" then expected else (builtins.fetchTree selected.locked).outPath; dir = path + (if selected.subdir == "" then "" else "/" + selected.subdir); lock = @@ -750,7 +813,9 @@ let (builtins.attrNames lock.nodes); key = builtins.head missing; in - if missing != [] then + if selectedPath != null && builtins.substring 0 1 selectedPath == "/" && !builtins.pathExists selectedPath then + throw "flake ''${builtins.getEnv "FLAKE"}: input ''${builtins.getEnv "INPUT"} refers to ''${selectedPath}, which does not exist on the target" + else if missing != [] then throw "flake ''${builtins.getEnv "FLAKE"}: input ''${key} refers to ''${localPath lock.nodes.''${key}.locked}, which does not exist on the target" else path + "\n" + selected.subdir @@ -758,8 +823,10 @@ let ) path=$(printf '%s\n' "$fetched" | sed -n 1p) subdir=$(printf '%s\n' "$fetched" | sed -n 2p) - ${closure "nix"} - { printf '%s\n' "$subdir"; closure "$path" "$subdir"; } | { + ${closure { nix = "nix"; mktemp = "mktemp"; }} + # Not in the pipeline, where failing wouldn't fail populate. + paths=$(closure "$path" "$subdir") + printf '%s\n%s\n' "$subdir" "$paths" | { ${link} } ''; @@ -768,33 +835,20 @@ let set -efu ${requireFlakes "fetching flakes on the target"} metadata=$(nix flake metadata --json --no-update-lock-file ${quote source.url}) - archive=$(nix flake archive --json --no-update-lock-file ${quote source.url}) - lines=$( - METADATA=$metadata ARCHIVE=$archive \ - nix eval --impure --raw --expr ' + # The store path and the subdirectory, on separate lines. + fetched=$( + METADATA=$metadata nix eval --impure --raw --expr ' let - # Inputs given by relative paths are part of their parent. - paths = node: - (if node ? path then [ node.path ] else []) ++ - builtins.concatMap paths (builtins.attrValues (node.inputs or {})); metadata = builtins.fromJSON (builtins.getEnv "METADATA"); - archive = builtins.fromJSON (builtins.getEnv "ARCHIVE"); - # The subdirectory, as attribute dir or as query parameter dir - # of the URL, depending on the version of Nix. - subdir = ref: - ref.dir or (let - match = builtins.match ".*[?&]dir=([^&]*).*" (ref.url or ""); - in - if match == null then "" else builtins.head match); - subdirs = builtins.filter (dir: dir != "") - (map subdir [ (metadata.locked or {}) (metadata.original or {}) ]); in - builtins.concatStringsSep "\n" - ([ (if subdirs == [] then "" else builtins.head subdirs) ] ++ - paths archive) + metadata.path + "\n" + (${metadataSubdir}) metadata ' ) - printf '%s\n' "$lines" | { + path=$(printf '%s\n' "$fetched" | sed -n 1p) + subdir=$(printf '%s\n' "$fetched" | sed -n 2p) + ${closure { nix = "nix"; mktemp = "mktemp"; }} + paths=$(closure "$path" "$subdir") + printf '%s\n%s\n' "$subdir" "$paths" | { ${link} } ''; @@ -855,22 +909,46 @@ let ''} fi + # The store path the lock file pins what's selected to, which is the + # one of a directory named source with the locked hash. A store that + # has it needs no fetching, which contacts where the input comes from + # even then. + expected= + hash=$(printf '%s\n' "$selected" | ${jq}/bin/jq -r '.locked.narHash // empty') + case $hash in + sha256-*) + expected=$( + ${nix}/bin/nix-store --print-fixed-path --recursive sha256 "$( + printf %s "''${hash#sha256-}" | + ${coreutils}/bin/base64 -d | + ${coreutils}/bin/od -An -v -tx1 | + ${coreutils}/bin/tr -d ' \n' + )" source + ) + ;; + esac + ${if fetchOnTarget then /* sh */ '' - printf '%s\n%s\n' "$selected" "$self" | { + printf '%s\n%s\n%s\n' "$selected" "$self" "$expected" | { ${runShell target fetchAndLink} } '' else /* sh */ '' path=$self if test -z "$path"; then - path=$( - SELECTED=$selected ${nix'} eval --impure --raw --expr ' - (builtins.fetchTree - (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath - ' - ) + if test -n "$expected" && + ${nix}/bin/nix-store --check-validity "$expected" 2>/dev/null; then + path=$expected + else + path=$( + SELECTED=$selected ${nix'} eval --impure --raw --expr ' + (builtins.fetchTree + (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath + ' + ) + fi fi subdir=$(printf '%s\n' "$selected" | ${jq}/bin/jq -r .subdir) - ${closure nix'} + ${closure { nix = nix'; mktemp = "${coreutils}/bin/mktemp"; }} paths=$(closure "$path" "$subdir") ${optionalString (!isLocalTarget target) /* sh */ '' diff --git a/pkgs/populate/flake-jq.nix b/pkgs/populate/flake-jq.nix index 41edcc5..63afa9d 100644 --- a/pkgs/populate/flake-jq.nix +++ b/pkgs/populate/flake-jq.nix @@ -1,6 +1,21 @@ -# jq programs populate uses for flake and flake-input sources, in a file of -# their own, so tests can run them. +# Programs populate uses for flake and flake-input sources, in a file of +# their own, so tests can run them: jq programs, which run on the deploying +# machine, and a Nix function for the target, which may lack jq. rec { + # Nix function returning the subdirectory containing the flake that nix + # flake metadata describes, like subdirOf does in jq. + metadataSubdir = /* nix */ '' + metadata: let + subdir = ref: + ref.dir or (let + match = builtins.match ".*[?&]dir=([^&]*).*" (ref.url or ""); + in + if match == null then "" else builtins.head match); + subdirs = builtins.filter (dir: dir != "") + (map subdir [ (metadata.locked or {}) (metadata.original or {}) ]); + in + if subdirs == [] then "" else builtins.head subdirs + ''; # jq definitions for subdirectories containing flakes, which flake # references and lock entries name either as attribute dir or as query # parameter dir of their URL, depending on the version of Nix. fetchTree diff --git a/tests/fixtures.sh b/tests/fixtures.sh index 3839c1d..d9d8b2b 100755 --- a/tests/fixtures.sh +++ b/tests/fixtures.sh @@ -359,3 +359,105 @@ cat > /fixtures/patches/bad.patch <<'PATCH' -nothing like this +anything PATCH + +# broken: a flake with an input that can't be fetched anymore. +mkdir /fixtures/www/src/vanishing +cat > /fixtures/www/src/vanishing/flake.nix <<'NIX' +{ outputs = _: { who = "vanishing"; }; } +NIX +tarball vanishing +mkdir /fixtures/broken +cat > /fixtures/broken/flake.nix <<NIX +{ + inputs.vanishing.url = "$url/vanishing.tar.gz"; + outputs = { vanishing, ... }: { msg = "broken:\${vanishing.who}"; }; +} +NIX +(cd /fixtures/broken && nix flake lock) + +# pinned, usepinned: a tarball flake whose own lock file pins an input that +# can't be fetched anymore, and a flake that draws that input from its own +# input instead, so nothing it evaluates needs the pin. +mkdir /fixtures/www/src/pinned +cat > /fixtures/www/src/pinned/flake.nix <<NIX +{ + inputs.inner.url = "$url/vanishing.tar.gz"; + outputs = { inner, ... }: { greeting = "pinned+\${inner.who}"; }; +} +NIX +(cd /fixtures/www/src/pinned && nix flake lock) +tarball pinned +mkdir /fixtures/usepinned +cat > /fixtures/usepinned/flake.nix <<NIX +{ + inputs.inner.url = "$url/inner.tar.gz"; + inputs.dep.url = "$url/pinned.tar.gz"; + inputs.dep.inputs.inner.follows = "inner"; + outputs = { dep, ... }: { msg = "usepinned:\${dep.greeting}"; }; +} +NIX +(cd /fixtures/usepinned && nix flake lock) +rm /fixtures/www/vanishing.tar.gz + +# stale, usestale: a tarball flake whose lock file lacks an input its +# flake.nix has, and a flake with it as input, whose own lock file is +# complete. +mkdir /fixtures/www/src/stale +cat > /fixtures/www/src/stale/flake.nix <<NIX +{ + inputs.inner.url = "$url/inner.tar.gz"; + outputs = { inner, ... }: { greeting = "stale+\${inner.who}"; }; +} +NIX +(cd /fixtures/www/src/stale && nix flake lock) +cat > /fixtures/www/src/stale/flake.nix <<NIX +{ + inputs.inner.url = "$url/inner.tar.gz"; + inputs.extra.url = "$url/inner2.tar.gz"; + outputs = { inner, extra, ... }: { greeting = "stale+\${inner.who}+\${extra.who}"; }; +} +NIX +tarball stale +mkdir /fixtures/usestale +cat > /fixtures/usestale/flake.nix <<NIX +{ + inputs.dep.url = "$url/stale.tar.gz"; + outputs = { dep, ... }: { msg = "usestale:\${dep.greeting}"; }; +} +NIX +(cd /fixtures/usestale && nix flake lock) + +# offline: a flake with an input of its own, which the tests make +# unreachable between deploys. +mkdir /fixtures/www/src/gap +cat > /fixtures/www/src/gap/flake.nix <<'NIX' +{ outputs = _: { who = "gap"; }; } +NIX +tarball gap +mkdir /fixtures/offline +cat > /fixtures/offline/flake.nix <<NIX +{ + inputs.gap.url = "$url/gap.tar.gz"; + outputs = { gap, ... }: { msg = "offline:\${gap.who}"; }; +} +NIX +(cd /fixtures/offline && nix flake lock) + +# c3: a flake with an input, to draw another flake's input from. +mkdir /fixtures/c3 +cat > /fixtures/c3/flake.nix <<NIX +{ + inputs.z.url = "$url/inner.tar.gz"; + outputs = { z, ... }: { who = "c3(\${z.who})"; }; +} +NIX +(cd /fixtures/c3 && nix flake lock) + +# filtered-parent, filtered-parent-link: like filtered, but the filter file +# applies to a subdirectory, which is what gets synced. +mkdir -p /fixtures/filtered-parent/sub +printf -- '- garbage\n' > /fixtures/filtered-parent/.rsync-filter +echo keep > /fixtures/filtered-parent/sub/keep +echo garbage > /fixtures/filtered-parent/sub/garbage +ln -s filtered-parent /fixtures/filtered-parent-link +chown -h nobody /fixtures/filtered-parent-link diff --git a/tests/scenarios.nix b/tests/scenarios.nix index af072b9..7743482 100644 --- a/tests/scenarios.nix +++ b/tests/scenarios.nix @@ -1104,6 +1104,124 @@ assert !(krops.isLocalTarget (targets.remote "/")); check = checkFlake { attr = "msg"; expected = "rel:lib"; }; }; }) + # The target can't fetch an input of the flake. The deploying machine + # fetched it when locking, so its fetcher cache has to go. + (s "fetch-failure" { + prepareDeployer = '' + rm -fR /root/.cache/nix + ''; + phases = [{ + sources.x.flake = { url = "/fixtures/broken"; fetchOnTarget = true; }; + expect.fail = "vanishing.tar.gz"; + }]; + }) + # The delivered input itself is locked to a path the target lacks. + (s "gone-input" { + phases = [{ + sources.x.flake-input = { url = "/fixtures/usegone"; name = "gone"; fetchOnTarget = true; }; + expect.fail = "input gone refers to /fixtures/gone, which does not exist on the target"; + }]; + }) + # An input that gets drawn from a source isn't needed on the target, with + # all it pins. + (modes "overridden-roots" ({ draw, mode, ... }: { + phases = twice { + sources = alt // { + x.flake-input = { + url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; + } // draw { inner.follows = "alt"; }; + }; + check = checkFlake { + attr = "greeting"; expected = "dep+alt"; + generated = mode == "generated"; + } + roots 0; + }; + })) + # The input's own lock file pins something unreachable, which gets drawn + # from a source anyway. + (modes "own-lock-unreachable" ({ draw, mode, ... }: { + prepareDeployer = '' + rm -fR /root/.cache/nix + ''; + phases = twice { + sources = alt // { + x.flake-input = { + url = "/fixtures/usepinned"; name = "dep"; inherit fetchOnTarget; + } // draw { inner.follows = "alt"; }; + }; + check = checkFlake { + attr = "greeting"; expected = "pinned+alt"; + generated = mode == "generated"; + }; + }; + })) + # The input's own lock file pins something unreachable, or lacks an input + # its flake.nix has. That fails, unless the input isn't required to come + # with its inputs. + (s "own-lock-broken" (let + x = url: requireInputs: { + x.flake-input = { inherit url; name = "dep"; inherit fetchOnTarget requireInputs; }; + }; + delivered = '' + test -e "$T"/x/flake.nix || fail "input not delivered" + ''; + in { + prepareDeployer = '' + rm -fR /root/.cache/nix + ''; + phases = [ + { + sources = x "/fixtures/usepinned" true; + expect.fail = "error: source x: the inputs its own lock file pins can't be delivered. Set its requireInputs to false to deliver it anyway."; + } + { + sources = x "/fixtures/usepinned" false; + check = delivered + roots 0; + } + { + sources = x "/fixtures/usestale" true; + expect.fail = "error: source x: the inputs its own lock file pins can't be delivered. Set its requireInputs to false to deliver it anyway."; + } + { + sources = x "/fixtures/usestale" false; + check = delivered + roots 2; + } + { + sources = x "/fixtures/usestale" false; + check = delivered + roots 2; + } + ]; + })) + # Redeploying an unchanged input while where it comes from is + # unreachable and the fetcher caches are gone: the store has it. + (s "offline-redeploy" (let + sources.x.flake-input = { url = "/fixtures/offline"; name = "gap"; inherit fetchOnTarget; }; + check = checkFlake { attr = "who"; expected = "gap"; }; + in { + prepareDeployer = '' + if test -e /fixtures/www/gap.tar.gz.off; then + mv /fixtures/www/gap.tar.gz.off /fixtures/www/gap.tar.gz + fi + ''; + phases = [ + { + inherit sources check; + mutate = '' + rm -fR /root/.cache/nix + ''; + mutateDeployer = '' + rm -fR /root/.cache/nix + mv /fixtures/www/gap.tar.gz /fixtures/www/gap.tar.gz.off + ''; + } + { + inherit sources check; + mutateDeployer = '' + mv /fixtures/www/gap.tar.gz.off /fixtures/www/gap.tar.gz + ''; + } + ]; + })) (s "flake-to-file" { phases = [ { |