diff options
| -rw-r--r-- | README.md | 9 | ||||
| -rw-r--r-- | pkgs/populate/default.nix | 312 | ||||
| -rwxr-xr-x | tests/fixtures.sh | 69 | ||||
| -rw-r--r-- | tests/scenarios.nix | 34 |
4 files changed, 283 insertions, 141 deletions
@@ -394,8 +394,10 @@ flake's lock file, to the target. This makes the lock file of a flake the single place that pins, e.g., nixpkgs, for flakes as well as for `NIX_PATH`. It works like the `flake` source type, but delivers the selected input instead of the flake, so the target path becomes a symlink to the input's -store path, and the input's own inputs get garbage collector roots in -`.populate/flake-inputs/<name>`. +store path. The inputs that the input's own lock file pins, which evaluating +the input needs, get delivered as well, with garbage collector roots in +`.populate/flake-inputs/<name>`. An input without a lock file comes without +its inputs, because locking it would change it. Because the target path points at the store path named by the lock file, it can be used in `NIX_PATH` as well as by flakes evaluated on the target, @@ -410,7 +412,8 @@ Supported attributes: * `name` - name of the input to deliver. Inputs of inputs are separated by `/`, - e.g. `stockholm/nixpkgs`. + e.g. `stockholm/nixpkgs`, and resolved as the flake's lock file resolves + them, so an input that follows another one names the other one. * `fetchOnTarget` (optional) - boolean that controls whether the target fetches the input itself instead diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index 553183e..708e7a3 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -264,19 +264,6 @@ let ${quote source.flake} ${quote (toString source.name)} exit 1 ''; - # jq filter selecting the flake or input in nix flake archive's output. - node = - if source.name == null then "." - else concatMapStrings (name: ".inputs[${toJSON name}]") inputPath; - # jq definitions for nix flake metadata's output, resolving a path of - # input names to the key of its lock node. - refs = /* jq */ '' - .locks as $l - | def ref($path): - reduce $path[] as $name ($l.root; - $l.nodes[.].inputs[$name] // error("no input \($name)") - | if type == "array" then ref(.) else . end); - ''; # jq definitions for subdirectories containing flakes, which flake # references and lock entries name either as attribute dir or as query # parameter dir of their URL, depending on the version of Nix. fetchTree @@ -285,7 +272,7 @@ let def subdir: .dir // ((.url // "") | capture("[?&]dir=(?<dir>[^&]*)").dir) // ""; def withoutSubdir: - del(.dir) + del(.__final, .dir) | if (.url | type) == "string" then .url |= (sub("(?<sep>[?&])dir=[^&]*&?"; "\(.sep)") | sub("[?&]$"; "")) else . end; @@ -294,42 +281,82 @@ let | map(select(. != null) | subdir | select(. != "")) | first // ""; ''; - # jq filter selecting the locked input and its own inputs in nix flake - # metadata's output, as lock node keys with fetchTree arguments, the input - # first. The flake itself has no lock entry, so its arguments are null. - locked = /* jq */ '' + # jq filter selecting the flake or input in nix flake metadata's output, + # by the path of input names $input, as what to fetch and the + # subdirectory containing it: {self, locked, subdir}. Names follow the + # flake's lock file, follows included; an empty follows path removes an + # input. The flake itself, and inputs given by paths relative to it, have + # self set instead of fetchTree arguments in locked. + selected = /* jq */ '' ${subdirs} - ${refs} - def deps($key): - $key, ($l.nodes[$key].inputs // {} | .[] - | if type == "array" then ref(.) else . end - | deps(.)); - ref($input) as $key - | [$key] + ([deps($key)] | unique - [$key]) - | map({ - key: ., - locked: ($l.nodes[.].locked | if . == null then . else withoutSubdir end) - }) + . as $m + | .locks as $l + | def ref($path; $depth): + if $depth > 64 then error("follows nest too deep") else + reduce $path[] as $name ($l.root; + (($l.nodes[.].inputs // {})[$name]) as $next + | if $next == null or $next == [] then error("no input \($name)") + elif ($next | type) == "array" then ref($next; $depth + 1) + else $next end) + end; + def join($a; $b): + if $a == "" then $b elif $b == "" then $a else "\($a)/\($b)" end; + def resolve($key; $depth): + $l.nodes[$key] as $node + | if $depth > 64 then error("parents nest too deep") + elif $key == $l.root then + { self: true, locked: null, subdir: subdirOf($m) } + elif $node.parent != null then + resolve(ref($node.parent; 0); $depth + 1) as $base + | $base + { + subdir: join(join($base.subdir; + $node.locked.path | ltrimstr("./") | rtrimstr("/") | if . == "." then "" else . end); + subdirOf($node)) + } + else + { self: false, locked: ($node.locked | withoutSubdir), subdir: subdirOf($node) } + end; + resolve(ref($input; 0); 0) ''; - # jq filter selecting the subdirectory containing the flake or input in - # nix flake metadata's output, or the empty string. - dir = /* jq */ '' - ${subdirs} - ${refs} - if $input == [] then subdirOf(.) else subdirOf($l.nodes[ref($input)]) end + # jq filter telling whether the target can fetch the flake itself, i.e. + # whether its locked reference doesn't refer to the deploying machine. + selfIsRemote = /* jq */ '' + .locked | (.type != "path") and ((.url // "") | startswith("file:") | not) ''; - # jq filter replacing the null fetchTree arguments of the flake itself in - # the JSON array $locked with its locked reference. + # jq filter replacing self in the selection $selected with the flake's + # locked reference. selfLocked = /* jq */ '' ${subdirs} .locked as $self - | $locked - | .[0].locked = ($self | del(.__final) | withoutSubdir) + | $selected + { self: false, locked: ($self | withoutSubdir) } ''; - # jq filter telling whether the target can fetch the flake itself, i.e. - # whether its locked reference doesn't refer to the deploying machine. - selfIsRemote = /* jq */ '' - .locked | (.type != "path") and ((.url // "") | startswith("file:") | not) + # Shell function printing the store path $1, followed by the inputs its + # flake in the subdirectory $2 pins in its own lock file, using the nix + # command nix. Without a lock file, locking would change the flake, so + # there are no such inputs. + closure = nix: /* sh */ '' + closure() { + printf '%s\n' "$1" + closure_dir=$1''${2:+/$2} + if test -e "$closure_dir"/flake.nix && test -e "$closure_dir"/flake.lock; then + ARCHIVE=$( + ${nix} flake archive --json --no-update-lock-file \ + "path:$1''${2:+?dir=$2}" + ) \ + ${nix} eval --impure --raw --expr ' + let + # Inputs given by relative paths are part of their parent. + paths = node: builtins.concatMap + (input: + (if input ? path then [ input.path ] else []) ++ paths input) + (builtins.attrValues (node.inputs or {})); + in + builtins.concatStringsSep "\n" + (paths (builtins.fromJSON (builtins.getEnv "ARCHIVE"))) + ' + echo + fi + } ''; sentinel = "${dirOf target.path}/.populate"; # Directory of garbage collector roots for the input's own inputs. @@ -370,6 +397,9 @@ let mkdir -p ${quote inputsRoots} roots= while read -r path; do + if test -z "$path"; then + continue + fi root=$(basename "$path") if test "$(readlink ${quote inputsRoots}/"$root")" != "$path"; then nix-store --realise "$path" \ @@ -389,18 +419,16 @@ let rmdir ${quote inputsRoots} fi ''; - # Reads lock node keys with fetchTree arguments as JSON array, the store - # path of the flake itself, and the subdirectory containing the flake or - # input from stdin, fetches them, and links the results. The flake - # itself, if given, stands in for the first fetchTree arguments, which are - # null then, and gets added from the upload directory unless the store has - # it already. + # Reads the selection, as selected produces it, and the store path of the + # flake itself from stdin, fetches what's selected, unless it's the flake + # itself, which gets added from the upload directory unless the store has + # it already, then fetches the inputs its own lock file pins, and links + # the results. fetchAndLink = /* sh */ '' set -efu ${requireFlakes "fetching flake inputs on the target"} - read -r locked + read -r selected read -r self - read -r dir if test -n "$self" && ! nix-store --check-validity "$self" 2>/dev/null; then name=$(basename "$self" | cut -c34-) mv ${quote upload}/tree ${quote upload}/"$name" @@ -412,31 +440,46 @@ let exit 1 fi fi - # Inputs referring to local paths can only be fetched if the target has - # them as well. - paths=$( - FLAKE=${quote source.flake} LOCKED=$locked SELF=$self \ + # The store path and the subdirectory, on separate lines. Inputs + # referring to local paths can only be fetched if the target has them + # as well. + fetched=$( + FLAKE=${quote source.flake} SELECTED=$selected SELF=$self \ nix eval --impure --raw --expr ' let + selected = builtins.fromJSON (builtins.getEnv "SELECTED"); + path = + if selected.self then builtins.getEnv "SELF" + else (builtins.fetchTree selected.locked).outPath; + dir = path + (if selected.subdir == "" then "" else "/" + selected.subdir); + lock = + if builtins.pathExists (dir + "/flake.lock") + then builtins.fromJSON (builtins.readFile (dir + "/flake.lock")) + else { nodes = {}; }; localPath = locked: - if locked.type == "path" then locked.path + if (locked.type or null) == "path" then locked.path else let match = builtins.match "file:(//)?(.*)" (locked.url or ""); in if match == null then null else builtins.elemAt match 1; - fetch = { key, locked }: - if locked == null then builtins.getEnv "SELF" - else let path = localPath locked; in - if path != null && !builtins.pathExists path then - throw "flake ''${builtins.getEnv "FLAKE"}: input ''${key} refers to ''${path}, which does not exist on the target" - else - (builtins.fetchTree locked).outPath; + missing = builtins.filter + (key: let + p = localPath (lock.nodes.''${key}.locked or {}); + in + p != null && builtins.substring 0 1 p == "/" && !builtins.pathExists p) + (builtins.attrNames lock.nodes); + key = builtins.head missing; in - builtins.concatStringsSep "\n" - (map fetch (builtins.fromJSON (builtins.getEnv "LOCKED"))) + if missing != [] then + throw "flake ''${builtins.getEnv "FLAKE"}: input ''${key} refers to ''${localPath lock.nodes.''${key}.locked}, which does not exist on the target" + else + path + "\n" + selected.subdir ' ) - printf '%s\n%s\n' "$dir" "$paths" | { + path=$(printf '%s\n' "$fetched" | sed -n 1p) + subdir=$(printf '%s\n' "$fetched" | sed -n 2p) + ${closure "nix"} + { printf '%s\n' "$subdir"; closure "$path" "$subdir"; } | { ${link} } ''; @@ -450,9 +493,10 @@ let METADATA=$metadata ARCHIVE=$archive \ nix eval --impure --raw --expr ' let + # Inputs given by relative paths are part of their parent. paths = node: - [ node.path ] ++ - builtins.concatMap paths (builtins.attrValues node.inputs); + (if node ? path then [ node.path ] else []) ++ + builtins.concatMap paths (builtins.attrValues (node.inputs or {})); metadata = builtins.fromJSON (builtins.getEnv "METADATA"); archive = builtins.fromJSON (builtins.getEnv "ARCHIVE"); # The subdirectory, as attribute dir or as query parameter dir @@ -479,94 +523,86 @@ let throw "krops: source ${target.path}: copying flakes to targets with sudo is not supported" else if fetchOnTarget && source.name == null && isRemoteRef source.flake then runShell target archiveAndLink - else if fetchOnTarget then /* sh */ '' + else /* sh */ '' set -efu ${cached "metadata" /* sh */ '' ${nix'} flake metadata --json --no-update-lock-file ${quote source.flake} ''} - if ! locked=$( - ${jq}/bin/jq -ce --argjson input ${quote (toJSON inputPath)} ${quote locked} \ + if ! selected=$( + ${jq}/bin/jq -ce --argjson input ${quote (toJSON inputPath)} ${quote selected} \ "$cache" 2>/dev/null ); then ${noInput} fi - dir=$( - ${jq}/bin/jq -r --argjson input ${quote (toJSON inputPath)} ${quote dir} \ - "$cache" - ) self= - ${optionalString (source.name == null) /* sh */ '' - # The flake itself has no lock entry, so the target fetches it by its - # locked reference, if it can, or else receives it. - if ${jq}/bin/jq -e ${quote selfIsRemote} "$cache" >/dev/null; then - locked=$( - ${jq}/bin/jq -c --argjson locked "$locked" \ - ${quote selfLocked} \ - "$cache" - ) - else + if printf '%s\n' "$selected" | ${jq}/bin/jq -e .self >/dev/null; then + ${if fetchOnTarget then /* sh */ '' + # The flake itself has no lock entry, so the target fetches it by + # its locked reference, if it can, or else receives it. + if ${jq}/bin/jq -e ${quote selfIsRemote} "$cache" >/dev/null; then + selected=$( + ${jq}/bin/jq -c --argjson selected "$selected" ${quote selfLocked} \ + "$cache" + ) + else + self=$(${jq}/bin/jq -r .path "$cache") + ${optionalString (!isLocalTarget target) /* sh */ '' + if ! printf '%s\n' "$self" | { + ${runShell target /* sh */ '' + read -r self + nix-store --check-validity "$self" 2>/dev/null + ''} + }; then + ${runShell target /* sh */ '' + set -efu + ${sentinelDirectory sentinel} + ${removeUpload} + mkdir -p ${quote upload} + ''} + # Without applying .rsync-filter files, which would change the flake. + ${rsync' (target // { path = "${upload}/tree"; }) + (rsyncDefaultConfig // { filterFiles = false; }) + /* sh */ ''"$self"''} + fi + ''} + fi + '' else /* sh */ '' self=$(${jq}/bin/jq -r .path "$cache") - ${optionalString (!isLocalTarget target) /* sh */ '' - if ! printf '%s\n' "$self" | { - ${runShell target /* sh */ '' - read -r self - nix-store --check-validity "$self" 2>/dev/null - ''} - }; then - ${runShell target /* sh */ '' - set -efu - ${sentinelDirectory sentinel} - ${removeUpload} - mkdir -p ${quote upload} - ''} - # Without applying .rsync-filter files, which would change the flake. - ${rsync' (target // { path = "${upload}/tree"; }) - (rsyncDefaultConfig // { filterFiles = false; }) - /* sh */ ''"$self"''} - fi - ''} - fi - ''} - - printf '%s\n%s\n%s\n' "$locked" "$self" "$dir" | { - ${runShell target fetchAndLink} - } - '' else /* sh */ '' - set -efu - - ${cached "archive" /* sh */ '' - ${nix'} flake archive --json --no-update-lock-file ${quote source.flake} - ''} - - if ! ${jq}/bin/jq -e ${quote "${node} | .path"} "$cache" >/dev/null; then - ${noInput} + ''} fi - # The input followed by its own inputs, which flakes evaluated on the - # target need as well. - paths=$(${jq}/bin/jq -r ${quote "${node} | .. | .path? | strings"} "$cache") + ${if fetchOnTarget then /* sh */ '' + printf '%s\n%s\n' "$selected" "$self" | { + ${runShell target fetchAndLink} + } + '' else /* sh */ '' + path=$self + if test -z "$path"; then + path=$( + SELECTED=$selected ${nix'} eval --impure --raw --expr ' + (builtins.fetchTree + (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath + ' + ) + fi + subdir=$(printf '%s\n' "$selected" | ${jq}/bin/jq -r .subdir) + ${closure nix'} + paths=$(closure "$path" "$subdir") - ${cached "metadata" /* sh */ '' - ${nix'} flake metadata --json --no-update-lock-file ${quote source.flake} - ''} - dir=$( - ${jq}/bin/jq -r --argjson input ${quote (toJSON inputPath)} ${quote dir} \ - "$cache" - ) + ${optionalString (!isLocalTarget target) /* sh */ '' + mkdir -p "$PWD"/bin + ln -fns ${sshNix' target} "$PWD"/bin/ssh + PATH=$PWD/bin:$PATH \ + ${nix'} copy --to ${quote "ssh://${target.host}"} $paths >&2 + ''} - ${optionalString (!isLocalTarget target) /* sh */ '' - mkdir -p "$PWD"/bin - ln -fns ${sshNix' target} "$PWD"/bin/ssh - PATH=$PWD/bin:$PATH \ - ${nix'} copy --to ${quote "ssh://${target.host}"} $paths >&2 + printf '%s\n%s\n' "$subdir" "$paths" | { + ${runShell target link} + } ''} - - printf '%s\n%s\n' "$dir" "$paths" | { - ${runShell target link} - } ''; pop.git = target: source: runShell target /* sh */ '' diff --git a/tests/fixtures.sh b/tests/fixtures.sh index 9608fcd..9813c8f 100755 --- a/tests/fixtures.sh +++ b/tests/fixtures.sh @@ -147,6 +147,75 @@ cat > /fixtures/uselocal/flake.nix <<'NIX' NIX (cd /fixtures/uselocal && nix flake lock) +# follow: a flake whose input dep draws its input inner from the flake's own +# input inner, which is inner2, so the lock file records a follows path, and +# dep's own lock file pins another inner than the flake's. +mkdir /fixtures/www/src/inner2 +cat > /fixtures/www/src/inner2/flake.nix <<'NIX' +{ outputs = _: { who = "inner2"; }; } +NIX +tarball inner2 +mkdir /fixtures/follow +cat > /fixtures/follow/flake.nix <<NIX +{ + inputs.inner.url = "$url/inner2.tar.gz"; + inputs.dep.url = "$url/dep.tar.gz"; + inputs.dep.inputs.inner.follows = "inner"; + outputs = { dep, ... }: { msg = "follow:\${dep.greeting}"; }; +} +NIX +(cd /fixtures/follow && nix flake lock) + +# nofollow: a flake that removes an input of its input, so the lock file +# records an empty follows path. +mkdir /fixtures/nofollow +cat > /fixtures/nofollow/flake.nix <<NIX +{ + inputs.dep.url = "$url/dep.tar.gz"; + inputs.dep.inputs.inner.follows = ""; + outputs = _: { msg = "nofollow"; }; +} +NIX +(cd /fixtures/nofollow && nix flake lock) + +# cyc: a flake whose inputs cyca and cycb follow each other. +for name in cyca cycb; do + other=$(test $name = cyca && echo cycb || echo cyca) + mkdir /fixtures/www/src/$name + cat > /fixtures/www/src/$name/flake.nix <<NIX +{ + inputs.other.url = "$url/$other.tar.gz"; + outputs = _: { who = "$name"; }; +} +NIX + tarball $name +done +mkdir /fixtures/cyc +cat > /fixtures/cyc/flake.nix <<NIX +{ + inputs.a.url = "$url/cyca.tar.gz"; + inputs.b.url = "$url/cycb.tar.gz"; + inputs.a.inputs.other.follows = "b"; + inputs.b.inputs.other.follows = "a"; + outputs = { a, b, ... }: { msg = "cyc:\${a.who},\${b.who}"; }; +} +NIX +(cd /fixtures/cyc && nix flake lock) + +# rel: a flake in a Git work tree with an input in a subdirectory, referred +# to by a relative path. +mkdir -p /fixtures/rel/lib +cat > /fixtures/rel/lib/flake.nix <<'NIX' +{ outputs = _: { who = "lib"; }; } +NIX +cat > /fixtures/rel/flake.nix <<'NIX' +{ + inputs.lib.url = "path:./lib"; + outputs = { lib, ... }: { msg = "rel:${lib.who}"; }; +} +NIX +(cd /fixtures/rel && nix flake lock && git init -q && commit rel) + # noflake: a directory without flake.nix. mkdir /fixtures/noflake echo noflake > /fixtures/noflake/README diff --git a/tests/scenarios.nix b/tests/scenarios.nix index ef80d05..a738eca 100644 --- a/tests/scenarios.nix +++ b/tests/scenarios.nix @@ -774,6 +774,40 @@ assert !(krops.isLocalTarget (targets.remote "/")); ''; }; }) + (s "follows-empty" { + phases = twice { + sources.x.flake = { flake = "/fixtures/nofollow"; fetchOnTarget = true; }; + check = checkFlake { attr = "msg"; expected = "nofollow"; }; + }; + }) + (s "follows-cycle" { + phases = twice { + sources.x.flake = { flake = "/fixtures/cyc"; fetchOnTarget = true; }; + check = checkFlake { attr = "msg"; expected = "cyc:cyca,cycb"; }; + }; + }) + # Names resolve through the parent's lock file, so dep/inner is the + # parent's inner, which dep's inner follows. + (s "follows-name" { + phases = twice { + sources.x.flake-input = { flake = "/fixtures/follow"; name = "dep/inner"; inherit fetchOnTarget; }; + check = checkFlake { attr = "who"; expected = "inner2"; }; + }; + }) + # The delivered input comes with the inputs its own lock file pins, which + # evaluating it needs, not with the parent's inner, which it follows there. + (s "follows-closure" { + phases = twice { + sources.x.flake-input = { flake = "/fixtures/follow"; name = "dep"; inherit fetchOnTarget; }; + check = checkFlake { attr = "greeting"; expected = "dep+inner"; } + roots 1; + }; + }) + (s "relative-input" { + phases = twice { + sources.x.flake = { flake = "/fixtures/rel"; fetchOnTarget = true; }; + check = checkFlake { attr = "msg"; expected = "rel:lib"; }; + }; + }) ] ++ lib.optional (targetKind == "local") # The target has the path the input is locked to. (s "uselocal" { |