diff options
| author | tv <tv@krebsco.de> | 2026-11-12 19:44:41 +0000 |
|---|---|---|
| committer | tv <tv@krebsco.de> | 2026-11-13 03:24:12 +0100 |
| commit | 470de08082a8d251c3854ceb6da9cef8bd43b74b (patch) | |
| tree | c576ac7e5f8c55bf0eb9899c28726b81ab3cdad3 | |
| parent | 333144080f4a03f1adc1eb7ded0a9d3ae1bfa9ee (diff) | |
populate: stay inside the target directory
Populate replaces what's at the target paths of sources, removes what
sources don't deliver anymore, and prunes. Nothing of that may reach
beyond the target directory, whatever names sources have and whatever the
target directory contains:
- Source names that aren't file names in the target directory don't
evaluate: the empty name, ., .., .populate, and names with slashes or
control characters. E.g. a git source named "" removes the target
directory, and a file source named .. syncs into its parent.
- Target paths that aren't absolute, that are the root directory, or that
contain .. don't evaluate.
- A symlink at the target path of a source gets replaced, not followed,
and populate says so. rsync-based and flake sources do that already.
git sources replace the link as well, and also a work tree whose .git is
a symlink or a file naming a repository elsewhere, instead of fetching,
resetting, and cleaning there. derivation sources replace a link that
doesn't point into the store, instead of failing.
- Populate refuses to use the sentinel, or a state directory therein, if
it is a symlink, instead of changing and removing what it points to.
- What populate removes recursively gets removed as a link if it is one:
chmod -R follows a link named on the command line.
- Flake and flake-input sources that get patched, get their lock file
rewritten, or are materialized replace a symlink at their target path
like every source does, and say so.
- Nix rewrites lock files in place, which changes other hard links to
them as well, e.g. in a backup or outside the target directory, and
what a lock file that is a symlink points to. Populate replaces a lock
file by a copy of its own before rewriting it.
The target directory itself may be a symlink. Nix refuses path: URLs with
symlinks on the way, so the flakes step resolves the paths it hands to
Nix.
Backups work on a sentinel that is still a file, which becomes a
directory, like for everything else that keeps state.
The tests create canaries outside the target directories: a file and a
directory beside them, and a directory elsewhere with a read-only
directory and a Git work tree with uncommitted changes. The check of
every phase of every scenario fails if a canary changed, or if a store
path that the target directory links to doesn't match its hash anymore.
A new test, safety, points links at the canaries: at the target path of
every source type, inside what sources replace, in place of populate's
state, and among what it removes and prunes. It also covers the names
and target paths that don't evaluate, a target directory that is a
symlink, and a lock file with another hard link.
| -rw-r--r-- | README.md | 29 | ||||
| -rw-r--r-- | pkgs/populate/default.nix | 208 | ||||
| -rw-r--r-- | tests/default.nix | 4 | ||||
| -rw-r--r-- | tests/lib.sh | 64 | ||||
| -rw-r--r-- | tests/scenarios.nix | 360 | ||||
| -rw-r--r-- | tests/vm.nix | 14 |
6 files changed, 610 insertions, 69 deletions
@@ -290,6 +290,29 @@ pkgs.krops.writeCommand "deploy-with-swap" { whether the ssh session should do a pseudo-terminal allocation. sets `-t` on the ssh command. +## The target directory + +Populate owns the target directory, `/var/src` by default: it replaces +whatever is at the target path of a source, and with `prune` it removes +everything else. It doesn't change or remove anything outside: + +* Target paths must be absolute, must not be the root directory, and must not + contain `..`. Source names must be file names other than `.`, `..`, and + `.populate`, without slashes and control characters. Anything else fails + when evaluating. + +* A symlink at the target path of a source gets replaced by what the source + delivers, not followed, and populate says so. To keep a source elsewhere, + e.g. on another disk or in memory, mount that place at the target path, or + make the target directory itself a symlink. A Git work tree whose `.git` + is a symlink, or a file naming a repository elsewhere, gets replaced by a + `git` source as well. + +* Populate keeps its state in the sentinel, `.populate`, which becomes a + directory once there is state to keep. Populating fails if the sentinel, + or a directory populate needs therein, is a symlink. + + ## Source Types ### `derivation` @@ -658,8 +681,10 @@ flakes, as long as the inputs drawing from them are declared with `tests/` contains NixOS VM tests for populate: one test per source type, one for features spanning sources, such as flake inputs from sources, one for edge -cases of flakes, such as unusual lock files, and one for the arguments -`writeDeploy` passes to `nixos-rebuild`. The tests deploy every combination +cases of flakes, such as unusual lock files, one for staying inside the target +directory, whatever links it contains, and one for the arguments `writeDeploy` +passes to `nixos-rebuild`. After every deploy, all tests check that nothing +outside the target directory changed. The tests deploy every combination of a source type's options, except for patches, which the features test covers, to the deploying machine itself, to a remote target as `root`, and to a remote target using `sudo`, and check the result after diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index a7e351a..002326a 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -4,9 +4,119 @@ with shell; { coreutils, dash, findutils, git, jq, nix, openssh, pass, passage, rsync, writers }: let + # Populate replaces and removes what's at the target paths of sources, so + # it refuses names that lead out of the target directory, to the directory + # itself, or to its own state. + checkSourceName = name: + if elem name [ "" "." ".." ".populate" ] || + builtins.match "[^/[:cntrl:]]+" name == null then + throw "krops: source name ${toJSON name} can't be used: it must be a file name other than ., .., and .populate, without slashes and control characters" + else + true; + + # Populate removes what it finds in the target directory, so it refuses + # paths that aren't absolute, that are the root directory, or that the + # target would resolve to another directory than the one they name. + checkTargetPath = path: let + components = filter (x: x != "" && x != ".") (splitString "/" path); + in + if !isString path || substring 0 1 path != "/" then + throw "krops: target path ${toJSON path} can't be used: it must be absolute" + else if components == [] then + throw "krops: target path ${toJSON path} can't be used: it must not be the root directory" + else if elem ".." components || + builtins.match "[^[:cntrl:]]+" path == null then + throw "krops: target path ${toJSON path} can't be used: it must not contain .. or control characters" + else + true; + + # The directories below the sentinel, once that is a directory, which + # have a directory for each source that needs one. Besides, there is + # backup, which holds copies of the sources. + stateKinds = [ "derived" "flake-inputs" "flake-uploads" "flakes" "patches" ]; + + # Shell code that prepares keeping state below the sentinel for the sources + # named names. The sentinel only needs to exist, so it may be a file, + # which becomes a directory. Populate changes and removes what's in its + # state directories, and a symlink in place of one would make it do that to + # whatever the link points to, so it refuses to use those. + useState = sentinel: names: /* sh */ '' + refuse_link() { + if test -L "$1"; then + >&2 printf 'error: %s is a symlink, but populate needs a directory of its own there\n' "$1" + exit 1 + fi + } + refuse_link ${quote sentinel} + refuse_link ${quote sentinel}/backup + for state_kind in ${toString stateKinds}; do + refuse_link ${quote sentinel}/"$state_kind" + for state_name in ${concatMapStringsSep " " quote names}; do + refuse_link ${quote sentinel}/"$state_kind"/"$state_name" + done + done + if test -f ${quote sentinel}; then + rm ${quote sentinel} + mkdir ${quote sentinel} + fi + ''; + + # Shell code that removes what's at path, a shell word, with everything + # below it, also what isn't writable. A symlink gets removed itself, + # instead of followed. + removeTree = path: /* sh */ '' + if test -L ${path}; then + rm ${path} + elif test -e ${path}; then + chmod -R u+w ${path} + rm -fR ${path} + fi + ''; + + # Shell code that removes a symlink at path, saying so, because the link + # may be there on purpose, but populate never follows it: what a source + # delivers would change or remove what the link points to. + removeLink = path: /* sh */ '' + if test -L ${quote path}; then + >&2 printf 'replacing symlink %s, which points to %s\n' \ + ${quote path} "$(readlink ${quote path})" + rm ${quote path} + fi + ''; + + # Shell code that removes a symlink at path like removeLink does, unless + # it points into the store, like the links populate puts there itself. + removeForeignLink = path: /* sh */ '' + if test -L ${quote path}; then + case $(readlink ${quote path}) in + /nix/store/*) ;; + *) + ${removeLink path} + ;; + esac + fi + ''; + + # Shell code that makes room at path for a link into the store, which Nix + # only puts in place of another link into the store. + makeRoomForLink = path: /* sh */ '' + ${removeForeignLink path} + if ! test -L ${quote path} && test -e ${quote path}; then + rm -fR ${quote path} + fi + ''; + check = { force, target }: let sentinelFile = "${target.path}/.populate"; in runShell target /* sh */ '' + # Before anything touches it, which would touch what it points to. + if test -L ${quote sentinelFile}; then + >&2 printf 'error: sentinel file is a symlink: %s\n' ${quote ( + optionalString (!isLocalTarget target) "${target.host}:" + + sentinelFile + )} + exit 1 + fi ${optionalString force /* sh */ '' mkdir -vp ${quote (dirOf sentinelFile)} >&2 touch ${quote sentinelFile} @@ -24,6 +134,7 @@ let sentinelFile = "${target.path}/.populate"; in runShell target /* sh */ '' + ${useState sentinelFile []} if ! test -d ${quote sentinelFile}; then >&2 printf 'error: sentinel file is not a directory: %s\n' ${quote ( optionalString (!isLocalTarget target) "${target.host}:" + @@ -36,19 +147,10 @@ let --delete \ --exclude /.populate \ --link-dest=${quote target.path} \ - ${target.path}/ \ - ${target.path}/.populate/backup/ + ${quote target.path}/ \ + ${quote target.path}/.populate/backup/ ''; - # Shell code that turns the sentinel file into a directory, which populate - # uses for keeping state. The sentinel only needs to exist. - sentinelDirectory = sentinel: /* sh */ '' - if test -f ${quote sentinel}; then - rm ${quote sentinel} - mkdir ${quote sentinel} - fi - ''; - # Draws inputs of flakes in sources from other sources, and derives what the # target paths of flake and flake-input sources that get patched, get their # lock files rewritten, or are materialized get. Sources get processed in @@ -85,9 +187,11 @@ let # Shell word for the URL of the flake that inputs drawn from a source use: # its generated flake, if it has one, so overrides don't need to be # repeated, or else the flake in the source. + # Nix refuses path: URLs with symlinks on the way, which the target + # directory may be, so paths get resolved. url = name: if generated ? ${name} - then quote "path:${flakesDir}/${name}" + then ''"path:$(readlink -f ${quote "${flakesDir}/${name}"})"'' else ''"$(flake_url ${quote name})"''; check = name: source: if source == name then @@ -108,6 +212,7 @@ let rewrite = name: /* sh */ '' ( ${overrideArgs name} + own_lock "$(readlink -f ${quote "${target.path}/${name}"})" nix flake lock "$@" "$(flake_url ${quote name})" >&2 ) ''; @@ -143,12 +248,7 @@ let ''; # Removes the work directory, which an interrupted run may have left # behind. - removeWork = /* sh */ '' - if test -e ${quote "${dir}/work"}; then - chmod -R u+w ${quote "${dir}/work"} - rm -fR ${quote "${dir}/work"} - fi - ''; + removeWork = removeTree (quote "${dir}/work"); in /* sh */ '' ( original=$(readlink -f ${quote "${dir}/original"}) @@ -192,12 +292,14 @@ let cd ${quote "${dir}/work"} ${applyPatches} ) + ${removeForeignLink targetPath} rm -fR ${quote targetPath} mv ${quote "${dir}/work"} ${quote targetPath} printf '%s\n' "$key" > ${quote "${dir}/key"} fi ${optionalString rewriteLock /* sh */ '' ${overrideArgs name} + own_lock ${quote targetPath} nix flake lock "$@" "$(flake_url ${quote name})" >&2 ''} '' else /* sh */ '' @@ -208,8 +310,8 @@ let : else # The copy keeps the name of the delivered store path. - work=${quote "${dir}/work"}/$(basename "$path" | cut -c34-) mkdir ${quote "${dir}/work"} + work=$(readlink -f ${quote "${dir}/work"})/$(basename "$path" | cut -c34-) cp -R "$path" "$work" chmod -R u+w "$work" ( @@ -228,9 +330,7 @@ let --add-root ${quote "${dir}/result"} --indirect >/dev/null printf '%s\n' "$key" > ${quote "${dir}/key"} fi - if test -d ${quote targetPath} && ! test -L ${quote targetPath}; then - rm -fR ${quote targetPath} - fi + ${makeRoomForLink targetPath} # .populate/derived/<name>/result roots the store path. ln -fns "$result''${subdir:+/$subdir}" ${quote targetPath} ''} @@ -286,7 +386,7 @@ let echo '}' } > flake.nix # Not just ., which Nix takes for the enclosing Git work tree, if any. - nix flake lock ${quote "path:${flakesDir}/${name}"} >&2 + nix flake lock "path:$(pwd -P)" >&2 ) ''; process = name: @@ -301,8 +401,7 @@ let case ${quote " ${toString names} "} in *" $name "*) ;; *) - chmod -R u+w ${quote dir}/"$name" - rm -fR ${quote dir}/"$name" + ${removeTree ''${quote dir}/"$name"''} ;; esac done @@ -316,10 +415,22 @@ let throw "krops: sources draw inputs from each other: ${toString sorted.cycle}" else runShell target /* sh */ '' set -efu + ${useState sentinel (attrNames sources)} ${optionalString (names != []) /* sh */ '' ${optionalString (drawing != {}) (requireFlakes "drawing flake inputs from sources")} + # Nix rewrites lock files in place, which also changes what else is + # a hard link to the lock file, e.g. in a backup, or what it is a + # symlink to. So the lock file of the flake in the directory $1 + # gets replaced by a copy first. + own_lock() { + if test -e "$1"/flake.lock; then + cp -p "$1"/flake.lock "$1"/flake.lock.populate + mv -f "$1"/flake.lock.populate "$1"/flake.lock + fi + } + # Prints the flake URL of the source named $1. flake_url() { path=$(readlink -f ${quote target.path}/"$1") @@ -356,7 +467,6 @@ let fi ${optionalString (generated != {}) /* sh */ '' - ${sentinelDirectory sentinel} mkdir -p ${quote flakesDir} ''} ${concatMapStrings process sorted.result} @@ -388,6 +498,7 @@ let sources); in runShell target /* sh */ '' set -efu + ${useState "${target.path}/.populate" []} cd ${quote target.path} ls -A | while read -r name; do case $name in @@ -410,13 +521,11 @@ let rmdir "$roots" fi fi - if test -e .populate/flake-uploads; then - chmod -R u+w .populate/flake-uploads - rm -fR .populate/flake-uploads - fi + ${removeTree ".populate/flake-uploads"} ''; pop.derivation = target: source: runShell target /* sh */ '' + ${makeRoomForLink target.path} nix-build -E ${quote source.text} -o ${quote target.path} >&2 ''; @@ -463,7 +572,7 @@ let '') patches} ${runShell target /* sh */ '' set -efu - ${sentinelDirectory sentinel} + ${useState sentinel [ name ]} mkdir -p ${quote "${sentinel}/patches"} ''} ${rsync' (target // { path = "${sentinel}/patches/${name}"; }) @@ -533,10 +642,7 @@ let uploads = "${sentinel}/flake-uploads"; upload = "${uploads}/${name}"; removeUpload = /* sh */ '' - if test -e ${quote upload}; then - chmod -R u+w ${quote upload} - rm -fR ${quote upload} - fi + ${removeTree (quote upload)} # Sources get populated one after another, so no other source uses the # directory meanwhile. if test -d ${quote uploads} && test -z "$(ls -A ${quote uploads})"; then @@ -552,26 +658,17 @@ let set -efu read -r dir read -r path + ${useState sentinel [ name ]} ${optionalString derived /* sh */ '' - ${sentinelDirectory sentinel} mkdir -p ${quote (dirOf linkPath)} ''} - # nix-store --add-root only replaces links into the store. - if test -L ${quote linkPath}; then - case $(readlink ${quote linkPath}) in - /nix/store/*) ;; - *) rm ${quote linkPath} ;; - esac - elif test -e ${quote linkPath}; then - rm -fR ${quote linkPath} - fi + ${makeRoomForLink linkPath} nix-store --realise "$path" --add-root ${quote linkPath} --indirect >/dev/null # The garbage collector follows the link into the store path. if test -n "$dir"; then ln -fns "$path/$dir" ${quote linkPath} fi - ${sentinelDirectory sentinel} # Left behind by an interrupted run, or not needed anymore. ${removeUpload} mkdir -p ${quote inputsRoots} @@ -742,7 +839,7 @@ let }; then ${runShell target /* sh */ '' set -efu - ${sentinelDirectory sentinel} + ${useState sentinel [ name ]} ${removeUpload} mkdir -p ${quote upload} ''} @@ -792,8 +889,13 @@ let pop.git = target: source: runShell target /* sh */ '' set -efu # Remove target path if it doesn't look like a git worktree. - # This can happen e.g. when it had a different type earlier. - if ! test -e ${quote target.path}/.git; then + # This can happen e.g. when it had a different type earlier. A work + # tree that a symlink points to, or whose .git is a symlink or a file + # naming a repository elsewhere, isn't populate's own: fetching, + # resetting, and cleaning would change what's elsewhere. + ${removeLink target.path} + if test -L ${quote target.path}/.git || + ! test -d ${quote target.path}/.git; then rm -fR ${quote target.path} fi if ! test -e ${quote target.path}; then @@ -955,11 +1057,7 @@ let rsync' = target: config: sourcePath: /* sh */ '' # rsync writes through a symlink at the target path, e.g. one a symlink or # flake source left there, into what it points to. - ${runShell target /* sh */ '' - if test -L ${quote target.path}; then - rm ${quote target.path} - fi - ''} + ${runShell target (removeLink target.path)} source_path=${sourcePath} if test -d "$source_path"; then source_path=$source_path/ @@ -1088,6 +1186,8 @@ let in { backup ? false, force ? false, prune ? false, source, target }: +assert checkTargetPath target.path; +assert all checkSourceName (attrNames source); writers.writeDash "populate.${target.host}" '' set -efu diff --git a/tests/default.nix b/tests/default.nix index 285ed86..7b82bce 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -1,6 +1,6 @@ # NixOS VM tests for populate: one test per source type, one for features -# spanning sources, one for edge cases of flakes, and one for writeDeploy's -# arguments to nixos-rebuild. Build all with nix-build tests, or one with, +# spanning sources, one for edge cases of flakes, one for staying inside the +# target directory, and one for writeDeploy's arguments to nixos-rebuild. Build all with nix-build tests, or one with, # e.g., nix-build tests -A flake. jq runs populate's jq programs for flakes # without VMs. { pkgs ? import ../pkgs {} }: diff --git a/tests/lib.sh b/tests/lib.sh index 33a0c6e..4f62890 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -48,3 +48,67 @@ gc() { nix-store --gc >/dev/null fi } + +# Canaries are files outside the target directories, which populate must +# never change, whatever the target directories contain. Some scenarios +# point links at them. The manifest lists every entry with its type, mode, +# and link target, and every file with its checksum. +canary_manifest() { + { + find /var/krops/canary /var/krops/sibling /var/krops-outside \ + -printf '%y %m %p -> %l\n' + find /var/krops/canary /var/krops/sibling /var/krops-outside \ + -type f -exec sha256sum {} + + } 2>&1 | sort +} + +# Creates the canaries, unless they are there unchanged: a file beside the +# target directories, a directory beside them, and a directory elsewhere +# with files, a read-only directory, and a Git work tree with a changed and +# an untracked file, which resetting and cleaning would lose. +canary_setup() { + if test -e /var/krops-outside.manifest && + canary_manifest | cmp -s - /var/krops-outside.manifest; then + return + fi + rm -fR /var/krops/canary /var/krops/sibling /var/krops-outside + mkdir -p /var/krops/sibling /var/krops-outside/dir/sub + echo canary > /var/krops/canary + echo sibling > /var/krops/sibling/file + echo outside > /var/krops-outside/file + echo outside > /var/krops-outside/dir/file + echo outside > /var/krops-outside/dir/sub/file + chmod a-w /var/krops-outside/dir/sub/file /var/krops-outside/dir/sub + git init -q /var/krops-outside/repo + ( + cd /var/krops-outside/repo + echo tracked > tracked + git add tracked + GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 \ + git -c user.name=krops -c user.email=spam@krebsco.de commit -q -m canary + echo changed > tracked + echo untracked > untracked + ) + canary_manifest > /var/krops-outside.manifest +} + +# Fails if a canary changed, or if a store path that an entry of the target +# directory $T links to doesn't match its hash anymore. +canary_check() { + if ! canary_manifest | cmp -s - /var/krops-outside.manifest; then + canary_manifest | diff /var/krops-outside.manifest - >&2 || : + fail "something outside the target directory changed" + fi + if test -n "${T-}" && test -d "$T"; then + ls -A "$T" | while IFS= read -r canary_name; do + canary_link=$(readlink "$T/$canary_name" 2>/dev/null) || continue + case $canary_link in + /nix/store/*) + nix-store --verify-path \ + "$(printf '%s\n' "$canary_link" | cut -d/ -f1-4)" || + fail "$T/$canary_name links to a store path that changed" + ;; + esac + done + fi +} diff --git a/tests/scenarios.nix b/tests/scenarios.nix index 2815c3f..af072b9 100644 --- a/tests/scenarios.nix +++ b/tests/scenarios.nix @@ -94,37 +94,49 @@ let # differ only in what makes evaluation fail, or a function returning them # for the phase's sources, and that must evaluate. So the phase doesn't # pass for failing for an unrelated reason, e.g. a misspelt option. + # targetPath deploys to another path than the scenario's target + # directory, and controlTargetPath is the control for a path that makes + # evaluation fail. phase = { - sources, prune ? false, deploy ? null, - expect ? "success", control ? null, + sources, backup ? false, prune ? false, deploy ? null, + targetPath ? path, + expect ? "success", control ? null, controlTargetPath ? null, check ? "", mutate ? null, checkDeployer ? null, mutateDeployer ? null, }: let - command = sources: let + command = sources: targetPath: let args = { source = krops.evalSource [ sources ]; - inherit prune target; + inherit backup prune; + target = target // { path = targetPath; }; force = true; }; in if deploy == null then populate args else writeDeploy "deploy" (args // { useNixOutputMonitor = false; } // deploy); - evaluates = sources: - (builtins.tryEval (builtins.seq (toString (command sources)) true)).success; + evaluates = sources: targetPath: + (builtins.tryEval + (builtins.seq (toString (command sources targetPath)) true)).success; script = text: if text == null then null else prelude + vars + text; in if expect == "eval-error" then - if control == null then + if control == null && controlTargetPath == null then throw "scenario ${name}: expecting evaluation to fail needs a control" - else if evaluates sources then + else if evaluates sources targetPath then throw "scenario ${name}: populate evaluates, but is expected to fail" - else if !evaluates (if lib.isFunction control then control sources else control) then + else if !( + if controlTargetPath != null then evaluates sources controlTargetPath + else evaluates + (if lib.isFunction control then control sources else control) + targetPath + ) then throw "scenario ${name}: the control doesn't evaluate either" else null else { - populate = toString (command sources); + populate = toString (command sources targetPath); expect = if expect == "success" then null else expect.fail; - check = script check; + # No phase gets around checking the canaries. + check = script (check + "\ncanary_check\n"); mutate = script mutate; checkDeployer = script checkDeployer; mutateDeployer = script mutateDeployer; @@ -140,6 +152,10 @@ let '' + prepare; prepareDeployer = if prepareDeployer == null then null else prelude + vars + prepareDeployer; + # For all machines: before the scenario, and after it for the ones its + # checks don't run on. + canarySetup = prelude + "canary_setup\n"; + canaryCheck = prelude + "canary_check\n"; phases = lib.filter (x: x != null) (map phase phases); }; @@ -1255,4 +1271,326 @@ assert !(krops.isLocalTarget (targets.remote "/")); }) ) targetKinds; + + # Populate replaces and removes things in the target directory. These + # scenarios check that it doesn't anywhere else: names and paths that lead + # out of the target directory don't evaluate, and symlinks in it get + # replaced or refused instead of followed. The canaries, which every phase + # of every scenario checks, tell if something outside changed; these + # scenarios point links at them. + safety = concatMap (targetKind: let + s = name: args: scenario ({ + name = "${name}-${targetKind}"; + inherit targetKind; + } // args); + copying = targetKind != "sudo"; + fetchOnTarget = !copying; + outside = "/var/krops-outside"; + hello = [ "/fixtures/patches/hello.patch" ]; + # Sources that use every kind of state populate keeps: x a copy with + # patches, y garbage collector roots and, where the flake gets uploaded, + # an upload, and z a generated flake. + stateful = alt // { + x.flake-input = { + url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; + patches = hello; + inputs.inner.follows = "alt"; + }; + y.flake = { url = "/fixtures/top"; inherit fetchOnTarget; }; + z.file = { + path = "/fixtures/filesrc"; + rewriteLock = false; + inputs.dep.follows = "alt"; + }; + }; + # A source deployed where a symlink to something outside is. The link + # gets replaced, never followed. + link = type: { sources, to, check }: s "link-${type}" { + isolate = true; + prepare = '' + mkdir -p "$T" + ln -s ${outside}/${to} "$T"/x + ''; + phases = twice { + inherit sources; + check = '' + case $(readlink "$T"/x || :) in + ${outside}/*) fail "x still links outside" ;; + esac + '' + check; + }; + }; + # Something of populate's state replaced by a symlink to outside between + # two deploys. The second one refuses to use it. + state = name: { + entry, backup ? false, + message ? "/${entry} is a symlink, but populate needs a directory of its own there", + }: s "state-${name}" { + isolate = true; + phases = [ + { + sources = stateful; + inherit backup; + mutate = '' + rm -fR "$T"/${entry} + mkdir -p "$(dirname "$T"/${entry})" + ln -s ${outside}/dir "$T"/${entry} + ''; + } + { + sources = stateful; + inherit backup; + expect.fail = message; + } + ]; + }; + in lib.flatten [ + (link "file" { + sources.x.file = "/fixtures/alt"; + to = "dir"; + check = '' + test -e "$T"/x/flake.nix || fail "flake.nix missing" + ''; + }) + # The link points at a Git work tree, which resetting and cleaning would + # change. + (link "git" { + sources.x.git = { url = gitUrl; ref = gitHead; }; + to = "repo"; + check = '' + expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitHead} + ''; + }) + (link "pass" { + sources.x.pass = { dir = "/fixtures/pass"; name = "x"; }; + to = "dir"; + check = '' + expect_eq a "$(cat "$T"/x/a)" secret-a + ''; + }) + (link "passage" { + sources.x.passage = { + dir = "/fixtures/passage/store/x"; + identities_file = "/fixtures/passage/identities"; + }; + to = "dir"; + check = '' + expect_eq a "$(cat "$T"/x/a)" secret-a + ''; + }) + (link "pipe" { + sources.x.pipe = "/fixtures/pipe"; + to = "file"; + check = '' + expect_eq x "$(cat "$T"/x)" piped + ''; + }) + (link "derivation" { + sources.x.derivation = '' + derivation { + name = "krops-test"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo built > $out" ]; + } + ''; + to = "file"; + check = '' + expect_eq x "$(cat "$T"/x)" built + ''; + }) + (link "symlink" { + sources = alt // { x.symlink = "alt"; }; + to = "dir"; + check = '' + expect_eq link "$(readlink "$T"/x)" alt + ''; + }) + (link "flake" { + sources.x.flake = { url = "/fixtures/top"; inherit fetchOnTarget; }; + to = "dir"; + check = checkFlake { attr = "msg"; expected = "top:dep+inner,sub"; }; + }) + (link "flake-materialized" { + sources.x.flake = { + url = "/fixtures/top"; inherit fetchOnTarget; + materialize = true; + }; + to = "dir"; + check = checkFlake { + attr = "msg"; expected = "top:dep+inner,sub"; materialized = true; + }; + }) + (link "flake-input-patched" { + sources.x.flake-input = { + url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; + patches = hello; + }; + to = "dir"; + check = checkFlake { attr = "greeting"; expected = "hello+inner"; }; + }) + # Symlinks to outside in what a source replaces: rsync and Git remove + # them, and so does replacing a directory by a link. + (s "inner-links" (let + prepare = '' + for name in a b c; do + mkdir -p "$T"/$name + ln -s ${outside}/dir "$T"/$name/keep + ln -s ${outside}/dir "$T"/$name/stale + ln -s ${outside}/repo/.git "$T"/$name/.git + done + # A work tree whose repository is elsewhere. + mkdir -p "$T"/d + echo "gitdir: ${outside}/repo/.git" > "$T"/d/.git + ''; + in { + isolate = true; + inherit prepare; + phases = twice { + sources = { + a.file = "/fixtures/filesrc"; + b.git = { url = gitUrl; ref = gitHead; }; + c.flake = { url = "/fixtures/top"; inherit fetchOnTarget; }; + d.git = { url = gitUrl; ref = gitHead; }; + }; + check = '' + expect_eq keep "$(cat "$T"/a/keep)" keep + test ! -L "$T"/a/stale || fail "a/stale remains" + expect_eq HEAD "$(git -C "$T"/b rev-parse HEAD)" ${gitHead} + test ! -L "$T"/b/.git || fail "b/.git remains a link" + test ! -L "$T"/b/stale || fail "b/stale remains" + expect_eq HEAD "$(git -C "$T"/d rev-parse HEAD)" ${gitHead} + test -d "$T"/d/.git || fail "d/.git isn't a directory" + expect_eq "store directory" "$(dirname "$(readlink "$T"/c)")" /nix/store + ''; + }; + })) + (state "sentinel" { + entry = ".populate"; + message = "error: sentinel file is a symlink"; + }) + (state "backup" { entry = ".populate/backup"; backup = true; }) + (map + (entry: state (lib.replaceStrings ["/"] ["-"] entry) { + entry = ".populate/${entry}"; + }) + [ + "derived" "flake-inputs" "flake-uploads" "flakes" "patches" + "derived/x" "patches/x" "flake-inputs/y" "flake-uploads/y" "flakes/z" + ]) + # Symlinks to outside among what populate removes: leftovers of + # interrupted runs, state of sources that are gone, and, when pruning, + # entries of the target directory. They get removed as links. + (s "leftovers" { + isolate = true; + prepare = '' + mkdir -p "$T" + ln -s ${outside}/dir "$T"/other + ln -s ${outside}/repo "$T"/repo + ''; + phases = [ + { + sources = stateful; + mutate = '' + ln -s ${outside}/dir "$T"/.populate/derived/x/work + ln -s ${outside}/dir "$T"/.populate/derived/gone + ln -s ${outside}/dir "$T"/.populate/flakes/gone + ln -s ${outside}/dir "$T"/.populate/patches/gone + ln -s ${outside}/dir "$T"/.populate/flake-inputs/gone + mkdir -p "$T"/.populate/flake-uploads + ln -s ${outside}/dir "$T"/.populate/flake-uploads/gone + ''; + } + { + sources = stateful; + check = '' + for name in derived/x/work derived/gone flakes/gone patches/gone; do + test ! -L "$T"/.populate/$name || fail "$name remains" + done + for name in other repo .populate/flake-inputs/gone .populate/flake-uploads/gone; do + test -L "$T"/$name || fail "$name removed without pruning" + done + ''; + } + { + sources = stateful; + prune = true; + check = '' + for name in other repo .populate/flake-inputs/gone .populate/flake-uploads; do + test ! -L "$T"/$name && test ! -e "$T"/$name || fail "$name not pruned" + done + ''; + } + ]; + }) + # The target directory itself may be a symlink: what it points to is the + # target directory then. + (s "linked-target" { + isolate = true; + prepare = '' + rm -fR "$T"-real + mkdir "$T"-real + ln -s "$(basename "$T")"-real "$T" + ''; + phases = twice { + sources = stateful; + check = '' + test -L "$T" || fail "the target directory isn't a link anymore" + test -e "$T"-real/alt/flake.nix || fail "alt missing" + gc + expect_eq greeting "$(nix_eval "$(readlink -f "$T"/x)"#greeting)" hello+alt + expect_eq msg "$(nix_eval "$(readlink -f "$T"/y)"#msg)" top:dep+inner,sub + expect_eq msg "$(nix_eval_online "path:$(readlink -f "$T"/.populate/flakes/z)#msg")" top:alt,sub + ''; + }; + }) + (lib.optionals (targetKind == "local") [ + # Rewriting a lock file changes no other hard link to it. The lock + # file is like the source's, with its modification time, so rsync + # leaves the link in place. (The source is on the deploying machine, + # so this only runs there.) + (s "lock-hard-link" { + prepare = '' + mkdir -p "$T" + cp -a /fixtures/filesrc "$T"/x + cp -p "$T"/x/flake.lock "$T".lock + ln -f "$T".lock "$T"/x/flake.lock + ''; + phases = twice { + sources = alt // { + x.file = { path = "/fixtures/filesrc"; inputs.dep.follows = "alt"; }; + }; + check = '' + cmp -s "$T".lock /fixtures/filesrc/flake.lock || + fail "the other hard link changed" + expect_eq msg "$(nix_eval_online "$T"/x#msg)" top:alt,sub + ''; + }; + }) + # Names that lead out of the target directory, to the directory itself, + # or to populate's state. + (s "names" { + phases = map + (name: { + sources.${name}.file = "/fixtures/alt"; + expect = "eval-error"; + control = { ok.file = "/fixtures/alt"; }; + }) + [ "" "." ".." ".populate" "a/b" "../x" "/etc" "a\nb" ]; + }) + # Target paths that aren't absolute, that are the root directory, or + # that resolve to another directory than they name. + (s "paths" { + phases = map + (targetPath: { + sources = alt; + inherit targetPath; + expect = "eval-error"; + controlTargetPath = "/var/krops/paths-local"; + }) + [ "" "/" "//" "/." "relative" "/var/krops/.." "/var/krops/../.." "/var/krops/a\nb" ]; + }) + ]) + ]) targetKinds; + } diff --git a/tests/vm.nix b/tests/vm.nix index d808dd2..5eb4dbb 100644 --- a/tests/vm.nix +++ b/tests/vm.nix @@ -14,6 +14,8 @@ let plan = pkgs.writeText "krops-test-${name}.json" (builtins.toJSON (map (scenario: scenario // { prepare = pkgs.writeText "prepare" scenario.prepare; + canarySetup = pkgs.writeText "canary-setup" scenario.canarySetup; + canaryCheck = pkgs.writeText "canary-check" scenario.canaryCheck; prepareDeployer = script "prepare-deployer" scenario.prepareDeployer; phases = map (phase: phase // { @@ -130,6 +132,9 @@ in pkgs.testers.runNixOSTest { run(deployer, "HOME=/root sh ${./fixtures.sh} http://deployer:8000 ${gitHead}") failed = [] + # The canaries get created at the start, after isolating, which removes + # some, and after a failure, which may have changed some. + canaries = False for scenario in plan: node = nodes[scenario["node"]] try: @@ -137,6 +142,10 @@ in pkgs.testers.runNixOSTest { if scenario["isolate"]: for machine in nodes.values(): run(machine, "rm -fR /var/krops && nix-store --gc >/dev/null") + if scenario["isolate"] or not canaries: + for machine in nodes.values(): + run(machine, f"sh {scenario['canarySetup']}") + canaries = True if scenario["prepareDeployer"] is not None: run(deployer, f"sh {scenario['prepareDeployer']}") run(node, f"sh {scenario['prepare']}") @@ -160,9 +169,14 @@ in pkgs.testers.runNixOSTest { run(node, f"sh {phase['mutate']}") if phase["mutateDeployer"] is not None: run(deployer, f"sh {phase['mutateDeployer']}") + # Each phase's check covers the canaries of its machine. + for machine in nodes.values(): + if machine is not node: + run(machine, f"sh {scenario['canaryCheck']}") except Exception as e: print(f"scenario {scenario['name']} fails: {e}") failed.append(scenario["name"]) + canaries = False if failed: raise Exception(f"{len(failed)} scenarios fail: {' '.join(failed)}") |
