diff options
| -rw-r--r-- | README.md | 29 | ||||
| -rw-r--r-- | pkgs/populate/default.nix | 208 | ||||
| -rw-r--r-- | tests/default.nix | 4 | ||||
| -rw-r--r-- | tests/lib.sh | 64 | ||||
| -rw-r--r-- | tests/scenarios.nix | 360 | ||||
| -rw-r--r-- | tests/vm.nix | 14 |
6 files changed, 610 insertions, 69 deletions
@@ -290,6 +290,29 @@ pkgs.krops.writeCommand "deploy-with-swap" { whether the ssh session should do a pseudo-terminal allocation. sets `-t` on the ssh command. +## The target directory + +Populate owns the target directory, `/var/src` by default: it replaces +whatever is at the target path of a source, and with `prune` it removes +everything else. It doesn't change or remove anything outside: + +* Target paths must be absolute, must not be the root directory, and must not + contain `..`. Source names must be file names other than `.`, `..`, and + `.populate`, without slashes and control characters. Anything else fails + when evaluating. + +* A symlink at the target path of a source gets replaced by what the source + delivers, not followed, and populate says so. To keep a source elsewhere, + e.g. on another disk or in memory, mount that place at the target path, or + make the target directory itself a symlink. A Git work tree whose `.git` + is a symlink, or a file naming a repository elsewhere, gets replaced by a + `git` source as well. + +* Populate keeps its state in the sentinel, `.populate`, which becomes a + directory once there is state to keep. Populating fails if the sentinel, + or a directory populate needs therein, is a symlink. + + ## Source Types ### `derivation` @@ -658,8 +681,10 @@ flakes, as long as the inputs drawing from them are declared with `tests/` contains NixOS VM tests for populate: one test per source type, one for features spanning sources, such as flake inputs from sources, one for edge -cases of flakes, such as unusual lock files, and one for the arguments -`writeDeploy` passes to `nixos-rebuild`. The tests deploy every combination +cases of flakes, such as unusual lock files, one for staying inside the target +directory, whatever links it contains, and one for the arguments `writeDeploy` +passes to `nixos-rebuild`. After every deploy, all tests check that nothing +outside the target directory changed. The tests deploy every combination of a source type's options, except for patches, which the features test covers, to the deploying machine itself, to a remote target as `root`, and to a remote target using `sudo`, and check the result after diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index a7e351a..002326a 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -4,9 +4,119 @@ with shell; { coreutils, dash, findutils, git, jq, nix, openssh, pass, passage, rsync, writers }: let + # Populate replaces and removes what's at the target paths of sources, so + # it refuses names that lead out of the target directory, to the directory + # itself, or to its own state. + checkSourceName = name: + if elem name [ "" "." ".." ".populate" ] || + builtins.match "[^/[:cntrl:]]+" name == null then + throw "krops: source name ${toJSON name} can't be used: it must be a file name other than ., .., and .populate, without slashes and control characters" + else + true; + + # Populate removes what it finds in the target directory, so it refuses + # paths that aren't absolute, that are the root directory, or that the + # target would resolve to another directory than the one they name. + checkTargetPath = path: let + components = filter (x: x != "" && x != ".") (splitString "/" path); + in + if !isString path || substring 0 1 path != "/" then + throw "krops: target path ${toJSON path} can't be used: it must be absolute" + else if components == [] then + throw "krops: target path ${toJSON path} can't be used: it must not be the root directory" + else if elem ".." components || + builtins.match "[^[:cntrl:]]+" path == null then + throw "krops: target path ${toJSON path} can't be used: it must not contain .. or control characters" + else + true; + + # The directories below the sentinel, once that is a directory, which + # have a directory for each source that needs one. Besides, there is + # backup, which holds copies of the sources. + stateKinds = [ "derived" "flake-inputs" "flake-uploads" "flakes" "patches" ]; + + # Shell code that prepares keeping state below the sentinel for the sources + # named names. The sentinel only needs to exist, so it may be a file, + # which becomes a directory. Populate changes and removes what's in its + # state directories, and a symlink in place of one would make it do that to + # whatever the link points to, so it refuses to use those. + useState = sentinel: names: /* sh */ '' + refuse_link() { + if test -L "$1"; then + >&2 printf 'error: %s is a symlink, but populate needs a directory of its own there\n' "$1" + exit 1 + fi + } + refuse_link ${quote sentinel} + refuse_link ${quote sentinel}/backup + for state_kind in ${toString stateKinds}; do + refuse_link ${quote sentinel}/"$state_kind" + for state_name in ${concatMapStringsSep " " quote names}; do + refuse_link ${quote sentinel}/"$state_kind"/"$state_name" + done + done + if test -f ${quote sentinel}; then + rm ${quote sentinel} + mkdir ${quote sentinel} + fi + ''; + + # Shell code that removes what's at path, a shell word, with everything + # below it, also what isn't writable. A symlink gets removed itself, + # instead of followed. + removeTree = path: /* sh */ '' + if test -L ${path}; then + rm ${path} + elif test -e ${path}; then + chmod -R u+w ${path} + rm -fR ${path} + fi + ''; + + # Shell code that removes a symlink at path, saying so, because the link + # may be there on purpose, but populate never follows it: what a source + # delivers would change or remove what the link points to. + removeLink = path: /* sh */ '' + if test -L ${quote path}; then + >&2 printf 'replacing symlink %s, which points to %s\n' \ + ${quote path} "$(readlink ${quote path})" + rm ${quote path} + fi + ''; + + # Shell code that removes a symlink at path like removeLink does, unless + # it points into the store, like the links populate puts there itself. + removeForeignLink = path: /* sh */ '' + if test -L ${quote path}; then + case $(readlink ${quote path}) in + /nix/store/*) ;; + *) + ${removeLink path} + ;; + esac + fi + ''; + + # Shell code that makes room at path for a link into the store, which Nix + # only puts in place of another link into the store. + makeRoomForLink = path: /* sh */ '' + ${removeForeignLink path} + if ! test -L ${quote path} && test -e ${quote path}; then + rm -fR ${quote path} + fi + ''; + check = { force, target }: let sentinelFile = "${target.path}/.populate"; in runShell target /* sh */ '' + # Before anything touches it, which would touch what it points to. + if test -L ${quote sentinelFile}; then + >&2 printf 'error: sentinel file is a symlink: %s\n' ${quote ( + optionalString (!isLocalTarget target) "${target.host}:" + + sentinelFile + )} + exit 1 + fi ${optionalString force /* sh */ '' mkdir -vp ${quote (dirOf sentinelFile)} >&2 touch ${quote sentinelFile} @@ -24,6 +134,7 @@ let sentinelFile = "${target.path}/.populate"; in runShell target /* sh */ '' + ${useState sentinelFile []} if ! test -d ${quote sentinelFile}; then >&2 printf 'error: sentinel file is not a directory: %s\n' ${quote ( optionalString (!isLocalTarget target) "${target.host}:" + @@ -36,19 +147,10 @@ let --delete \ --exclude /.populate \ --link-dest=${quote target.path} \ - ${target.path}/ \ - ${target.path}/.populate/backup/ + ${quote target.path}/ \ + ${quote target.path}/.populate/backup/ ''; - # Shell code that turns the sentinel file into a directory, which populate - # uses for keeping state. The sentinel only needs to exist. - sentinelDirectory = sentinel: /* sh */ '' - if test -f ${quote sentinel}; then - rm ${quote sentinel} - mkdir ${quote sentinel} - fi - ''; - # Draws inputs of flakes in sources from other sources, and derives what the # target paths of flake and flake-input sources that get patched, get their # lock files rewritten, or are materialized get. Sources get processed in @@ -85,9 +187,11 @@ let # Shell word for the URL of the flake that inputs drawn from a source use: # its generated flake, if it has one, so overrides don't need to be # repeated, or else the flake in the source. + # Nix refuses path: URLs with symlinks on the way, which the target + # directory may be, so paths get resolved. url = name: if generated ? ${name} - then quote "path:${flakesDir}/${name}" + then ''"path:$(readlink -f ${quote "${flakesDir}/${name}"})"'' else ''"$(flake_url ${quote name})"''; check = name: source: if source == name then @@ -108,6 +212,7 @@ let rewrite = name: /* sh */ '' ( ${overrideArgs name} + own_lock "$(readlink -f ${quote "${target.path}/${name}"})" nix flake lock "$@" "$(flake_url ${quote name})" >&2 ) ''; @@ -143,12 +248,7 @@ let ''; # Removes the work directory, which an interrupted run may have left # behind. - removeWork = /* sh */ '' - if test -e ${quote "${dir}/work"}; then - chmod -R u+w ${quote "${dir}/work"} - rm -fR ${quote "${dir}/work"} - fi - ''; + removeWork = removeTree (quote "${dir}/work"); in /* sh */ '' ( original=$(readlink -f ${quote "${dir}/original"}) @@ -192,12 +292,14 @@ let cd ${quote "${dir}/work"} ${applyPatches} ) + ${removeForeignLink targetPath} rm -fR ${quote targetPath} mv ${quote "${dir}/work"} ${quote targetPath} printf '%s\n' "$key" > ${quote "${dir}/key"} fi ${optionalString rewriteLock /* sh */ '' ${overrideArgs name} + own_lock ${quote targetPath} nix flake lock "$@" "$(flake_url ${quote name})" >&2 ''} '' else /* sh */ '' @@ -208,8 +310,8 @@ let : else # The copy keeps the name of the delivered store path. - work=${quote "${dir}/work"}/$(basename "$path" | cut -c34-) mkdir ${quote "${dir}/work"} + work=$(readlink -f ${quote "${dir}/work"})/$(basename "$path" | cut -c34-) cp -R "$path" "$work" chmod -R u+w "$work" ( @@ -228,9 +330,7 @@ let --add-root ${quote "${dir}/result"} --indirect >/dev/null printf '%s\n' "$key" > ${quote "${dir}/key"} fi - if test -d ${quote targetPath} && ! test -L ${quote targetPath}; then - rm -fR ${quote targetPath} - fi + ${makeRoomForLink targetPath} # .populate/derived/<name>/result roots the store path. ln -fns "$result''${subdir:+/$subdir}" ${quote targetPath} ''} @@ -286,7 +386,7 @@ let echo '}' } > flake.nix # Not just ., which Nix takes for the enclosing Git work tree, if any. - nix flake lock ${quote "path:${flakesDir}/${name}"} >&2 + nix flake lock "path:$(pwd -P)" >&2 ) ''; process = name: @@ -301,8 +401,7 @@ let case ${quote " ${toString names} "} in *" $name "*) ;; *) - chmod -R u+w ${quote dir}/"$name" - rm -fR ${quote dir}/"$name" + ${removeTree ''${quote dir}/"$name"''} ;; esac done @@ -316,10 +415,22 @@ let throw "krops: sources draw inputs from each other: ${toString sorted.cycle}" else runShell target /* sh */ '' set -efu + ${useState sentinel (attrNames sources)} ${optionalString (names != []) /* sh */ '' ${optionalString (drawing != {}) (requireFlakes "drawing flake inputs from sources")} + # Nix rewrites lock files in place, which also changes what else is + # a hard link to the lock file, e.g. in a backup, or what it is a + # symlink to. So the lock file of the flake in the directory $1 + # gets replaced by a copy first. + own_lock() { + if test -e "$1"/flake.lock; then + cp -p "$1"/flake.lock "$1"/flake.lock.populate + mv -f "$1"/flake.lock.populate "$1"/flake.lock + fi + } + # Prints the flake URL of the source named $1. flake_url() { path=$(readlink -f ${quote target.path}/"$1") @@ -356,7 +467,6 @@ let fi ${optionalString (generated != {}) /* sh */ '' - ${sentinelDirectory sentinel} mkdir -p ${quote flakesDir} ''} ${concatMapStrings process sorted.result} @@ -388,6 +498,7 @@ let sources); in runShell target /* sh */ '' set -efu + ${useState "${target.path}/.populate" []} cd ${quote target.path} ls -A | while read -r name; do case $name in @@ -410,13 +521,11 @@ let rmdir "$roots" fi fi - if test -e .populate/flake-uploads; then - chmod -R u+w .populate/flake-uploads - rm -fR .populate/flake-uploads - fi + ${removeTree ".populate/flake-uploads"} ''; pop.derivation = target: source: runShell target /* sh */ '' + ${makeRoomForLink target.path} nix-build -E ${quote source.text} -o ${quote target.path} >&2 ''; @@ -463,7 +572,7 @@ let '') patches} ${runShell target /* sh */ '' set -efu - ${sentinelDirectory sentinel} + ${useState sentinel [ name ]} mkdir -p ${quote "${sentinel}/patches"} ''} ${rsync' (target // { path = "${sentinel}/patches/${name}"; }) @@ -533,10 +642,7 @@ let uploads = "${sentinel}/flake-uploads"; upload = "${uploads}/${name}"; removeUpload = /* sh */ '' - if test -e ${quote upload}; then - chmod -R u+w ${quote upload} - rm -fR ${quote upload} - fi + ${removeTree (quote upload)} # Sources get populated one after another, so no other source uses the # directory meanwhile. if test -d ${quote uploads} && test -z "$(ls -A ${quote uploads})"; then @@ -552,26 +658,17 @@ let set -efu read -r dir read -r path + ${useState sentinel [ name ]} ${optionalString derived /* sh */ '' - ${sentinelDirectory sentinel} mkdir -p ${quote (dirOf linkPath)} ''} - # nix-store --add-root only replaces links into the store. - if test -L ${quote linkPath}; then - case $(readlink ${quote linkPath}) in - /nix/store/*) ;; - *) rm ${quote linkPath} ;; - esac - elif test -e ${quote linkPath}; then - rm -fR ${quote linkPath} - fi + ${makeRoomForLink linkPath} nix-store --realise "$path" --add-root ${quote linkPath} --indirect >/dev/null # The garbage collector follows the link into the store path. if test -n "$dir"; then ln -fns "$path/$dir" ${quote linkPath} fi - ${sentinelDirectory sentinel} # Left behind by an interrupted run, or not needed anymore. ${removeUpload} mkdir -p ${quote inputsRoots} @@ -742,7 +839,7 @@ let }; then ${runShell target /* sh */ '' set -efu - ${sentinelDirectory sentinel} + ${useState sentinel [ name ]} ${removeUpload} mkdir -p ${quote upload} ''} @@ -792,8 +889,13 @@ let pop.git = target: source: runShell target /* sh */ '' set -efu # Remove target path if it doesn't look like a git worktree. - # This can happen e.g. when it had a different type earlier. - if ! test -e ${quote target.path}/.git; then + # This can happen e.g. when it had a different type earlier. A work + # tree that a symlink points to, or whose .git is a symlink or a file + # naming a repository elsewhere, isn't populate's own: fetching, + # resetting, and cleaning would change what's elsewhere. + ${removeLink target.path} + if test -L ${quote target.path}/.git || + ! test -d ${quote target.path}/.git; then rm -fR ${quote target.path} fi if ! test -e ${quote target.path}; then @@ -955,11 +1057,7 @@ let rsync' = target: config: sourcePath: /* sh */ '' # rsync writes through a symlink at the target path, e.g. one a symlink or # flake source left there, into what it points to. - ${runShell target /* sh */ '' - if test -L ${quote target.path}; then - rm ${quote target.path} - fi - ''} + ${runShell target (removeLink target.path)} source_path=${sourcePath} if test -d "$source_path"; then source_path=$source_path/ @@ -1088,6 +1186,8 @@ let in { backup ? false, force ? false, prune ? false, source, target }: +assert checkTargetPath target.path; +assert all checkSourceName (attrNames source); writers.writeDash "populate.${target.host}" '' set -efu diff --git a/tests/default.nix b/tests/default.nix index 285ed86..7b82bce 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -1,6 +1,6 @@ # NixOS VM tests for populate: one test per source type, one for features -# spanning sources, one for edge cases of flakes, and one for writeDeploy's -# arguments to nixos-rebuild. Build all with nix-build tests, or one with, +# spanning sources, one for edge cases of flakes, one for staying inside the +# target directory, and one for writeDeploy's arguments to nixos-rebuild. Build all with nix-build tests, or one with, # e.g., nix-build tests -A flake. jq runs populate's jq programs for flakes # without VMs. { pkgs ? import ../pkgs {} }: diff --git a/tests/lib.sh b/tests/lib.sh index 33a0c6e..4f62890 100644 --- a/tests/lib.sh +++ b/tests/lib.sh @@ -48,3 +48,67 @@ gc() { nix-store --gc >/dev/null fi } + +# Canaries are files outside the target directories, which populate must +# never change, whatever the target directories contain. Some scenarios +# point links at them. The manifest lists every entry with its type, mode, +# and link target, and every file with its checksum. +canary_manifest() { + { + find /var/krops/canary /var/krops/sibling /var/krops-outside \ + -printf '%y %m %p -> %l\n' + find /var/krops/canary /var/krops/sibling /var/krops-outside \ + -type f -exec sha256sum {} + + } 2>&1 | sort +} + +# Creates the canaries, unless they are there unchanged: a file beside the +# target directories, a directory beside them, and a directory elsewhere +# with files, a read-only directory, and a Git work tree with a changed and +# an untracked file, which resetting and cleaning would lose. +canary_setup() { + if test -e /var/krops-outside.manifest && + canary_manifest | cmp -s - /var/krops-outside.manifest; then + return + fi + rm -fR /var/krops/canary /var/krops/sibling /var/krops-outside + mkdir -p /var/krops/sibling /var/krops-outside/dir/sub + echo canary > /var/krops/canary + echo sibling > /var/krops/sibling/file + echo outside > /var/krops-outside/file + echo outside > /var/krops-outside/dir/file + echo outside > /var/krops-outside/dir/sub/file + chmod a-w /var/krops-outside/dir/sub/file /var/krops-outside/dir/sub + git init -q /var/krops-outside/repo + ( + cd /var/krops-outside/repo + echo tracked > tracked + git add tracked + GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 \ + git -c user.name=krops -c user.email=spam@krebsco.de commit -q -m canary + echo changed > tracked + echo untracked > untracked + ) + canary_manifest > /var/krops-outside.manifest +} + +# Fails if a canary changed, or if a store path that an entry of the target +# directory $T links to doesn't match its hash anymore. +canary_check() { + if ! canary_manifest | cmp -s - /var/krops-outside.manifest; then + canary_manifest | diff /var/krops-outside.manifest - >&2 || : + fail "something outside the target directory changed" + fi + if test -n "${T-}" && test -d "$T"; then + ls -A "$T" | while IFS= read -r canary_name; do + canary_link=$(readlink "$T/$canary_name" 2>/dev/null) || continue + case $canary_link in + /nix/store/*) + nix-store --verify-path \ + "$(printf '%s\n' "$canary_link" | cut -d/ -f1-4)" || + fail "$T/$canary_name links to a store path that changed" + ;; + esac + done + fi +} diff --git a/tests/scenarios.nix b/tests/scenarios.nix index 2815c3f..af072b9 100644 --- a/tests/scenarios.nix +++ b/tests/scenarios.nix @@ -94,37 +94,49 @@ let # differ only in what makes evaluation fail, or a function returning them # for the phase's sources, and that must evaluate. So the phase doesn't # pass for failing for an unrelated reason, e.g. a misspelt option. + # targetPath deploys to another path than the scenario's target + # directory, and controlTargetPath is the control for a path that makes + # evaluation fail. phase = { - sources, prune ? false, deploy ? null, - expect ? "success", control ? null, + sources, backup ? false, prune ? false, deploy ? null, + targetPath ? path, + expect ? "success", control ? null, controlTargetPath ? null, check ? "", mutate ? null, checkDeployer ? null, mutateDeployer ? null, }: let - command = sources: let + command = sources: targetPath: let args = { source = krops.evalSource [ sources ]; - inherit prune target; + inherit backup prune; + target = target // { path = targetPath; }; force = true; }; in if deploy == null then populate args else writeDeploy "deploy" (args // { useNixOutputMonitor = false; } // deploy); - evaluates = sources: - (builtins.tryEval (builtins.seq (toString (command sources)) true)).success; + evaluates = sources: targetPath: + (builtins.tryEval + (builtins.seq (toString (command sources targetPath)) true)).success; script = text: if text == null then null else prelude + vars + text; in if expect == "eval-error" then - if control == null then + if control == null && controlTargetPath == null then throw "scenario ${name}: expecting evaluation to fail needs a control" - else if evaluates sources then + else if evaluates sources targetPath then throw "scenario ${name}: populate evaluates, but is expected to fail" - else if !evaluates (if lib.isFunction control then control sources else control) then + else if !( + if controlTargetPath != null then evaluates sources controlTargetPath + else evaluates + (if lib.isFunction control then control sources else control) + targetPath + ) then throw "scenario ${name}: the control doesn't evaluate either" else null else { - populate = toString (command sources); + populate = toString (command sources targetPath); expect = if expect == "success" then null else expect.fail; - check = script check; + # No phase gets around checking the canaries. + check = script (check + "\ncanary_check\n"); mutate = script mutate; checkDeployer = script checkDeployer; mutateDeployer = script mutateDeployer; @@ -140,6 +152,10 @@ let '' + prepare; prepareDeployer = if prepareDeployer == null then null else prelude + vars + prepareDeployer; + # For all machines: before the scenario, and after it for the ones its + # checks don't run on. + canarySetup = prelude + "canary_setup\n"; + canaryCheck = prelude + "canary_check\n"; phases = lib.filter (x: x != null) (map phase phases); }; @@ -1255,4 +1271,326 @@ assert !(krops.isLocalTarget (targets.remote "/")); }) ) targetKinds; + + # Populate replaces and removes things in the target directory. These + # scenarios check that it doesn't anywhere else: names and paths that lead + # out of the target directory don't evaluate, and symlinks in it get + # replaced or refused instead of followed. The canaries, which every phase + # of every scenario checks, tell if something outside changed; these + # scenarios point links at them. + safety = concatMap (targetKind: let + s = name: args: scenario ({ + name = "${name}-${targetKind}"; + inherit targetKind; + } // args); + copying = targetKind != "sudo"; + fetchOnTarget = !copying; + outside = "/var/krops-outside"; + hello = [ "/fixtures/patches/hello.patch" ]; + # Sources that use every kind of state populate keeps: x a copy with + # patches, y garbage collector roots and, where the flake gets uploaded, + # an upload, and z a generated flake. + stateful = alt // { + x.flake-input = { + url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; + patches = hello; + inputs.inner.follows = "alt"; + }; + y.flake = { url = "/fixtures/top"; inherit fetchOnTarget; }; + z.file = { + path = "/fixtures/filesrc"; + rewriteLock = false; + inputs.dep.follows = "alt"; + }; + }; + # A source deployed where a symlink to something outside is. The link + # gets replaced, never followed. + link = type: { sources, to, check }: s "link-${type}" { + isolate = true; + prepare = '' + mkdir -p "$T" + ln -s ${outside}/${to} "$T"/x + ''; + phases = twice { + inherit sources; + check = '' + case $(readlink "$T"/x || :) in + ${outside}/*) fail "x still links outside" ;; + esac + '' + check; + }; + }; + # Something of populate's state replaced by a symlink to outside between + # two deploys. The second one refuses to use it. + state = name: { + entry, backup ? false, + message ? "/${entry} is a symlink, but populate needs a directory of its own there", + }: s "state-${name}" { + isolate = true; + phases = [ + { + sources = stateful; + inherit backup; + mutate = '' + rm -fR "$T"/${entry} + mkdir -p "$(dirname "$T"/${entry})" + ln -s ${outside}/dir "$T"/${entry} + ''; + } + { + sources = stateful; + inherit backup; + expect.fail = message; + } + ]; + }; + in lib.flatten [ + (link "file" { + sources.x.file = "/fixtures/alt"; + to = "dir"; + check = '' + test -e "$T"/x/flake.nix || fail "flake.nix missing" + ''; + }) + # The link points at a Git work tree, which resetting and cleaning would + # change. + (link "git" { + sources.x.git = { url = gitUrl; ref = gitHead; }; + to = "repo"; + check = '' + expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitHead} + ''; + }) + (link "pass" { + sources.x.pass = { dir = "/fixtures/pass"; name = "x"; }; + to = "dir"; + check = '' + expect_eq a "$(cat "$T"/x/a)" secret-a + ''; + }) + (link "passage" { + sources.x.passage = { + dir = "/fixtures/passage/store/x"; + identities_file = "/fixtures/passage/identities"; + }; + to = "dir"; + check = '' + expect_eq a "$(cat "$T"/x/a)" secret-a + ''; + }) + (link "pipe" { + sources.x.pipe = "/fixtures/pipe"; + to = "file"; + check = '' + expect_eq x "$(cat "$T"/x)" piped + ''; + }) + (link "derivation" { + sources.x.derivation = '' + derivation { + name = "krops-test"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo built > $out" ]; + } + ''; + to = "file"; + check = '' + expect_eq x "$(cat "$T"/x)" built + ''; + }) + (link "symlink" { + sources = alt // { x.symlink = "alt"; }; + to = "dir"; + check = '' + expect_eq link "$(readlink "$T"/x)" alt + ''; + }) + (link "flake" { + sources.x.flake = { url = "/fixtures/top"; inherit fetchOnTarget; }; + to = "dir"; + check = checkFlake { attr = "msg"; expected = "top:dep+inner,sub"; }; + }) + (link "flake-materialized" { + sources.x.flake = { + url = "/fixtures/top"; inherit fetchOnTarget; + materialize = true; + }; + to = "dir"; + check = checkFlake { + attr = "msg"; expected = "top:dep+inner,sub"; materialized = true; + }; + }) + (link "flake-input-patched" { + sources.x.flake-input = { + url = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; + patches = hello; + }; + to = "dir"; + check = checkFlake { attr = "greeting"; expected = "hello+inner"; }; + }) + # Symlinks to outside in what a source replaces: rsync and Git remove + # them, and so does replacing a directory by a link. + (s "inner-links" (let + prepare = '' + for name in a b c; do + mkdir -p "$T"/$name + ln -s ${outside}/dir "$T"/$name/keep + ln -s ${outside}/dir "$T"/$name/stale + ln -s ${outside}/repo/.git "$T"/$name/.git + done + # A work tree whose repository is elsewhere. + mkdir -p "$T"/d + echo "gitdir: ${outside}/repo/.git" > "$T"/d/.git + ''; + in { + isolate = true; + inherit prepare; + phases = twice { + sources = { + a.file = "/fixtures/filesrc"; + b.git = { url = gitUrl; ref = gitHead; }; + c.flake = { url = "/fixtures/top"; inherit fetchOnTarget; }; + d.git = { url = gitUrl; ref = gitHead; }; + }; + check = '' + expect_eq keep "$(cat "$T"/a/keep)" keep + test ! -L "$T"/a/stale || fail "a/stale remains" + expect_eq HEAD "$(git -C "$T"/b rev-parse HEAD)" ${gitHead} + test ! -L "$T"/b/.git || fail "b/.git remains a link" + test ! -L "$T"/b/stale || fail "b/stale remains" + expect_eq HEAD "$(git -C "$T"/d rev-parse HEAD)" ${gitHead} + test -d "$T"/d/.git || fail "d/.git isn't a directory" + expect_eq "store directory" "$(dirname "$(readlink "$T"/c)")" /nix/store + ''; + }; + })) + (state "sentinel" { + entry = ".populate"; + message = "error: sentinel file is a symlink"; + }) + (state "backup" { entry = ".populate/backup"; backup = true; }) + (map + (entry: state (lib.replaceStrings ["/"] ["-"] entry) { + entry = ".populate/${entry}"; + }) + [ + "derived" "flake-inputs" "flake-uploads" "flakes" "patches" + "derived/x" "patches/x" "flake-inputs/y" "flake-uploads/y" "flakes/z" + ]) + # Symlinks to outside among what populate removes: leftovers of + # interrupted runs, state of sources that are gone, and, when pruning, + # entries of the target directory. They get removed as links. + (s "leftovers" { + isolate = true; + prepare = '' + mkdir -p "$T" + ln -s ${outside}/dir "$T"/other + ln -s ${outside}/repo "$T"/repo + ''; + phases = [ + { + sources = stateful; + mutate = '' + ln -s ${outside}/dir "$T"/.populate/derived/x/work + ln -s ${outside}/dir "$T"/.populate/derived/gone + ln -s ${outside}/dir "$T"/.populate/flakes/gone + ln -s ${outside}/dir "$T"/.populate/patches/gone + ln -s ${outside}/dir "$T"/.populate/flake-inputs/gone + mkdir -p "$T"/.populate/flake-uploads + ln -s ${outside}/dir "$T"/.populate/flake-uploads/gone + ''; + } + { + sources = stateful; + check = '' + for name in derived/x/work derived/gone flakes/gone patches/gone; do + test ! -L "$T"/.populate/$name || fail "$name remains" + done + for name in other repo .populate/flake-inputs/gone .populate/flake-uploads/gone; do + test -L "$T"/$name || fail "$name removed without pruning" + done + ''; + } + { + sources = stateful; + prune = true; + check = '' + for name in other repo .populate/flake-inputs/gone .populate/flake-uploads; do + test ! -L "$T"/$name && test ! -e "$T"/$name || fail "$name not pruned" + done + ''; + } + ]; + }) + # The target directory itself may be a symlink: what it points to is the + # target directory then. + (s "linked-target" { + isolate = true; + prepare = '' + rm -fR "$T"-real + mkdir "$T"-real + ln -s "$(basename "$T")"-real "$T" + ''; + phases = twice { + sources = stateful; + check = '' + test -L "$T" || fail "the target directory isn't a link anymore" + test -e "$T"-real/alt/flake.nix || fail "alt missing" + gc + expect_eq greeting "$(nix_eval "$(readlink -f "$T"/x)"#greeting)" hello+alt + expect_eq msg "$(nix_eval "$(readlink -f "$T"/y)"#msg)" top:dep+inner,sub + expect_eq msg "$(nix_eval_online "path:$(readlink -f "$T"/.populate/flakes/z)#msg")" top:alt,sub + ''; + }; + }) + (lib.optionals (targetKind == "local") [ + # Rewriting a lock file changes no other hard link to it. The lock + # file is like the source's, with its modification time, so rsync + # leaves the link in place. (The source is on the deploying machine, + # so this only runs there.) + (s "lock-hard-link" { + prepare = '' + mkdir -p "$T" + cp -a /fixtures/filesrc "$T"/x + cp -p "$T"/x/flake.lock "$T".lock + ln -f "$T".lock "$T"/x/flake.lock + ''; + phases = twice { + sources = alt // { + x.file = { path = "/fixtures/filesrc"; inputs.dep.follows = "alt"; }; + }; + check = '' + cmp -s "$T".lock /fixtures/filesrc/flake.lock || + fail "the other hard link changed" + expect_eq msg "$(nix_eval_online "$T"/x#msg)" top:alt,sub + ''; + }; + }) + # Names that lead out of the target directory, to the directory itself, + # or to populate's state. + (s "names" { + phases = map + (name: { + sources.${name}.file = "/fixtures/alt"; + expect = "eval-error"; + control = { ok.file = "/fixtures/alt"; }; + }) + [ "" "." ".." ".populate" "a/b" "../x" "/etc" "a\nb" ]; + }) + # Target paths that aren't absolute, that are the root directory, or + # that resolve to another directory than they name. + (s "paths" { + phases = map + (targetPath: { + sources = alt; + inherit targetPath; + expect = "eval-error"; + controlTargetPath = "/var/krops/paths-local"; + }) + [ "" "/" "//" "/." "relative" "/var/krops/.." "/var/krops/../.." "/var/krops/a\nb" ]; + }) + ]) + ]) targetKinds; + } diff --git a/tests/vm.nix b/tests/vm.nix index d808dd2..5eb4dbb 100644 --- a/tests/vm.nix +++ b/tests/vm.nix @@ -14,6 +14,8 @@ let plan = pkgs.writeText "krops-test-${name}.json" (builtins.toJSON (map (scenario: scenario // { prepare = pkgs.writeText "prepare" scenario.prepare; + canarySetup = pkgs.writeText "canary-setup" scenario.canarySetup; + canaryCheck = pkgs.writeText "canary-check" scenario.canaryCheck; prepareDeployer = script "prepare-deployer" scenario.prepareDeployer; phases = map (phase: phase // { @@ -130,6 +132,9 @@ in pkgs.testers.runNixOSTest { run(deployer, "HOME=/root sh ${./fixtures.sh} http://deployer:8000 ${gitHead}") failed = [] + # The canaries get created at the start, after isolating, which removes + # some, and after a failure, which may have changed some. + canaries = False for scenario in plan: node = nodes[scenario["node"]] try: @@ -137,6 +142,10 @@ in pkgs.testers.runNixOSTest { if scenario["isolate"]: for machine in nodes.values(): run(machine, "rm -fR /var/krops && nix-store --gc >/dev/null") + if scenario["isolate"] or not canaries: + for machine in nodes.values(): + run(machine, f"sh {scenario['canarySetup']}") + canaries = True if scenario["prepareDeployer"] is not None: run(deployer, f"sh {scenario['prepareDeployer']}") run(node, f"sh {scenario['prepare']}") @@ -160,9 +169,14 @@ in pkgs.testers.runNixOSTest { run(node, f"sh {phase['mutate']}") if phase["mutateDeployer"] is not None: run(deployer, f"sh {phase['mutateDeployer']}") + # Each phase's check covers the canaries of its machine. + for machine in nodes.values(): + if machine is not node: + run(machine, f"sh {scenario['canaryCheck']}") except Exception as e: print(f"scenario {scenario['name']} fails: {e}") failed.append(scenario["name"]) + canaries = False if failed: raise Exception(f"{len(failed)} scenarios fail: {' '.join(failed)}") |