diff options
| -rw-r--r-- | README.md | 16 | ||||
| -rw-r--r-- | tests/default.nix | 24 | ||||
| -rwxr-xr-x | tests/fixtures.sh | 178 | ||||
| -rw-r--r-- | tests/lib.sh | 43 | ||||
| -rw-r--r-- | tests/scenarios.nix | 527 | ||||
| -rw-r--r-- | tests/vm.nix | 142 |
6 files changed, 930 insertions, 0 deletions
@@ -516,6 +516,22 @@ flakes, as long as the inputs drawing from them are declared with `flake = false`. +## Tests + +`tests/` contains NixOS VM tests for populate: one test per source type, and +one for features spanning sources, such as flake inputs from sources. Each +test deploys every combination of a source type's options to the deploying +machine itself, to a remote target as `root`, and to a remote target using +`sudo`, and checks the result after deploying and after deploying again. +The tests require KVM and nixpkgs with `pkgs.testers.runNixOSTest` +(23.05 or later), and use `<nixpkgs>` by default. + +```sh +nix-build tests # all tests +nix-build tests -A flake # the test for one source type +``` + + ## References - [In-depth example](http://tech.ingolf-wagner.de/nixos/krops/) by [Ingolf Wagner](https://ingolf-wagner.de/) diff --git a/tests/default.nix b/tests/default.nix new file mode 100644 index 0000000..e9ea769 --- /dev/null +++ b/tests/default.nix @@ -0,0 +1,24 @@ +# NixOS VM tests for populate, one test per source type, plus one for +# features spanning sources. Build all with nix-build tests, or one with, +# e.g., nix-build tests -A flake. +{ pkgs ? import ../pkgs {} }: + +let + # HEAD of /fixtures/gitsrc, which fixtures.sh creates deterministically. + # fixtures.sh fails if it differs, naming the actual one. + gitHead = "e42d44ef9cdb76ba5a84dff4c89c38222703bf0c"; + + scenarios = import ./scenarios.nix { + inherit (pkgs) lib; + krops = import ../lib; + inherit (pkgs) populate; + url = "http://deployer:8000"; + gitUrl = "git://deployer/gitsrc.git"; + remoteHost = "target"; + agePath = "${pkgs.age}/bin/age"; + inherit gitHead; + }; +in + builtins.mapAttrs + (name: scenarios: import ./vm.nix { inherit pkgs name scenarios gitHead; }) + scenarios diff --git a/tests/fixtures.sh b/tests/fixtures.sh new file mode 100755 index 0000000..eb62a61 --- /dev/null +++ b/tests/fixtures.sh @@ -0,0 +1,178 @@ +#! /bin/sh +# Creates the fixtures the tests deploy from in /fixtures. +# +# usage: fixtures.sh URL GIT_HEAD +# +# URL is where /fixtures/www gets served via HTTP, e.g. http://deployer:8000. +# The tarball flakes there get locked against it, so it must be served while +# this script runs. /fixtures/git gets served by git daemon. GIT_HEAD is the +# commit the tests expect as HEAD of /fixtures/gitsrc. +set -efu + +url=$1 +git_head=$2 + +nix() { + command nix --extra-experimental-features 'nix-command flakes' "$@" +} + +export GIT_AUTHOR_NAME=krops GIT_AUTHOR_EMAIL=spam@krebsco.de +export GIT_COMMITTER_NAME=krops GIT_COMMITTER_EMAIL=spam@krebsco.de +export GIT_AUTHOR_DATE='1000000000 +0000' GIT_COMMITTER_DATE='1000000000 +0000' +# Configuration such as commit.gpgsign would change the commits. +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 + +# Commits everything in the current directory. Commits are deterministic, so +# the tests can refer to them by hash. +commit() { + git add -A + git commit -q -m "$1" +} + +tarball() { + tar -C /fixtures/www/src --sort=name --mtime=@1 --owner=0 --group=0 \ + --numeric-owner -czf /fixtures/www/"$1".tar.gz "$1" +} + +rm -fR /fixtures +mkdir -p /fixtures/www/src + +# Tarball flakes, fetched via HTTP: inner, dep (with input inner), mono (with +# a flake in subdirectory sub), and remote (with input dep). +mkdir /fixtures/www/src/inner +cat > /fixtures/www/src/inner/flake.nix <<'NIX' +{ outputs = _: { who = "inner"; }; } +NIX +tarball inner + +mkdir /fixtures/www/src/dep +cat > /fixtures/www/src/dep/flake.nix <<NIX +{ + inputs.inner.url = "$url/inner.tar.gz"; + outputs = { inner, ... }: { greeting = "dep+\${inner.who}"; }; +} +NIX +(cd /fixtures/www/src/dep && nix flake lock) +tarball dep + +mkdir -p /fixtures/www/src/mono/sub +echo mono > /fixtures/www/src/mono/README +cat > /fixtures/www/src/mono/sub/flake.nix <<'NIX' +{ outputs = _: { who = "sub"; }; } +NIX +tarball mono + +mkdir /fixtures/www/src/remote +cat > /fixtures/www/src/remote/flake.nix <<NIX +{ + inputs.dep.url = "$url/dep.tar.gz"; + outputs = { dep, ... }: { msg = "remote:\${dep.greeting}"; }; +} +NIX +(cd /fixtures/www/src/remote && nix flake lock) +tarball remote + +# top: a flake in a Git work tree, with inputs dep and sub (a flake in a +# subdirectory). +mkdir /fixtures/top +cat > /fixtures/top/flake.nix <<NIX +{ + inputs.dep.url = "$url/dep.tar.gz"; + inputs.sub.url = "$url/mono.tar.gz?dir=sub"; + outputs = { dep, sub, ... }: { msg = "top:\${dep.greeting},\${sub.who}"; }; +} +NIX +(cd /fixtures/top && nix flake lock && git init -q && commit top) + +# mono: a Git work tree with a flake in subdirectory sub. +cp -R /fixtures/www/src/mono /fixtures/mono +(cd /fixtures/mono && git init -q && commit mono) + +# alt: a flake to draw inputs from. +mkdir /fixtures/alt +cat > /fixtures/alt/flake.nix <<'NIX' +{ outputs = _: { greeting = "alt"; who = "alt"; }; } +NIX + +# usegone: a flake with an input locked to a path that doesn't exist anymore. +mkdir /fixtures/gone /fixtures/usegone +cat > /fixtures/gone/flake.nix <<'NIX' +{ outputs = _: {}; } +NIX +cat > /fixtures/usegone/flake.nix <<'NIX' +{ + inputs.gone.url = "path:/fixtures/gone"; + outputs = _: {}; +} +NIX +(cd /fixtures/usegone && nix flake lock) +rm -R /fixtures/gone + +# noflake: a directory without flake.nix. +mkdir /fixtures/noflake +echo noflake > /fixtures/noflake/README + +# filesrc: a copy of top, with files for testing the file source type's +# options. same has the same size and modification time as the file the +# tests put on the target beforehand, while skip-ex and skip-f have the same +# size but an older modification time than theirs, so rsync's quick check +# tells them apart. +mkdir /fixtures/filesrc +cp /fixtures/top/flake.nix /fixtures/top/flake.lock /fixtures/filesrc +echo keep > /fixtures/filesrc/keep +echo src > /fixtures/filesrc/skip-ex +echo src > /fixtures/filesrc/skip-f +echo AAAA > /fixtures/filesrc/same +touch -d @1000000000 /fixtures/filesrc/same +touch -d @999999999 /fixtures/filesrc/skip-ex /fixtures/filesrc/skip-f + +# gitsrc: a Git repository with two commits, served by git daemon. Its flake +# has an input that only exists when drawn from a source. +mkdir /fixtures/gitsrc +cat > /fixtures/gitsrc/flake.nix <<'NIX' +{ + inputs.dep.url = "path:/nonexistent"; + outputs = { dep, ... }: { msg = "git:${dep.greeting}"; }; +} +NIX +(cd /fixtures/gitsrc && git init -q -b master && commit one) +echo two > /fixtures/gitsrc/two +(cd /fixtures/gitsrc && commit two) +if test "$(git -C /fixtures/gitsrc rev-parse HEAD)" != "$git_head"; then + >&2 printf 'error: HEAD of /fixtures/gitsrc is %s, but the tests expect %s\n' \ + "$(git -C /fixtures/gitsrc rev-parse HEAD)" "$git_head" + exit 1 +fi +mkdir /fixtures/git +git clone -q --bare /fixtures/gitsrc /fixtures/git/gitsrc.git +git -C /fixtures/git/gitsrc.git config uploadpack.allowReachableSHA1InWant true + +# pass: password stores with x/a, without and with Git. +if ! gpg --list-secret-keys spam@krebsco.de >/dev/null 2>&1; then + gpg --batch --passphrase '' --quick-gen-key spam@krebsco.de default default never +fi +for store in pass pass-git; do + export PASSWORD_STORE_DIR=/fixtures/$store + pass init spam@krebsco.de >/dev/null + if test $store = pass-git; then + pass git init >/dev/null + fi + printf 'secret-a\n' | pass insert -m x/a >/dev/null +done +unset PASSWORD_STORE_DIR + +# passage: a passage store with x/a. +mkdir -p /fixtures/passage/store +age-keygen -o /fixtures/passage/identities 2>/dev/null +age-keygen -y /fixtures/passage/identities > /fixtures/passage/store/.age-recipients +printf 'secret-a\n' | + PASSAGE_DIR=/fixtures/passage/store \ + PASSAGE_IDENTITIES_FILE=/fixtures/passage/identities \ + passage insert -m x/a >/dev/null + +# pipe: a command for the pipe source type. +cat > /fixtures/pipe <<'SH' +#! /bin/sh +echo piped +SH +chmod +x /fixtures/pipe diff --git a/tests/lib.sh b/tests/lib.sh new file mode 100644 index 0000000..13b1ded --- /dev/null +++ b/tests/lib.sh @@ -0,0 +1,43 @@ +# Helpers for preparing and checking scenarios, sourced on the machine +# populate deploys to. +set -efu + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +# usage: expect_eq WHAT ACTUAL EXPECTED +expect_eq() { + if test "$2" != "$3"; then + fail "$1: expected '$3', got '$2'" + fi +} + +nix() { + command nix --extra-experimental-features 'nix-command flakes' "$@" +} + +# Evaluates a flake output without network access to the fixtures, so all +# inputs must be in the store. +nix_eval() { + env \ + http_proxy=http://127.0.0.1:9 https_proxy=http://127.0.0.1:9 \ + HTTP_PROXY=http://127.0.0.1:9 HTTPS_PROXY=http://127.0.0.1:9 \ + all_proxy=http://127.0.0.1:9 ALL_PROXY=http://127.0.0.1:9 \ + no_proxy= NO_PROXY= \ + nix --extra-experimental-features 'nix-command flakes' eval --raw "$@" +} + +# Evaluates a flake output, fetching missing inputs. +nix_eval_online() { + nix eval --raw "$@" +} + +# Collects garbage if KROPS_TEST_GC is set, so evaluating afterwards checks +# that delivered store paths are registered as garbage collector roots. +gc() { + if test -n "${KROPS_TEST_GC-}"; then + nix-store --gc >/dev/null + fi +} diff --git a/tests/scenarios.nix b/tests/scenarios.nix new file mode 100644 index 0000000..22431a2 --- /dev/null +++ b/tests/scenarios.nix @@ -0,0 +1,527 @@ +# Test scenarios for populate: every combination of each source type's +# options, deployed to the local machine, to a remote target as root, and to +# a remote target using sudo, plus scenarios for features spanning sources. +# +# A scenario prepares the target directory, then runs its phases. Each phase +# runs populate, expecting it to succeed or to fail with a message, and checks +# the result on the target. Phases expecting evaluation to fail get checked +# here and don't run. +{ + # nixpkgs' lib. + lib, + # krops' lib. + krops, + # Function from populate's arguments to the command running populate. + populate, + # Where the deploying machine serves /fixtures/www via HTTP. + url, + # URL of the Git repository /fixtures/git/gitsrc.git. + gitUrl, + # Host name of the remote target. + remoteHost, + # Absolute path of age on the deploying machine. + agePath, + # HEAD of /fixtures/gitsrc. + gitHead, +}: + +let + inherit (lib) + attrNames concatMap concatStringsSep foldl' isBool mapAttrsToList + optional optionalAttrs optionalString; + + prelude = ". ${./lib.sh}\n"; + + cartesian = attrs: foldl' + (acc: name: concatMap + (combination: map + (value: combination // { ${name} = value; }) + attrs.${name}) + acc) + [{}] + (attrNames attrs); + + flagsName = flags: concatStringsSep "-" (mapAttrsToList + (name: value: name + (if isBool value then (if value then "1" else "0") else value)) + flags); + + targets = { + local = path: krops.mkTarget "" // { host = "localhost"; inherit path; }; + remote = path: { + user = "root"; host = remoteHost; port = null; inherit path; + sudo = false; extraOptions = []; + }; + sudo = path: { + user = "deploy"; host = remoteHost; port = null; inherit path; + sudo = true; extraOptions = []; + }; + }; + + alt = { alt.file = "/fixtures/alt"; }; + + # Builds a scenario from a name, a target kind, and phases with sources. + scenario = { name, targetKind, prepare ? "", phases }: let + path = "/var/krops/${name}"; + target = targets.${targetKind} path; + vars = "T=${path}\n"; + phase = { sources, expect ? "success", check ? "", mutate ? null }: let + command = populate { + source = krops.evalSource [ sources ]; + inherit target; + force = true; + }; + evaluated = builtins.tryEval (builtins.seq (toString command) true); + in + if expect == "eval-error" then + if evaluated.success then + throw "scenario ${name}: populate evaluates, but is expected to fail" + else + null + else { + populate = toString command; + expect = if expect == "success" then null else expect.fail; + check = prelude + vars + check; + mutate = if mutate == null then null else prelude + vars + mutate; + }; + in { + inherit name; + node = if targetKind == "local" then "deployer" else "target"; + prepare = prelude + vars + '' + rm -fR "$T" + '' + prepare; + phases = lib.filter (x: x != null) (map phase phases); + }; + + # Same sources twice, so the second phase checks redeploying. + twice = phase: [ phase phase ]; + + targetKinds = [ "local" "remote" "sudo" ]; + + # Scenarios for all combinations of options and target kinds. f gets the + # options and the scenario's name, target kind, and target directory, and + # returns the scenario's arguments, or null for invalid combinations. + matrix = type: options: f: + concatMap + (targetKind: lib.filter (x: x != null) (map + (flags: let + name = "${type}-${targetKind}" + + optionalString (flags != {}) "-${flagsName flags}"; + args = f flags { inherit name targetKind; path = "/var/krops/${name}"; }; + in + if args == null then null + else scenario ({ inherit name targetKind; } // args)) + (cartesian options))) + targetKinds; + + # Checks that the source at $T/x is a link to a store path, or to the + # subdirectory sub therein, and evaluates the attribute attr of the flake + # there, through the link, or of its generated flake, after collecting + # garbage. + checkFlake = { attr, expected, subdir ? false, generated ? false }: '' + link=$(readlink "$T"/x) + ${if subdir then '' + expect_eq subdirectory "$(basename "$link")" sub + link=$(dirname "$link") + '' else ""} + expect_eq "store directory" "$(dirname "$link")" /nix/store + test ! -e "$T"/.populate/flake-uploads || fail "upload directory remains" + gc + expect_eq ${attr} "$(nix_eval ${ + if generated then ''"$T"/.populate/flakes/x'' else ''"$T"/x'' + }#${attr})" ${lib.escapeShellArg expected} + ''; + + copyingWithSudo = flags: ctx: + ctx.targetKind == "sudo" && !(flags.fetchOnTarget or false); + +in { + + derivation = matrix "derivation" {} (_: _: { + phases = twice { + sources.x.derivation = '' + derivation { + name = "krops-test"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo built > $out" ]; + } + ''; + check = '' + gc + expect_eq x "$(cat "$T"/x)" built + ''; + }; + }); + + file = let + f = flags: _: { + prepare = '' + mkdir -p "$T"/x + echo old > "$T"/x/skip-ex + echo old > "$T"/x/skip-f + echo stale > "$T"/x/stale + echo BBBB > "$T"/x/same + touch -d @1000000000 "$T"/x/same + ''; + phases = twice { + sources = { + x.file = { + path = "/fixtures/filesrc"; + useChecksum = flags.checksum; + inherit (flags) deleteExcluded; + exclude = optional (flags.exclude or false) "skip-ex"; + filters = optional flags.filters { type = "exclude"; pattern = "skip-f"; }; + inputs = optionalAttrs flags.inputs { dep.follows = "alt"; }; + }; + } // optionalAttrs flags.inputs alt; + check = let + excluded = file: set: '' + ${if !set then '' + expect_eq ${file} "$(cat "$T"/x/${file})" src + '' else if flags.deleteExcluded then '' + test ! -e "$T"/x/${file} || fail "${file} not deleted" + '' else '' + expect_eq ${file} "$(cat "$T"/x/${file})" old + ''} + ''; + in '' + expect_eq keep "$(cat "$T"/x/keep)" keep + ${excluded "skip-ex" (flags.exclude or false)} + ${excluded "skip-f" flags.filters} + ${if flags.deleteExcluded then '' + test ! -e "$T"/x/stale || fail "stale not deleted" + '' else '' + expect_eq stale "$(cat "$T"/x/stale)" stale + ''} + expect_eq same "$(cat "$T"/x/same)" ${if flags.checksum then "AAAA" else "BBBB"} + '' + optionalString flags.inputs '' + # Inputs not drawn from sources aren't delivered, so this needs the + # network and can't collect garbage before. + expect_eq msg "$(nix_eval_online "$T"/.populate/flakes/x#msg)" top:alt,sub + ''; + }; + }; + in + matrix "file" { + checksum = [ false true ]; + deleteExcluded = [ false true ]; + filters = [ false true ]; + inputs = [ false true ]; + } f + ++ [ + # The deprecated exclude option, once, because it warns. + (scenario (let + args = f { + checksum = false; + deleteExcluded = true; + exclude = true; + filters = false; + inputs = false; + } null; + in args // { + name = "file-local-exclude"; + targetKind = "local"; + phases = [ (builtins.head args.phases) ]; + })) + ]; + + flake = matrix "flake" { + kind = [ "git" "path" "subdir" "remote" ]; + fetchOnTarget = [ false true ]; + inputs = [ false true ]; + } (flags: ctx: let + flake = { + git = "/fixtures/top"; + path = "path:/fixtures/top"; + subdir = "git+file:/fixtures/mono?dir=sub"; + remote = "${url}/remote.tar.gz"; + }.${flags.kind}; + msg = { + git = "top:${if flags.inputs then "alt" else "dep+inner"},sub"; + path = msg.git; + remote = "remote:${if flags.inputs then "alt" else "dep+inner"}"; + }; + in + # The flake in a subdirectory has no inputs to draw from sources. + if flags.kind == "subdir" && flags.inputs then null else { + phases = twice ({ + sources = { + x.flake = { + inherit flake; + inherit (flags) fetchOnTarget; + inputs = optionalAttrs flags.inputs { dep.follows = "alt"; }; + }; + } // optionalAttrs flags.inputs alt; + } // (if copyingWithSudo flags ctx then { + expect = "eval-error"; + } else { + check = if flags.kind == "subdir" then checkFlake { + attr = "who"; expected = "sub"; subdir = true; + } else checkFlake { + attr = "msg"; expected = msg.${flags.kind}; generated = flags.inputs; + }; + })); + }); + + flake-input = concatMap + (name: matrix "flake-input-${lib.replaceStrings ["/"] ["."] name}" ({ + fetchOnTarget = [ false true ]; + } // optionalAttrs (name == "dep") { + inputs = [ false true ]; + }) (flags: ctx: let + inputs = flags.inputs or false; + in { + phases = twice ({ + sources = { + x.flake-input = { + flake = "/fixtures/top"; + inherit name; + inherit (flags) fetchOnTarget; + inputs = optionalAttrs inputs { inner.follows = "alt"; }; + }; + } // optionalAttrs inputs alt; + } // (if copyingWithSudo flags ctx then { + expect = "eval-error"; + } else { + check = { + dep = checkFlake { + attr = "greeting"; + expected = if inputs then "dep+alt" else "dep+inner"; + generated = inputs; + }; + "dep/inner" = checkFlake { attr = "who"; expected = "inner"; }; + sub = checkFlake { attr = "who"; expected = "sub"; subdir = true; }; + }.${name}; + })); + })) + [ "dep" "dep/inner" "sub" ]; + + git = matrix "git" { + shallow = [ false true ]; + fetchAlways = [ false true ]; + cleanExclude = [ false true ]; + inputs = [ false true ]; + } (flags: _: let + check = '' + expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitHead} + expect_eq commits "$(git -C "$T"/x rev-list --count HEAD)" ${ + if flags.shallow then "1" else "2" + } + test -e "$T"/x/two || fail "two missing" + '' + optionalString flags.inputs '' + gc + expect_eq msg "$(nix_eval "$T"/.populate/flakes/x#msg)" git:alt + ''; + sources = { + x.git = { + url = gitUrl; + ref = gitHead; + inherit (flags) shallow fetchAlways; + clean.exclude = optional flags.cleanExclude "keepme"; + inputs = optionalAttrs flags.inputs { dep.follows = "alt"; }; + }; + } // optionalAttrs flags.inputs alt; + in { + phases = [ + { + inherit sources check; + mutate = '' + echo junk > "$T"/x/junk + echo keepme > "$T"/x/keepme + ''; + } + { + inherit sources; + check = check + '' + test ! -e "$T"/x/junk || fail "junk not cleaned" + ${if flags.cleanExclude then '' + test -e "$T"/x/keepme || fail "keepme cleaned" + '' else '' + test ! -e "$T"/x/keepme || fail "keepme not cleaned" + ''} + ''; + } + ]; + }); + + pass = matrix "pass" { + store = [ "pass" "pass-git" ]; + } (flags: _: { + phases = twice { + sources.x.pass = { dir = "/fixtures/${flags.store}"; name = "x"; }; + check = '' + expect_eq a "$(cat "$T"/x/a)" secret-a + ''; + }; + }); + + passage = matrix "passage" { + age = [ "default" "path" ]; + dir = [ "store" "subdir" ]; + } (flags: _: { + phases = twice { + sources.x.passage = { + dir = "/fixtures/passage/store" + optionalString (flags.dir == "subdir") "/x"; + identities_file = "/fixtures/passage/identities"; + } // optionalAttrs (flags.age == "path") { + age = agePath; + }; + check = '' + expect_eq a "$(cat "$T"/x/${optionalString (flags.dir == "store") "x/"}a)" secret-a + ''; + }; + }); + + pipe = matrix "pipe" {} (_: _: { + phases = twice { + sources.x.pipe = "/fixtures/pipe"; + check = '' + expect_eq x "$(cat "$T"/x)" piped + ''; + }; + }); + + symlink = matrix "symlink" { + absolute = [ false true ]; + inputs = [ false true ]; + } (flags: ctx: let + target = optionalString flags.absolute "${ctx.path}/" + "topcopy"; + in { + phases = twice { + sources = { + x.symlink = { + inherit target; + inputs = optionalAttrs flags.inputs { dep.follows = "alt"; }; + }; + topcopy.file = "/fixtures/filesrc"; + } // optionalAttrs flags.inputs alt; + check = '' + expect_eq link "$(readlink "$T"/x)" ${target} + test -e "$T"/x/flake.nix || fail "flake.nix missing" + '' + optionalString flags.inputs '' + expect_eq msg "$(nix_eval_online "$T"/.populate/flakes/x#msg)" top:alt,sub + ''; + }; + }); + + # Features spanning several sources. + features = concatMap (targetKind: let + s = name: args: scenario ({ name = "${name}-${targetKind}"; inherit targetKind; } // args); + copying = targetKind != "sudo"; + fetchOnTarget = !copying; + in [ + # A generated flake draws an input from a source with a generated flake. + (s "chain" { + phases = twice { + sources = alt // { + a.flake = { flake = "/fixtures/top"; inherit fetchOnTarget; inputs.dep.follows = "b"; }; + b.flake-input = { + flake = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; + inputs.inner.follows = "alt"; + }; + }; + check = '' + gc + expect_eq msg "$(nix_eval "$T"/.populate/flakes/a#msg)" top:dep+alt,sub + ''; + }; + }) + # An input of an input drawn from a source gets overridden at that source. + (s "nested" { + phases = twice { + sources = alt // { + x.flake = { + flake = "/fixtures/top"; inherit fetchOnTarget; + inputs.dep = { follows = "b"; inputs.inner.follows = "alt"; }; + }; + b.flake-input = { flake = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; }; + }; + check = '' + gc + expect_eq msg "$(nix_eval "$T"/.populate/flakes/x#msg)" top:dep+alt,sub + ''; + }; + }) + # A generated flake gets removed once its source draws no inputs anymore. + (s "removal" { + phases = [ + { + sources = alt // { + x.flake = { flake = "/fixtures/top"; inherit fetchOnTarget; inputs.dep.follows = "alt"; }; + }; + check = '' + test -e "$T"/.populate/flakes/x/flake.lock || fail "generated flake missing" + ''; + } + { + sources = alt // { + x.flake = { flake = "/fixtures/top"; inherit fetchOnTarget; }; + }; + check = '' + test ! -e "$T"/.populate/flakes/x || fail "generated flake remains" + ''; + } + ]; + }) + # Roots follow the delivered input. + (s "retarget" { + phases = [ + { + sources.x.flake-input = { flake = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; }; + check = '' + expect_eq roots "$(ls "$T"/.populate/flake-inputs/x | wc -l)" 1 + ''; + } + { + sources.x.flake-input = { flake = "/fixtures/top"; name = "sub"; inherit fetchOnTarget; }; + check = '' + test ! -e "$T"/.populate/flake-inputs/x || fail "roots remain" + '' + checkFlake { attr = "who"; expected = "sub"; subdir = true; }; + } + ]; + }) + (s "cycle" { + phases = [{ + sources = { + a.file = { path = "/fixtures/alt"; inputs.foo.follows = "b"; }; + b.file = { path = "/fixtures/alt"; inputs.foo.follows = "a"; }; + }; + expect = "eval-error"; + }]; + }) + (s "self" { + phases = [{ + sources.a.file = { path = "/fixtures/alt"; inputs.foo.follows = "a"; }; + expect = "eval-error"; + }]; + }) + (s "missing" { + phases = [{ + sources.a.file = { path = "/fixtures/alt"; inputs.foo.follows = "nope"; }; + expect = "eval-error"; + }]; + }) + (s "noflake" { + phases = [{ + sources = alt // { + x.file = { path = "/fixtures/noflake"; inputs.foo.follows = "alt"; }; + }; + expect.fail = "error: source x draws flake inputs from other sources, but contains no flake.nix"; + }]; + }) + (s "noinput" { + phases = [{ + sources.x.flake-input = { flake = "/fixtures/top"; name = "nope"; inherit fetchOnTarget; }; + expect.fail = "error: flake /fixtures/top has no input nope"; + }]; + }) + # An input locked to a path the target doesn't have. + (s "gone" { + phases = [{ + sources.x.flake = { flake = "/fixtures/usegone"; fetchOnTarget = true; }; + expect.fail = "error: flake /fixtures/usegone: input gone refers to /fixtures/gone, which does not exist on the target"; + }]; + }) + ]) targetKinds; + +} diff --git a/tests/vm.nix b/tests/vm.nix new file mode 100644 index 0000000..cb6385d --- /dev/null +++ b/tests/vm.nix @@ -0,0 +1,142 @@ +# Runs scenarios in a NixOS VM test with two machines: deployer, which runs +# populate, and target, which it deploys to via ssh. Local scenarios deploy +# to deployer itself. +{ pkgs, name, scenarios, gitHead }: + +let + inherit (pkgs) lib; + + sshKeys = import (pkgs.path + "/nixos/tests/ssh-keys.nix") pkgs; + + # The scenarios as JSON, with scripts as files. + plan = pkgs.writeText "krops-test-${name}.json" (builtins.toJSON (map + (scenario: scenario // { + prepare = pkgs.writeText "prepare" scenario.prepare; + phases = map + (phase: phase // { + check = pkgs.writeText "check" phase.check; + mutate = + if phase.mutate == null then null + else pkgs.writeText "mutate" phase.mutate; + }) + scenario.phases; + }) + scenarios)); + + common = { pkgs, ... }: { + # The plan refers to all scripts the scenarios run. Being part of the + # system, they survive the scenarios collecting garbage. + system.extraDependencies = [ plan "${./fixtures.sh}" ]; + nix.settings.substituters = lib.mkForce []; + environment.systemPackages = [ pkgs.git pkgs.rsync ]; + # Only warnings and errors on the console, which ends up in the log. + services.journald.extraConfig = lib.mkAfter "MaxLevelConsole=warning"; + boot.initrd.systemd.contents."/etc/systemd/journald.conf".text = + lib.mkAfter "MaxLevelConsole=warning"; + boot.consoleLogLevel = lib.mkForce 4; + # Neither probing BIOS disk information, nor iPXE option ROMs, which + # network boot would need, nor systemd-oomd, which warns about missing + # swap. + boot.kernelParams = lib.optional pkgs.stdenv.hostPlatform.isx86 "edd=off"; + virtualisation.qemu.options = [ "-global virtio-net-pci.romfile=" ]; + systemd.oomd.enable = false; + virtualisation.memorySize = 2048; + virtualisation.diskSize = 4096; + }; + +in pkgs.testers.runNixOSTest { + name = "krops-${name}"; + + globalTimeout = 4 * 60 * 60; + + nodes.deployer = { pkgs, ... }: { + imports = [ common ]; + environment.systemPackages = [ + pkgs.age pkgs.gnupg pkgs.pass pkgs.passage + ]; + networking.firewall.allowedTCPPorts = [ 8000 9418 ]; + programs.ssh.extraConfig = '' + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + LogLevel ERROR + ''; + systemd.services.fixtures-http = { + wantedBy = [ "multi-user.target" ]; + preStart = "mkdir -p /fixtures/www"; + serviceConfig.ExecStart = + "${pkgs.python3}/bin/python3 -m http.server 8000 --directory /fixtures/www"; + }; + systemd.services.fixtures-git = { + wantedBy = [ "multi-user.target" ]; + preStart = "mkdir -p /fixtures/git"; + serviceConfig.ExecStart = + "${pkgs.git}/bin/git daemon --reuseaddr --export-all --base-path=/fixtures/git"; + }; + }; + + nodes.target = { ... }: { + imports = [ common ]; + services.openssh.enable = true; + users.users.root.openssh.authorizedKeys.keys = [ sshKeys.snakeOilPublicKey ]; + users.users.deploy = { + isNormalUser = true; + extraGroups = [ "wheel" ]; + openssh.authorizedKeys.keys = [ sshKeys.snakeOilPublicKey ]; + }; + security.sudo.wheelNeedsPassword = false; + }; + + testScript = /* python */ '' + import json + + plan = json.load(open("${plan}")) + nodes = {"deployer": deployer, "target": target} + + + # Runs a command, logging its output only if it fails. + def run(node, command): + status, out = node.execute(f"{command} 2>&1") + if status != 0: + raise Exception(f"{command} fails:\n{out}") + + + start_all() + deployer.wait_for_unit("fixtures-http.service") + deployer.wait_for_unit("fixtures-git.service") + deployer.wait_for_open_port(8000) + deployer.wait_for_open_port(9418) + target.wait_for_unit("sshd.service") + target.wait_for_open_port(22) + + with subtest("setup"): + run(deployer, "install -Dm600 ${sshKeys.snakeOilPrivateKey} /root/.ssh/id_ecdsa") + run(deployer, "ssh root@target true") + run(deployer, "ssh deploy@target sudo true") + run(deployer, "HOME=/root sh ${./fixtures.sh} http://deployer:8000 ${gitHead}") + + failed = [] + for scenario in plan: + node = nodes[scenario["node"]] + try: + with subtest(scenario["name"]): + run(node, f"sh {scenario['prepare']}") + for i, phase in enumerate(scenario["phases"]): + status, out = deployer.execute(f"HOME=/root {phase['populate']} 2>&1") + if phase["expect"] is None: + if status != 0: + raise Exception(f"populate {i} fails:\n{out}") + elif status == 0 or phase["expect"] not in out: + raise Exception( + f"populate {i} doesn't fail with {phase['expect']!r}:\n{out}" + ) + run(node, f"KROPS_TEST_GC=1 sh {phase['check']}") + if phase["mutate"] is not None: + run(node, f"sh {phase['mutate']}") + except Exception as e: + print(f"scenario {scenario['name']} fails: {e}") + failed.append(scenario["name"]) + + if failed: + raise Exception(f"{len(failed)} scenarios fail: {' '.join(failed)}") + ''; +} |