tests: add NixOS VM tests for populate

tests/default.nix provides one VM test per source type, and one for
features spanning sources.  Each test runs scenarios from
tests/scenarios.nix, which cover every combination of a source type's
options, deployed to the deploying machine itself, to a remote target as
root, and to a remote target using sudo.  A scenario prepares the target
directory, deploys, checks the result, and deploys and checks again.
Scenarios expecting populate to fail check its error message, or check
at evaluation time if populate is expected not to evaluate.

The deploying machine creates fixtures using tests/fixtures.sh and serves
them via HTTP and git daemon, so flakes can be fetched on the target
without further network access.  Checks evaluate delivered flakes without
network access to the fixtures after collecting garbage, so they verify
that the delivered store paths are complete and rooted.

6 files changed, 930 insertions, 0 deletions

diff --git a/README.md b/README.md
index fe28fca..79ace5b 100644
--- a/README.md
+++ b/README.md
@@ -516,6 +516,22 @@ flakes, as long as the inputs drawing from them are declared with
 `flake = false`.
 
 
+## Tests
+
+`tests/` contains NixOS VM tests for populate: one test per source type, and
+one for features spanning sources, such as flake inputs from sources.  Each
+test deploys every combination of a source type's options to the deploying
+machine itself, to a remote target as `root`, and to a remote target using
+`sudo`, and checks the result after deploying and after deploying again.
+The tests require KVM and nixpkgs with `pkgs.testers.runNixOSTest`
+(23.05 or later), and use `<nixpkgs>` by default.
+
+```sh
+nix-build tests            # all tests
+nix-build tests -A flake   # the test for one source type
+```
+
+
 ## References
 
 - [In-depth example](http://tech.ingolf-wagner.de/nixos/krops/) by [Ingolf Wagner](https://ingolf-wagner.de/)
diff --git a/tests/default.nix b/tests/default.nix
new file mode 100644
index 0000000..e9ea769
--- /dev/null
+++ b/tests/default.nix
@@ -0,0 +1,24 @@
+# NixOS VM tests for populate, one test per source type, plus one for
+# features spanning sources.  Build all with nix-build tests, or one with,
+# e.g., nix-build tests -A flake.
+{ pkgs ? import ../pkgs {} }:
+
+let
+  # HEAD of /fixtures/gitsrc, which fixtures.sh creates deterministically.
+  # fixtures.sh fails if it differs, naming the actual one.
+  gitHead = "e42d44ef9cdb76ba5a84dff4c89c38222703bf0c";
+
+  scenarios = import ./scenarios.nix {
+    inherit (pkgs) lib;
+    krops = import ../lib;
+    inherit (pkgs) populate;
+    url = "http://deployer:8000";
+    gitUrl = "git://deployer/gitsrc.git";
+    remoteHost = "target";
+    agePath = "${pkgs.age}/bin/age";
+    inherit gitHead;
+  };
+in
+  builtins.mapAttrs
+    (name: scenarios: import ./vm.nix { inherit pkgs name scenarios gitHead; })
+    scenarios
diff --git a/tests/fixtures.sh b/tests/fixtures.sh
new file mode 100755
index 0000000..eb62a61
--- /dev/null
+++ b/tests/fixtures.sh
@@ -0,0 +1,178 @@
+#! /bin/sh
+# Creates the fixtures the tests deploy from in /fixtures.
+#
+# usage: fixtures.sh URL GIT_HEAD
+#
+# URL is where /fixtures/www gets served via HTTP, e.g. http://deployer:8000.
+# The tarball flakes there get locked against it, so it must be served while
+# this script runs.  /fixtures/git gets served by git daemon.  GIT_HEAD is the
+# commit the tests expect as HEAD of /fixtures/gitsrc.
+set -efu
+
+url=$1
+git_head=$2
+
+nix() {
+  command nix --extra-experimental-features 'nix-command flakes' "$@"
+}
+
+export GIT_AUTHOR_NAME=krops GIT_AUTHOR_EMAIL=spam@krebsco.de
+export GIT_COMMITTER_NAME=krops GIT_COMMITTER_EMAIL=spam@krebsco.de
+export GIT_AUTHOR_DATE='1000000000 +0000' GIT_COMMITTER_DATE='1000000000 +0000'
+# Configuration such as commit.gpgsign would change the commits.
+export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1
+
+# Commits everything in the current directory.  Commits are deterministic, so
+# the tests can refer to them by hash.
+commit() {
+  git add -A
+  git commit -q -m "$1"
+}
+
+tarball() {
+  tar -C /fixtures/www/src --sort=name --mtime=@1 --owner=0 --group=0 \
+      --numeric-owner -czf /fixtures/www/"$1".tar.gz "$1"
+}
+
+rm -fR /fixtures
+mkdir -p /fixtures/www/src
+
+# Tarball flakes, fetched via HTTP: inner, dep (with input inner), mono (with
+# a flake in subdirectory sub), and remote (with input dep).
+mkdir /fixtures/www/src/inner
+cat > /fixtures/www/src/inner/flake.nix <<'NIX'
+{ outputs = _: { who = "inner"; }; }
+NIX
+tarball inner
+
+mkdir /fixtures/www/src/dep
+cat > /fixtures/www/src/dep/flake.nix <<NIX
+{
+  inputs.inner.url = "$url/inner.tar.gz";
+  outputs = { inner, ... }: { greeting = "dep+\${inner.who}"; };
+}
+NIX
+(cd /fixtures/www/src/dep && nix flake lock)
+tarball dep
+
+mkdir -p /fixtures/www/src/mono/sub
+echo mono > /fixtures/www/src/mono/README
+cat > /fixtures/www/src/mono/sub/flake.nix <<'NIX'
+{ outputs = _: { who = "sub"; }; }
+NIX
+tarball mono
+
+mkdir /fixtures/www/src/remote
+cat > /fixtures/www/src/remote/flake.nix <<NIX
+{
+  inputs.dep.url = "$url/dep.tar.gz";
+  outputs = { dep, ... }: { msg = "remote:\${dep.greeting}"; };
+}
+NIX
+(cd /fixtures/www/src/remote && nix flake lock)
+tarball remote
+
+# top: a flake in a Git work tree, with inputs dep and sub (a flake in a
+# subdirectory).
+mkdir /fixtures/top
+cat > /fixtures/top/flake.nix <<NIX
+{
+  inputs.dep.url = "$url/dep.tar.gz";
+  inputs.sub.url = "$url/mono.tar.gz?dir=sub";
+  outputs = { dep, sub, ... }: { msg = "top:\${dep.greeting},\${sub.who}"; };
+}
+NIX
+(cd /fixtures/top && nix flake lock && git init -q && commit top)
+
+# mono: a Git work tree with a flake in subdirectory sub.
+cp -R /fixtures/www/src/mono /fixtures/mono
+(cd /fixtures/mono && git init -q && commit mono)
+
+# alt: a flake to draw inputs from.
+mkdir /fixtures/alt
+cat > /fixtures/alt/flake.nix <<'NIX'
+{ outputs = _: { greeting = "alt"; who = "alt"; }; }
+NIX
+
+# usegone: a flake with an input locked to a path that doesn't exist anymore.
+mkdir /fixtures/gone /fixtures/usegone
+cat > /fixtures/gone/flake.nix <<'NIX'
+{ outputs = _: {}; }
+NIX
+cat > /fixtures/usegone/flake.nix <<'NIX'
+{
+  inputs.gone.url = "path:/fixtures/gone";
+  outputs = _: {};
+}
+NIX
+(cd /fixtures/usegone && nix flake lock)
+rm -R /fixtures/gone
+
+# noflake: a directory without flake.nix.
+mkdir /fixtures/noflake
+echo noflake > /fixtures/noflake/README
+
+# filesrc: a copy of top, with files for testing the file source type's
+# options.  same has the same size and modification time as the file the
+# tests put on the target beforehand, while skip-ex and skip-f have the same
+# size but an older modification time than theirs, so rsync's quick check
+# tells them apart.
+mkdir /fixtures/filesrc
+cp /fixtures/top/flake.nix /fixtures/top/flake.lock /fixtures/filesrc
+echo keep > /fixtures/filesrc/keep
+echo src > /fixtures/filesrc/skip-ex
+echo src > /fixtures/filesrc/skip-f
+echo AAAA > /fixtures/filesrc/same
+touch -d @1000000000 /fixtures/filesrc/same
+touch -d @999999999 /fixtures/filesrc/skip-ex /fixtures/filesrc/skip-f
+
+# gitsrc: a Git repository with two commits, served by git daemon.  Its flake
+# has an input that only exists when drawn from a source.
+mkdir /fixtures/gitsrc
+cat > /fixtures/gitsrc/flake.nix <<'NIX'
+{
+  inputs.dep.url = "path:/nonexistent";
+  outputs = { dep, ... }: { msg = "git:${dep.greeting}"; };
+}
+NIX
+(cd /fixtures/gitsrc && git init -q -b master && commit one)
+echo two > /fixtures/gitsrc/two
+(cd /fixtures/gitsrc && commit two)
+if test "$(git -C /fixtures/gitsrc rev-parse HEAD)" != "$git_head"; then
+  >&2 printf 'error: HEAD of /fixtures/gitsrc is %s, but the tests expect %s\n' \
+      "$(git -C /fixtures/gitsrc rev-parse HEAD)" "$git_head"
+  exit 1
+fi
+mkdir /fixtures/git
+git clone -q --bare /fixtures/gitsrc /fixtures/git/gitsrc.git
+git -C /fixtures/git/gitsrc.git config uploadpack.allowReachableSHA1InWant true
+
+# pass: password stores with x/a, without and with Git.
+if ! gpg --list-secret-keys spam@krebsco.de >/dev/null 2>&1; then
+  gpg --batch --passphrase '' --quick-gen-key spam@krebsco.de default default never
+fi
+for store in pass pass-git; do
+  export PASSWORD_STORE_DIR=/fixtures/$store
+  pass init spam@krebsco.de >/dev/null
+  if test $store = pass-git; then
+    pass git init >/dev/null
+  fi
+  printf 'secret-a\n' | pass insert -m x/a >/dev/null
+done
+unset PASSWORD_STORE_DIR
+
+# passage: a passage store with x/a.
+mkdir -p /fixtures/passage/store
+age-keygen -o /fixtures/passage/identities 2>/dev/null
+age-keygen -y /fixtures/passage/identities > /fixtures/passage/store/.age-recipients
+printf 'secret-a\n' |
+  PASSAGE_DIR=/fixtures/passage/store \
+  PASSAGE_IDENTITIES_FILE=/fixtures/passage/identities \
+  passage insert -m x/a >/dev/null
+
+# pipe: a command for the pipe source type.
+cat > /fixtures/pipe <<'SH'
+#! /bin/sh
+echo piped
+SH
+chmod +x /fixtures/pipe
diff --git a/tests/lib.sh b/tests/lib.sh
new file mode 100644
index 0000000..13b1ded
--- /dev/null
+++ b/tests/lib.sh
@@ -0,0 +1,43 @@
+# Helpers for preparing and checking scenarios, sourced on the machine
+# populate deploys to.
+set -efu
+
+fail() {
+  printf 'FAIL: %s\n' "$*" >&2
+  exit 1
+}
+
+# usage: expect_eq WHAT ACTUAL EXPECTED
+expect_eq() {
+  if test "$2" != "$3"; then
+    fail "$1: expected '$3', got '$2'"
+  fi
+}
+
+nix() {
+  command nix --extra-experimental-features 'nix-command flakes' "$@"
+}
+
+# Evaluates a flake output without network access to the fixtures, so all
+# inputs must be in the store.
+nix_eval() {
+  env \
+      http_proxy=http://127.0.0.1:9 https_proxy=http://127.0.0.1:9 \
+      HTTP_PROXY=http://127.0.0.1:9 HTTPS_PROXY=http://127.0.0.1:9 \
+      all_proxy=http://127.0.0.1:9 ALL_PROXY=http://127.0.0.1:9 \
+      no_proxy= NO_PROXY= \
+    nix --extra-experimental-features 'nix-command flakes' eval --raw "$@"
+}
+
+# Evaluates a flake output, fetching missing inputs.
+nix_eval_online() {
+  nix eval --raw "$@"
+}
+
+# Collects garbage if KROPS_TEST_GC is set, so evaluating afterwards checks
+# that delivered store paths are registered as garbage collector roots.
+gc() {
+  if test -n "${KROPS_TEST_GC-}"; then
+    nix-store --gc >/dev/null
+  fi
+}
diff --git a/tests/scenarios.nix b/tests/scenarios.nix
new file mode 100644
index 0000000..22431a2
--- /dev/null
+++ b/tests/scenarios.nix
@@ -0,0 +1,527 @@
+# Test scenarios for populate: every combination of each source type's
+# options, deployed to the local machine, to a remote target as root, and to
+# a remote target using sudo, plus scenarios for features spanning sources.
+#
+# A scenario prepares the target directory, then runs its phases.  Each phase
+# runs populate, expecting it to succeed or to fail with a message, and checks
+# the result on the target.  Phases expecting evaluation to fail get checked
+# here and don't run.
+{
+  # nixpkgs' lib.
+  lib,
+  # krops' lib.
+  krops,
+  # Function from populate's arguments to the command running populate.
+  populate,
+  # Where the deploying machine serves /fixtures/www via HTTP.
+  url,
+  # URL of the Git repository /fixtures/git/gitsrc.git.
+  gitUrl,
+  # Host name of the remote target.
+  remoteHost,
+  # Absolute path of age on the deploying machine.
+  agePath,
+  # HEAD of /fixtures/gitsrc.
+  gitHead,
+}:
+
+let
+  inherit (lib)
+    attrNames concatMap concatStringsSep foldl' isBool mapAttrsToList
+    optional optionalAttrs optionalString;
+
+  prelude = ". ${./lib.sh}\n";
+
+  cartesian = attrs: foldl'
+    (acc: name: concatMap
+      (combination: map
+        (value: combination // { ${name} = value; })
+        attrs.${name})
+      acc)
+    [{}]
+    (attrNames attrs);
+
+  flagsName = flags: concatStringsSep "-" (mapAttrsToList
+    (name: value: name + (if isBool value then (if value then "1" else "0") else value))
+    flags);
+
+  targets = {
+    local = path: krops.mkTarget "" // { host = "localhost"; inherit path; };
+    remote = path: {
+      user = "root"; host = remoteHost; port = null; inherit path;
+      sudo = false; extraOptions = [];
+    };
+    sudo = path: {
+      user = "deploy"; host = remoteHost; port = null; inherit path;
+      sudo = true; extraOptions = [];
+    };
+  };
+
+  alt = { alt.file = "/fixtures/alt"; };
+
+  # Builds a scenario from a name, a target kind, and phases with sources.
+  scenario = { name, targetKind, prepare ? "", phases }: let
+    path = "/var/krops/${name}";
+    target = targets.${targetKind} path;
+    vars = "T=${path}\n";
+    phase = { sources, expect ? "success", check ? "", mutate ? null }: let
+      command = populate {
+        source = krops.evalSource [ sources ];
+        inherit target;
+        force = true;
+      };
+      evaluated = builtins.tryEval (builtins.seq (toString command) true);
+    in
+      if expect == "eval-error" then
+        if evaluated.success then
+          throw "scenario ${name}: populate evaluates, but is expected to fail"
+        else
+          null
+      else {
+        populate = toString command;
+        expect = if expect == "success" then null else expect.fail;
+        check = prelude + vars + check;
+        mutate = if mutate == null then null else prelude + vars + mutate;
+      };
+  in {
+    inherit name;
+    node = if targetKind == "local" then "deployer" else "target";
+    prepare = prelude + vars + ''
+      rm -fR "$T"
+    '' + prepare;
+    phases = lib.filter (x: x != null) (map phase phases);
+  };
+
+  # Same sources twice, so the second phase checks redeploying.
+  twice = phase: [ phase phase ];
+
+  targetKinds = [ "local" "remote" "sudo" ];
+
+  # Scenarios for all combinations of options and target kinds.  f gets the
+  # options and the scenario's name, target kind, and target directory, and
+  # returns the scenario's arguments, or null for invalid combinations.
+  matrix = type: options: f:
+    concatMap
+      (targetKind: lib.filter (x: x != null) (map
+        (flags: let
+          name = "${type}-${targetKind}" +
+            optionalString (flags != {}) "-${flagsName flags}";
+          args = f flags { inherit name targetKind; path = "/var/krops/${name}"; };
+        in
+          if args == null then null
+          else scenario ({ inherit name targetKind; } // args))
+        (cartesian options)))
+      targetKinds;
+
+  # Checks that the source at $T/x is a link to a store path, or to the
+  # subdirectory sub therein, and evaluates the attribute attr of the flake
+  # there, through the link, or of its generated flake, after collecting
+  # garbage.
+  checkFlake = { attr, expected, subdir ? false, generated ? false }: ''
+    link=$(readlink "$T"/x)
+    ${if subdir then ''
+      expect_eq subdirectory "$(basename "$link")" sub
+      link=$(dirname "$link")
+    '' else ""}
+    expect_eq "store directory" "$(dirname "$link")" /nix/store
+    test ! -e "$T"/.populate/flake-uploads || fail "upload directory remains"
+    gc
+    expect_eq ${attr} "$(nix_eval ${
+      if generated then ''"$T"/.populate/flakes/x'' else ''"$T"/x''
+    }#${attr})" ${lib.escapeShellArg expected}
+  '';
+
+  copyingWithSudo = flags: ctx:
+    ctx.targetKind == "sudo" && !(flags.fetchOnTarget or false);
+
+in {
+
+  derivation = matrix "derivation" {} (_: _: {
+    phases = twice {
+      sources.x.derivation = ''
+        derivation {
+          name = "krops-test";
+          system = builtins.currentSystem;
+          builder = "/bin/sh";
+          args = [ "-c" "echo built > $out" ];
+        }
+      '';
+      check = ''
+        gc
+        expect_eq x "$(cat "$T"/x)" built
+      '';
+    };
+  });
+
+  file = let
+    f = flags: _: {
+      prepare = ''
+        mkdir -p "$T"/x
+        echo old > "$T"/x/skip-ex
+        echo old > "$T"/x/skip-f
+        echo stale > "$T"/x/stale
+        echo BBBB > "$T"/x/same
+        touch -d @1000000000 "$T"/x/same
+      '';
+      phases = twice {
+        sources = {
+          x.file = {
+            path = "/fixtures/filesrc";
+            useChecksum = flags.checksum;
+            inherit (flags) deleteExcluded;
+            exclude = optional (flags.exclude or false) "skip-ex";
+            filters = optional flags.filters { type = "exclude"; pattern = "skip-f"; };
+            inputs = optionalAttrs flags.inputs { dep.follows = "alt"; };
+          };
+        } // optionalAttrs flags.inputs alt;
+        check = let
+          excluded = file: set: ''
+            ${if !set then ''
+              expect_eq ${file} "$(cat "$T"/x/${file})" src
+            '' else if flags.deleteExcluded then ''
+              test ! -e "$T"/x/${file} || fail "${file} not deleted"
+            '' else ''
+              expect_eq ${file} "$(cat "$T"/x/${file})" old
+            ''}
+          '';
+        in ''
+          expect_eq keep "$(cat "$T"/x/keep)" keep
+          ${excluded "skip-ex" (flags.exclude or false)}
+          ${excluded "skip-f" flags.filters}
+          ${if flags.deleteExcluded then ''
+            test ! -e "$T"/x/stale || fail "stale not deleted"
+          '' else ''
+            expect_eq stale "$(cat "$T"/x/stale)" stale
+          ''}
+          expect_eq same "$(cat "$T"/x/same)" ${if flags.checksum then "AAAA" else "BBBB"}
+        '' + optionalString flags.inputs ''
+          # Inputs not drawn from sources aren't delivered, so this needs the
+          # network and can't collect garbage before.
+          expect_eq msg "$(nix_eval_online "$T"/.populate/flakes/x#msg)" top:alt,sub
+        '';
+      };
+    };
+  in
+    matrix "file" {
+      checksum = [ false true ];
+      deleteExcluded = [ false true ];
+      filters = [ false true ];
+      inputs = [ false true ];
+    } f
+    ++ [
+      # The deprecated exclude option, once, because it warns.
+      (scenario (let
+        args = f {
+          checksum = false;
+          deleteExcluded = true;
+          exclude = true;
+          filters = false;
+          inputs = false;
+        } null;
+      in args // {
+        name = "file-local-exclude";
+        targetKind = "local";
+        phases = [ (builtins.head args.phases) ];
+      }))
+    ];
+
+  flake = matrix "flake" {
+    kind = [ "git" "path" "subdir" "remote" ];
+    fetchOnTarget = [ false true ];
+    inputs = [ false true ];
+  } (flags: ctx: let
+    flake = {
+      git = "/fixtures/top";
+      path = "path:/fixtures/top";
+      subdir = "git+file:/fixtures/mono?dir=sub";
+      remote = "${url}/remote.tar.gz";
+    }.${flags.kind};
+    msg = {
+      git = "top:${if flags.inputs then "alt" else "dep+inner"},sub";
+      path = msg.git;
+      remote = "remote:${if flags.inputs then "alt" else "dep+inner"}";
+    };
+  in
+    # The flake in a subdirectory has no inputs to draw from sources.
+    if flags.kind == "subdir" && flags.inputs then null else {
+    phases = twice ({
+      sources = {
+        x.flake = {
+          inherit flake;
+          inherit (flags) fetchOnTarget;
+          inputs = optionalAttrs flags.inputs { dep.follows = "alt"; };
+        };
+      } // optionalAttrs flags.inputs alt;
+    } // (if copyingWithSudo flags ctx then {
+      expect = "eval-error";
+    } else {
+      check = if flags.kind == "subdir" then checkFlake {
+        attr = "who"; expected = "sub"; subdir = true;
+      } else checkFlake {
+        attr = "msg"; expected = msg.${flags.kind}; generated = flags.inputs;
+      };
+    }));
+  });
+
+  flake-input = concatMap
+    (name: matrix "flake-input-${lib.replaceStrings ["/"] ["."] name}" ({
+      fetchOnTarget = [ false true ];
+    } // optionalAttrs (name == "dep") {
+      inputs = [ false true ];
+    }) (flags: ctx: let
+      inputs = flags.inputs or false;
+    in {
+      phases = twice ({
+        sources = {
+          x.flake-input = {
+            flake = "/fixtures/top";
+            inherit name;
+            inherit (flags) fetchOnTarget;
+            inputs = optionalAttrs inputs { inner.follows = "alt"; };
+          };
+        } // optionalAttrs inputs alt;
+      } // (if copyingWithSudo flags ctx then {
+        expect = "eval-error";
+      } else {
+        check = {
+          dep = checkFlake {
+            attr = "greeting";
+            expected = if inputs then "dep+alt" else "dep+inner";
+            generated = inputs;
+          };
+          "dep/inner" = checkFlake { attr = "who"; expected = "inner"; };
+          sub = checkFlake { attr = "who"; expected = "sub"; subdir = true; };
+        }.${name};
+      }));
+    }))
+    [ "dep" "dep/inner" "sub" ];
+
+  git = matrix "git" {
+    shallow = [ false true ];
+    fetchAlways = [ false true ];
+    cleanExclude = [ false true ];
+    inputs = [ false true ];
+  } (flags: _: let
+    check = ''
+      expect_eq HEAD "$(git -C "$T"/x rev-parse HEAD)" ${gitHead}
+      expect_eq commits "$(git -C "$T"/x rev-list --count HEAD)" ${
+        if flags.shallow then "1" else "2"
+      }
+      test -e "$T"/x/two || fail "two missing"
+    '' + optionalString flags.inputs ''
+      gc
+      expect_eq msg "$(nix_eval "$T"/.populate/flakes/x#msg)" git:alt
+    '';
+    sources = {
+      x.git = {
+        url = gitUrl;
+        ref = gitHead;
+        inherit (flags) shallow fetchAlways;
+        clean.exclude = optional flags.cleanExclude "keepme";
+        inputs = optionalAttrs flags.inputs { dep.follows = "alt"; };
+      };
+    } // optionalAttrs flags.inputs alt;
+  in {
+    phases = [
+      {
+        inherit sources check;
+        mutate = ''
+          echo junk > "$T"/x/junk
+          echo keepme > "$T"/x/keepme
+        '';
+      }
+      {
+        inherit sources;
+        check = check + ''
+          test ! -e "$T"/x/junk || fail "junk not cleaned"
+          ${if flags.cleanExclude then ''
+            test -e "$T"/x/keepme || fail "keepme cleaned"
+          '' else ''
+            test ! -e "$T"/x/keepme || fail "keepme not cleaned"
+          ''}
+        '';
+      }
+    ];
+  });
+
+  pass = matrix "pass" {
+    store = [ "pass" "pass-git" ];
+  } (flags: _: {
+    phases = twice {
+      sources.x.pass = { dir = "/fixtures/${flags.store}"; name = "x"; };
+      check = ''
+        expect_eq a "$(cat "$T"/x/a)" secret-a
+      '';
+    };
+  });
+
+  passage = matrix "passage" {
+    age = [ "default" "path" ];
+    dir = [ "store" "subdir" ];
+  } (flags: _: {
+    phases = twice {
+      sources.x.passage = {
+        dir = "/fixtures/passage/store" + optionalString (flags.dir == "subdir") "/x";
+        identities_file = "/fixtures/passage/identities";
+      } // optionalAttrs (flags.age == "path") {
+        age = agePath;
+      };
+      check = ''
+        expect_eq a "$(cat "$T"/x/${optionalString (flags.dir == "store") "x/"}a)" secret-a
+      '';
+    };
+  });
+
+  pipe = matrix "pipe" {} (_: _: {
+    phases = twice {
+      sources.x.pipe = "/fixtures/pipe";
+      check = ''
+        expect_eq x "$(cat "$T"/x)" piped
+      '';
+    };
+  });
+
+  symlink = matrix "symlink" {
+    absolute = [ false true ];
+    inputs = [ false true ];
+  } (flags: ctx: let
+    target = optionalString flags.absolute "${ctx.path}/" + "topcopy";
+  in {
+    phases = twice {
+      sources = {
+        x.symlink = {
+          inherit target;
+          inputs = optionalAttrs flags.inputs { dep.follows = "alt"; };
+        };
+        topcopy.file = "/fixtures/filesrc";
+      } // optionalAttrs flags.inputs alt;
+      check = ''
+        expect_eq link "$(readlink "$T"/x)" ${target}
+        test -e "$T"/x/flake.nix || fail "flake.nix missing"
+      '' + optionalString flags.inputs ''
+        expect_eq msg "$(nix_eval_online "$T"/.populate/flakes/x#msg)" top:alt,sub
+      '';
+    };
+  });
+
+  # Features spanning several sources.
+  features = concatMap (targetKind: let
+    s = name: args: scenario ({ name = "${name}-${targetKind}"; inherit targetKind; } // args);
+    copying = targetKind != "sudo";
+    fetchOnTarget = !copying;
+  in [
+    # A generated flake draws an input from a source with a generated flake.
+    (s "chain" {
+      phases = twice {
+        sources = alt // {
+          a.flake = { flake = "/fixtures/top"; inherit fetchOnTarget; inputs.dep.follows = "b"; };
+          b.flake-input = {
+            flake = "/fixtures/top"; name = "dep"; inherit fetchOnTarget;
+            inputs.inner.follows = "alt";
+          };
+        };
+        check = ''
+          gc
+          expect_eq msg "$(nix_eval "$T"/.populate/flakes/a#msg)" top:dep+alt,sub
+        '';
+      };
+    })
+    # An input of an input drawn from a source gets overridden at that source.
+    (s "nested" {
+      phases = twice {
+        sources = alt // {
+          x.flake = {
+            flake = "/fixtures/top"; inherit fetchOnTarget;
+            inputs.dep = { follows = "b"; inputs.inner.follows = "alt"; };
+          };
+          b.flake-input = { flake = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; };
+        };
+        check = ''
+          gc
+          expect_eq msg "$(nix_eval "$T"/.populate/flakes/x#msg)" top:dep+alt,sub
+        '';
+      };
+    })
+    # A generated flake gets removed once its source draws no inputs anymore.
+    (s "removal" {
+      phases = [
+        {
+          sources = alt // {
+            x.flake = { flake = "/fixtures/top"; inherit fetchOnTarget; inputs.dep.follows = "alt"; };
+          };
+          check = ''
+            test -e "$T"/.populate/flakes/x/flake.lock || fail "generated flake missing"
+          '';
+        }
+        {
+          sources = alt // {
+            x.flake = { flake = "/fixtures/top"; inherit fetchOnTarget; };
+          };
+          check = ''
+            test ! -e "$T"/.populate/flakes/x || fail "generated flake remains"
+          '';
+        }
+      ];
+    })
+    # Roots follow the delivered input.
+    (s "retarget" {
+      phases = [
+        {
+          sources.x.flake-input = { flake = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; };
+          check = ''
+            expect_eq roots "$(ls "$T"/.populate/flake-inputs/x | wc -l)" 1
+          '';
+        }
+        {
+          sources.x.flake-input = { flake = "/fixtures/top"; name = "sub"; inherit fetchOnTarget; };
+          check = ''
+            test ! -e "$T"/.populate/flake-inputs/x || fail "roots remain"
+          '' + checkFlake { attr = "who"; expected = "sub"; subdir = true; };
+        }
+      ];
+    })
+    (s "cycle" {
+      phases = [{
+        sources = {
+          a.file = { path = "/fixtures/alt"; inputs.foo.follows = "b"; };
+          b.file = { path = "/fixtures/alt"; inputs.foo.follows = "a"; };
+        };
+        expect = "eval-error";
+      }];
+    })
+    (s "self" {
+      phases = [{
+        sources.a.file = { path = "/fixtures/alt"; inputs.foo.follows = "a"; };
+        expect = "eval-error";
+      }];
+    })
+    (s "missing" {
+      phases = [{
+        sources.a.file = { path = "/fixtures/alt"; inputs.foo.follows = "nope"; };
+        expect = "eval-error";
+      }];
+    })
+    (s "noflake" {
+      phases = [{
+        sources = alt // {
+          x.file = { path = "/fixtures/noflake"; inputs.foo.follows = "alt"; };
+        };
+        expect.fail = "error: source x draws flake inputs from other sources, but contains no flake.nix";
+      }];
+    })
+    (s "noinput" {
+      phases = [{
+        sources.x.flake-input = { flake = "/fixtures/top"; name = "nope"; inherit fetchOnTarget; };
+        expect.fail = "error: flake /fixtures/top has no input nope";
+      }];
+    })
+    # An input locked to a path the target doesn't have.
+    (s "gone" {
+      phases = [{
+        sources.x.flake = { flake = "/fixtures/usegone"; fetchOnTarget = true; };
+        expect.fail = "error: flake /fixtures/usegone: input gone refers to /fixtures/gone, which does not exist on the target";
+      }];
+    })
+  ]) targetKinds;
+
+}
diff --git a/tests/vm.nix b/tests/vm.nix
new file mode 100644
index 0000000..cb6385d
--- /dev/null
+++ b/tests/vm.nix
@@ -0,0 +1,142 @@
+# Runs scenarios in a NixOS VM test with two machines: deployer, which runs
+# populate, and target, which it deploys to via ssh.  Local scenarios deploy
+# to deployer itself.
+{ pkgs, name, scenarios, gitHead }:
+
+let
+  inherit (pkgs) lib;
+
+  sshKeys = import (pkgs.path + "/nixos/tests/ssh-keys.nix") pkgs;
+
+  # The scenarios as JSON, with scripts as files.
+  plan = pkgs.writeText "krops-test-${name}.json" (builtins.toJSON (map
+    (scenario: scenario // {
+      prepare = pkgs.writeText "prepare" scenario.prepare;
+      phases = map
+        (phase: phase // {
+          check = pkgs.writeText "check" phase.check;
+          mutate =
+            if phase.mutate == null then null
+            else pkgs.writeText "mutate" phase.mutate;
+        })
+        scenario.phases;
+    })
+    scenarios));
+
+  common = { pkgs, ... }: {
+    # The plan refers to all scripts the scenarios run.  Being part of the
+    # system, they survive the scenarios collecting garbage.
+    system.extraDependencies = [ plan "${./fixtures.sh}" ];
+    nix.settings.substituters = lib.mkForce [];
+    environment.systemPackages = [ pkgs.git pkgs.rsync ];
+    # Only warnings and errors on the console, which ends up in the log.
+    services.journald.extraConfig = lib.mkAfter "MaxLevelConsole=warning";
+    boot.initrd.systemd.contents."/etc/systemd/journald.conf".text =
+      lib.mkAfter "MaxLevelConsole=warning";
+    boot.consoleLogLevel = lib.mkForce 4;
+    # Neither probing BIOS disk information, nor iPXE option ROMs, which
+    # network boot would need, nor systemd-oomd, which warns about missing
+    # swap.
+    boot.kernelParams = lib.optional pkgs.stdenv.hostPlatform.isx86 "edd=off";
+    virtualisation.qemu.options = [ "-global virtio-net-pci.romfile=" ];
+    systemd.oomd.enable = false;
+    virtualisation.memorySize = 2048;
+    virtualisation.diskSize = 4096;
+  };
+
+in pkgs.testers.runNixOSTest {
+  name = "krops-${name}";
+
+  globalTimeout = 4 * 60 * 60;
+
+  nodes.deployer = { pkgs, ... }: {
+    imports = [ common ];
+    environment.systemPackages = [
+      pkgs.age pkgs.gnupg pkgs.pass pkgs.passage
+    ];
+    networking.firewall.allowedTCPPorts = [ 8000 9418 ];
+    programs.ssh.extraConfig = ''
+      StrictHostKeyChecking no
+      UserKnownHostsFile /dev/null
+      LogLevel ERROR
+    '';
+    systemd.services.fixtures-http = {
+      wantedBy = [ "multi-user.target" ];
+      preStart = "mkdir -p /fixtures/www";
+      serviceConfig.ExecStart =
+        "${pkgs.python3}/bin/python3 -m http.server 8000 --directory /fixtures/www";
+    };
+    systemd.services.fixtures-git = {
+      wantedBy = [ "multi-user.target" ];
+      preStart = "mkdir -p /fixtures/git";
+      serviceConfig.ExecStart =
+        "${pkgs.git}/bin/git daemon --reuseaddr --export-all --base-path=/fixtures/git";
+    };
+  };
+
+  nodes.target = { ... }: {
+    imports = [ common ];
+    services.openssh.enable = true;
+    users.users.root.openssh.authorizedKeys.keys = [ sshKeys.snakeOilPublicKey ];
+    users.users.deploy = {
+      isNormalUser = true;
+      extraGroups = [ "wheel" ];
+      openssh.authorizedKeys.keys = [ sshKeys.snakeOilPublicKey ];
+    };
+    security.sudo.wheelNeedsPassword = false;
+  };
+
+  testScript = /* python */ ''
+    import json
+
+    plan = json.load(open("${plan}"))
+    nodes = {"deployer": deployer, "target": target}
+
+
+    # Runs a command, logging its output only if it fails.
+    def run(node, command):
+        status, out = node.execute(f"{command} 2>&1")
+        if status != 0:
+            raise Exception(f"{command} fails:\n{out}")
+
+
+    start_all()
+    deployer.wait_for_unit("fixtures-http.service")
+    deployer.wait_for_unit("fixtures-git.service")
+    deployer.wait_for_open_port(8000)
+    deployer.wait_for_open_port(9418)
+    target.wait_for_unit("sshd.service")
+    target.wait_for_open_port(22)
+
+    with subtest("setup"):
+        run(deployer, "install -Dm600 ${sshKeys.snakeOilPrivateKey} /root/.ssh/id_ecdsa")
+        run(deployer, "ssh root@target true")
+        run(deployer, "ssh deploy@target sudo true")
+        run(deployer, "HOME=/root sh ${./fixtures.sh} http://deployer:8000 ${gitHead}")
+
+    failed = []
+    for scenario in plan:
+        node = nodes[scenario["node"]]
+        try:
+            with subtest(scenario["name"]):
+                run(node, f"sh {scenario['prepare']}")
+                for i, phase in enumerate(scenario["phases"]):
+                    status, out = deployer.execute(f"HOME=/root {phase['populate']} 2>&1")
+                    if phase["expect"] is None:
+                        if status != 0:
+                            raise Exception(f"populate {i} fails:\n{out}")
+                    elif status == 0 or phase["expect"] not in out:
+                        raise Exception(
+                            f"populate {i} doesn't fail with {phase['expect']!r}:\n{out}"
+                        )
+                    run(node, f"KROPS_TEST_GC=1 sh {phase['check']}")
+                    if phase["mutate"] is not None:
+                        run(node, f"sh {phase['mutate']}")
+        except Exception as e:
+            print(f"scenario {scenario['name']} fails: {e}")
+            failed.append(scenario["name"])
+
+    if failed:
+        raise Exception(f"{len(failed)} scenarios fail: {' '.join(failed)}")
+  '';
+}