From 1c7621261c4e9df8d572aef5eb6369d20eea23aa Mon Sep 17 00:00:00 2001
From: tv <tv@krebsco.de>
Date: Thu, 12 Nov 2026 02:07:10 +0000
Subject: tests: isolate flake scenarios, check eval errors against controls

Several weaknesses let checks pass although the feature under test is
broken:

- Nix restores fetched inputs from its fetcher cache, so evaluating a
  delivered flake offline after collecting garbage succeeds even if
  populate rooted none of its inputs.  nix_eval uses an empty cache
  directory.
- Scenarios share target directories and stores, so store paths and roots
  of earlier scenarios stand in for missing ones, and uploads of flakes
  never run in a passing scenario.  Scenarios of the flake, flake-input,
  and features tests are isolated now: they start without target
  directories and with a collected store on both machines.
- Phases expecting evaluation to fail pass on any evaluation failure,
  e.g. a misspelt option.  Each such phase has a control now: sources
  differing only in what makes evaluation fail, which must evaluate.

Further, populate runs with a time limit, so a hang fails its scenario
instead of the whole test, phases can run steps on the deploying machine
before and after, and evaluating the scenarios fails if the remote
target counts as local for whoever evaluates them.
---
 tests/vm.nix | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

(limited to 'tests/vm.nix')

diff --git a/tests/vm.nix b/tests/vm.nix
index b46b015..78b861b 100644
--- a/tests/vm.nix
+++ b/tests/vm.nix
@@ -8,16 +8,19 @@ let
 
   sshKeys = import (pkgs.path + "/nixos/tests/ssh-keys.nix") pkgs;
 
+  script = name: text: if text == null then null else pkgs.writeText name text;
+
   # The scenarios as JSON, with scripts as files.
   plan = pkgs.writeText "krops-test-${name}.json" (builtins.toJSON (map
     (scenario: scenario // {
       prepare = pkgs.writeText "prepare" scenario.prepare;
+      prepareDeployer = script "prepare-deployer" scenario.prepareDeployer;
       phases = map
         (phase: phase // {
           check = pkgs.writeText "check" phase.check;
-          mutate =
-            if phase.mutate == null then null
-            else pkgs.writeText "mutate" phase.mutate;
+          mutate = script "mutate" phase.mutate;
+          checkDeployer = script "check-deployer" phase.checkDeployer;
+          mutateDeployer = script "mutate-deployer" phase.mutateDeployer;
         })
         scenario.phases;
     })
@@ -125,9 +128,18 @@ in pkgs.testers.runNixOSTest {
         node = nodes[scenario["node"]]
         try:
             with subtest(scenario["name"]):
+                if scenario["isolate"]:
+                    for machine in nodes.values():
+                        run(machine, "rm -fR /var/krops && nix-store --gc >/dev/null")
+                if scenario["prepareDeployer"] is not None:
+                    run(deployer, f"sh {scenario['prepareDeployer']}")
                 run(node, f"sh {scenario['prepare']}")
                 for i, phase in enumerate(scenario["phases"]):
-                    status, out = deployer.execute(f"HOME=/root {phase['populate']} 2>&1")
+                    # A hang fails the scenario, not the whole test.
+                    status, out = deployer.execute(
+                        f"HOME=/root timeout 600 {phase['populate']} 2>&1",
+                        timeout=900,
+                    )
                     if phase["expect"] is None:
                         if status != 0:
                             raise Exception(f"populate {i} fails:\n{out}")
@@ -136,8 +148,12 @@ in pkgs.testers.runNixOSTest {
                             f"populate {i} doesn't fail with {phase['expect']!r}:\n{out}"
                         )
                     run(node, f"KROPS_TEST_GC=1 sh {phase['check']}")
+                    if phase["checkDeployer"] is not None:
+                        run(deployer, f"sh {phase['checkDeployer']}")
                     if phase["mutate"] is not None:
                         run(node, f"sh {phase['mutate']}")
+                    if phase["mutateDeployer"] is not None:
+                        run(deployer, f"sh {phase['mutateDeployer']}")
         except Exception as e:
             print(f"scenario {scenario['name']} fails: {e}")
             failed.append(scenario["name"])
-- 
cgit v1.3.1