From 69f0ab4533a6b233dca1193ee558de8994df1ffb Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 12 Nov 2026 00:02:06 +0000 Subject: populate: add prune Populate never removes what it delivered for sources that aren't configured anymore. For flake and flake-input sources, that keeps their garbage collector roots, so what they delivered never gets collected. With prune, populate removes everything from the target directory that isn't a source, except .populate, after syncing the sources. It also removes the roots in .populate/flake-inputs of names that aren't flake or flake-input sources, and uploads left behind by interrupted runs. Pruning is opt-in, because the target directory may contain other things worth keeping, or be shared by deployments with different sources. writeDeploy, writeTest, and writeCommand pass prune on to populate. A feature scenario deploys sources, deploys fewer without pruning, then with pruning, and checks that the removed sources' entries and roots are gone, while the remaining sources and .populate stay. --- README.md | 17 +++++++++++++++ pkgs/krops/default.nix | 11 ++++++---- pkgs/populate/default.nix | 41 +++++++++++++++++++++++++++++++++++- tests/scenarios.nix | 53 +++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 115 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 79ace5b..eda8dcd 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,15 @@ Create the sentinel file (`/var/src/.populate`) before syncing the new source. Specifies which `nixos-rebuild` operation to perform. +### `prune` (optional, defaults to false) + +Remove everything from the target directory that isn't a source, except +`.populate`, after syncing the sources. This also removes the garbage +collector roots of the inputs of former `flake` and `flake-input` sources in +`.populate/flake-inputs/`, so the garbage collector can collect what +they delivered. Don't use it if the target directory contains anything else +worth keeping, or if several deployments with different sources share it. + ### `useNixOutputMonitor` (optional, defaults to `"opportunistic"`) Specifies when to pipe `nixos-rebuild`'s output to @@ -190,6 +199,10 @@ below `/var/src`, and executes `NIX_PATH=/var/src nix-build -A system '&2 + ${populate { inherit backup force prune source; target = target'; }} >&2 NIX_PATH=${lib.escapeShellArg target'.path} \ ${nix}/bin/nix-build \ -A system \ diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index 5472b5d..553183e 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -169,6 +169,44 @@ let fi ''; + # Removes entries of the target directory that aren't sources, garbage + # collector roots of flake inputs of sources that aren't flake or + # flake-input sources, and uploads left behind by interrupted runs. + prune' = target: sources: let + pattern = names: concatMapStringsSep "|" quote names; + flakeNames = attrNames (filterAttrs + (_: source: elem source.type [ "flake" "flake-input" ]) + sources); + in runShell target /* sh */ '' + set -efu + cd ${quote target.path} + ls -A | while read -r name; do + case $name in + ${pattern (attrNames sources ++ [ ".populate" ])}) ;; + *) + >&2 printf 'pruning %s\n' ${quote target.path}/"$name" + rm -fR ./"$name" + ;; + esac + done + roots=.populate/flake-inputs + if test -d "$roots"; then + ls -A "$roots" | while read -r name; do + case $name in + ${optionalString (flakeNames != []) "${pattern flakeNames}) ;;"} + *) rm -fR "$roots"/"$name" ;; + esac + done + if test -z "$(ls -A "$roots")"; then + rmdir "$roots" + fi + fi + if test -e .populate/flake-uploads; then + chmod -R u+w .populate/flake-uploads + rm -fR .populate/flake-uploads + fi + ''; + pop.derivation = target: source: runShell target /* sh */ '' nix-build -E ${quote source.text} -o ${quote target.path} >&2 ''; @@ -761,7 +799,7 @@ let in -{ backup ? false, force ? false, source, target }: +{ backup ? false, force ? false, prune ? false, source, target }: writers.writeDash "populate.${target.host}" '' set -efu @@ -776,6 +814,7 @@ writers.writeDash "populate.${target.host}" '' ${optionalString backup (do-backup { inherit target; })} ${concatStringsSep "\n" (mapAttrsToList (populate target) source)} ${flakes target source} + ${optionalString prune (prune' target source)} # Stop the ssh master, if any source started one. if ${ssh' target} -q -O check ${quote target.host} 2>/dev/null; then diff --git a/tests/scenarios.nix b/tests/scenarios.nix index 22431a2..c9774bf 100644 --- a/tests/scenarios.nix +++ b/tests/scenarios.nix @@ -64,10 +64,10 @@ let path = "/var/krops/${name}"; target = targets.${targetKind} path; vars = "T=${path}\n"; - phase = { sources, expect ? "success", check ? "", mutate ? null }: let + phase = { sources, prune ? false, expect ? "success", check ? "", mutate ? null }: let command = populate { source = krops.evalSource [ sources ]; - inherit target; + inherit prune target; force = true; }; evaluated = builtins.tryEval (builtins.seq (toString command) true); @@ -480,6 +480,55 @@ in { } ]; }) + # Pruning removes everything sources don't deliver anymore, including + # garbage collector roots, and keeps everything else. + (s "prune" (let + sources = { + a.file = "/fixtures/alt"; + b.flake = { flake = "/fixtures/top"; inherit fetchOnTarget; }; + c.flake-input = { flake = "/fixtures/top"; name = "dep"; inherit fetchOnTarget; }; + d.symlink = "a"; + }; + in { + prepare = '' + mkdir -p "$T" + echo other > "$T"/other + ''; + phases = [ + { + inherit sources; + check = '' + test -e "$T"/.populate/flake-inputs/b || fail "roots of b missing" + test -e "$T"/.populate/flake-inputs/c || fail "roots of c missing" + ''; + } + { + # Without pruning, nothing gets removed. + sources = { inherit (sources) a; }; + check = '' + for name in b c d other .populate/flake-inputs/b .populate/flake-inputs/c; do + test -e "$T"/$name || fail "$name removed" + done + ''; + } + { + # b changes its type, so its roots go, while b stays. + sources = { inherit (sources) a; b.symlink = "a"; }; + prune = true; + check = '' + test -e "$T"/a/flake.nix || fail "a removed" + expect_eq b "$(readlink "$T"/b)" a + test -d "$T"/.populate || fail ".populate removed" + for name in c d other .populate/flake-inputs; do + test ! -e "$T"/$name || fail "$name not pruned" + done + if nix-store --gc --print-roots 2>/dev/null | grep -F "$T/"; then + fail "garbage collector roots remain" + fi + ''; + } + ]; + })) (s "cycle" { phases = [{ sources = { -- cgit v1.3.1