From c59f05ae0b5a789f36b57f5839cd22af608dbf3d Mon Sep 17 00:00:00 2001 From: tv Date: Wed, 11 Nov 2026 17:06:38 +0000 Subject: populate: generate flakes for inputs drawn from sources For each source that draws inputs from other sources, populate generates a flake in .populate/flakes/, which re-exports the outputs of the source's flake with those inputs drawn from the other sources. The source itself stays as it is, so NIX_PATH still refers to it. Generated flakes refer to each other's generated flakes, so a source's own declarations apply wherever it gets drawn from, and get generated in dependency order. Evaluation fails for sources drawing inputs from missing sources, from themselves, or from each other. Generated flakes get locked on every run, and get removed once their sources no longer draw inputs from other sources. --- README.md | 64 ++++++++++++++++++++++ lib/types/populate.nix | 27 +++++++++ pkgs/populate/default.nix | 136 ++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 222 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b6e3401..39ed6cb 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,10 @@ Supported attributes: if they exist on the target. This is passed to the `--delete-excluded` option of rsync. Defaults to `true`. +* `inputs` (optional) - + inputs of the flake in this source to draw from other sources, see + [Flake inputs from sources](#flake-inputs-from-sources). + ### `flake` @@ -300,6 +304,10 @@ Supported attributes: and Nix with flake support (Nix 2.4 or later) on the target. Defaults to `false`. +* `inputs` (optional) - + inputs of the flake in this source to draw from other sources, see + [Flake inputs from sources](#flake-inputs-from-sources). + Instead of an attribute set, the flake reference can be given as a string. The flake commands on the deploying machine run using krops' own Nix, so @@ -339,6 +347,10 @@ Supported attributes: boolean that controls whether only the requested commit ref. should be fetched instead of the whole history, to save disk space and bandwith. Defaults to `false`. +* `inputs` (optional) - + inputs of the flake in this source to draw from other sources, see + [Flake inputs from sources](#flake-inputs-from-sources). + ### `pass` @@ -397,6 +409,58 @@ Supported attributes: * `target` - Content of the symlink. This is typically a relative path. +* `inputs` (optional) - + inputs of the flake in this source to draw from other sources, see + [Flake inputs from sources](#flake-inputs-from-sources). + + +## Flake inputs from sources + +Sources of type `file`, `flake`, `git`, and `symlink` can declare which inputs +of the flake they contain get drawn from other sources, in the style of +`flake.nix`, where `follows` names a source instead of an input. This makes +flakes on the target use what krops delivers there, much like `NIX_PATH` does +for Nix expressions. + +```nix +{ + nixpkgs.flake = { + flake = toString ./.; + attribute = "nixpkgs"; + }; + crate2nix.flake = { + flake = "github:nix-community/crate2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + myconf.file = { + path = toString ./.; + inputs = { + crate2nix.follows = "crate2nix"; + stockholm.inputs.nixpkgs.follows = "nixpkgs"; + }; + }; +} +``` + +For each source that draws inputs from other sources, populate generates a +flake in `.populate/flakes/`, which re-exports the outputs of the flake +in the source, with those inputs drawn from the other sources. The source +itself stays as it is, so `NIX_PATH` still refers to it. When a generated +flake draws an input from a source that has a generated flake itself, it uses +that one, so in the example above, myconf's crate2nix uses nixpkgs from the +`nixpkgs` source as well. `writeDeploy`'s `flake` parameter uses a source's +generated flake automatically. + +Sources in a Git work tree are referred to as `git+file:` (with submodules if +there are any), all other sources as `path:`, with symlinks resolved. +Generated flakes get locked on every run, so their lock files match the +sources, and get removed once their sources no longer draw inputs from other +sources. This requires Nix with flake support (Nix 2.4 or later) on the +target. Populating fails if a source that draws inputs from other sources +contains no `flake.nix`. Sources that inputs get drawn from don't need to be +flakes, as long as the inputs drawing from them are declared with +`flake = false`. + ## References diff --git a/lib/types/populate.nix b/lib/types/populate.nix index 319a7ba..b1d4151 100644 --- a/lib/types/populate.nix +++ b/lib/types/populate.nix @@ -77,6 +77,21 @@ }; }); + # Inputs of the flake in a source that get drawn from other sources, in the + # style of flake.nix, e.g. { nixpkgs.follows = "nixpkgs"; }. + flake-inputs = lib.types.attrsOf (lib.types.submodule { + options = { + follows = lib.mkOption { + default = null; + type = lib.types.nullOr lib.types.str; # TODO lib.types.label + }; + inputs = lib.mkOption { + default = {}; + type = flake-inputs; + }; + }; + }); + filter = lib.types.submodule { options = { type = lib.mkOption { @@ -99,6 +114,10 @@ }; file = lib.types.submodule { options = { + inputs = lib.mkOption { + default = {}; + type = flake-inputs; + }; path = lib.mkOption { type = lib.types.absolute-pathname; }; @@ -172,6 +191,10 @@ type = lib.types.bool; default = false; }; + inputs = lib.mkOption { + default = {}; + type = flake-inputs; + }; ref = lib.mkOption { type = lib.types.str; # TODO lib.types.git.ref }; @@ -218,6 +241,10 @@ }; symlink = lib.types.submodule { options = { + inputs = lib.mkOption { + default = {}; + type = flake-inputs; + }; target = lib.mkOption { type = lib.types.pathname; # TODO relative-pathname }; diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index e06d961..adf8479 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -40,6 +40,135 @@ let ${target.path}/.populate/backup/ ''; + # Shell code that turns the sentinel file into a directory, which populate + # uses for keeping state. The sentinel only needs to exist. + sentinelDirectory = sentinel: /* sh */ '' + if test -f ${quote sentinel}; then + rm ${quote sentinel} + mkdir ${quote sentinel} + fi + ''; + + # Generates a flake for each source that draws inputs from other sources, in + # .populate/flakes/. Each flake re-exports the outputs of the flake in + # its source, with those inputs drawn from the other sources' flakes, and + # gets locked on every run, so its lock file matches the sources. + flakes = target: sources: let + flakesDir = "${target.path}/.populate/flakes"; + generated = filterAttrs (_: source: flakeFollows source != []) sources; + names = attrNames generated; + drawnFrom = name: map (x: x.source) (flakeFollows generated.${name}); + sorted = toposort (a: b: elem a (drawnFrom b)) names; + # Generated flakes refer to each other's generated flakes, so they don't + # need to repeat overrides. + url = name: + if generated ? ${name} + then "path:${flakesDir}/${name}" + else "$(flake_url ${quote name})"; + check = name: source: + if source == name then + throw "krops: source ${name} draws inputs from itself" + else if !(sources ? ${source}) then + throw "krops: source ${name} draws inputs from missing source ${source}" + else + source; + flake = name: let + follows = flakeFollows generated.${name}; + drawn = unique (map (x: check name x.source) follows); + # Nix ignores overrides below an input that follows another input, so + # an input of an input drawn from a source gets overridden at that + # source. + override = x: let + drawnParents = filter + (path: any (y: y.path == path) follows) + (map (n: take n x.path) (range 1 (length x.path - 1))); + parent = if drawnParents == [] then null else last drawnParents; + base = + if parent == null then name + else (findFirst (y: y.path == parent) null follows).source; + rest = if parent == null then x.path else drop (length parent) x.path; + in + "inputs.${toJSON base}" + + concatMapStrings (name: ".inputs.${toJSON name}") rest + + ".follows = ${toJSON x.source};"; + in /* sh */ '' + ( + url=$(flake_url ${quote name}) + ${concatImapStrings (i: source: /* sh */ '' + url${toString i}=${url source} + '') drawn} + rm -fR ${quote "${flakesDir}/${name}"} + mkdir ${quote "${flakesDir}/${name}"} + cd ${quote "${flakesDir}/${name}"} + { + echo '{' + printf ' inputs.%s.url = "%s";\n' ${quote (toJSON name)} "$url" + ${concatImapStrings (i: source: /* sh */ '' + printf ' inputs.%s.url = "%s";\n' ${quote (toJSON source)} "$url${toString i}" + '') drawn} + ${concatMapStrings (x: /* sh */ '' + printf ' %s\n' ${quote (override x)} + '') follows} + printf ' outputs = inputs: inputs.%s;\n' ${quote (toJSON name)} + echo '}' + } > flake.nix + nix flake lock >&2 + ) + ''; + in + if sorted ? cycle then + throw "krops: sources draw inputs from each other: ${toString sorted.cycle}" + else runShell target /* sh */ '' + set -efu + ${optionalString (names != []) /* sh */ '' + ${requireFlakes "drawing flake inputs from sources"} + + # Prints the flake URL of the source named $1. + flake_url() { + path=$(readlink -f ${quote target.path}/"$1") + if ! test -d "$path"; then + >&2 printf 'error: source %s is not a directory\n' "$1" + exit 1 + fi + if test -e "$path"/.git; then + if test -e "$path"/.gitmodules; then + printf 'git+file:%s?submodules=1' "$path" + else + printf 'git+file:%s' "$path" + fi + else + printf 'path:%s' "$path" + fi + } + + # Inputs can only be drawn into flakes, so check all sources before + # generating anything. + missing= + for name in ${concatMapStringsSep " " quote names}; do + if ! test -e ${quote target.path}/"$name"/flake.nix; then + missing="$missing $name" + fi + done + if test -n "$missing"; then + >&2 printf 'error: source %s draws flake inputs from other sources, but contains no flake.nix\n' $missing + exit 1 + fi + + ${sentinelDirectory "${target.path}/.populate"} + mkdir -p ${quote flakesDir} + ${concatMapStrings flake sorted.result} + ''} + # Remove flakes of sources that no longer draw inputs from other sources. + if test -d ${quote flakesDir}; then + ls -A ${quote flakesDir} | while read -r name; do + case ${quote " ${toString names} "} in + *" $name "*) ;; + *) rm -fR ${quote flakesDir}/"$name" ;; + esac + done + fi + ''; + pop.derivation = target: source: runShell target /* sh */ '' nix-build -E ${quote source.text} -o ${quote target.path} >&2 ''; @@ -119,11 +248,7 @@ let fi nix-store --realise "$path" --add-root ${quote target.path} --indirect >/dev/null - # The sentinel only needs to exist, so it can become a directory. - if test -f ${quote sentinel}; then - rm ${quote sentinel} - mkdir ${quote sentinel} - fi + ${sentinelDirectory sentinel} mkdir -p ${quote inputsRoots} roots= while read -r path; do @@ -453,6 +578,7 @@ writers.writeDash "populate.${target.host}" '' set -x ${optionalString backup (do-backup { inherit target; })} ${concatStringsSep "\n" (mapAttrsToList (populate target) source)} + ${flakes target source} ${ssh' target} -q -O stop ${quote target.host} '' -- cgit v1.3.1