aboutsummaryrefslogtreecommitdiffstats
path: root/pkgs/populate/src/flake.rs
diff options
context:
space:
mode:
Diffstat (limited to 'pkgs/populate/src/flake.rs')
-rw-r--r--pkgs/populate/src/flake.rs633
1 files changed, 23 insertions, 610 deletions
diff --git a/pkgs/populate/src/flake.rs b/pkgs/populate/src/flake.rs
index 5538288..f03e443 100644
--- a/pkgs/populate/src/flake.rs
+++ b/pkgs/populate/src/flake.rs
@@ -1,4 +1,4 @@
-//! Flake and flake-input sources, and drawing inputs from sources.
+//! Drawing inputs of flakes in sources from other sources.
//!
//! This builds the same shell code as shell.nix generates, with every value
//! quoted, so both implementations behave alike while the code moves to
@@ -10,12 +10,28 @@ use crate::spec::{Follows, Source};
use serde_json::{Map, Value};
use std::collections::{BTreeMap, BTreeSet};
-pub const LOCK_HAS_INPUT: &str = include_str!("flake/lock-has-input.nix");
-pub const METADATA_SUBDIR: &str = include_str!("flake/metadata-subdir.nix");
-const SUBDIRS: &str = include_str!("flake/subdirs.jq");
-const SELECTED: &str = include_str!("flake/selected.jq");
-const SELF_IS_REMOTE: &str = include_str!("flake/self-is-remote.jq");
-const SELF_LOCKED: &str = include_str!("flake/self-locked.jq");
+/// Nix function returning 1 if the lock file locks has an input at path, a
+/// list of input names, resolving inputs that follow others, and else 0.
+/// Like lockHasInput in flake-jq.nix, for the shell code drawing inputs.
+pub const LOCK_HAS_INPUT: &str = r#"locks: path: let
+ nodes = locks.nodes;
+ input = key: name: (nodes.${key}.inputs or {}).${name} or null;
+ # The node an input refers to, following follows from the root.
+ node = ref:
+ if ref == null then null
+ else if builtins.isList ref then
+ builtins.foldl'
+ (key: name: if key == null then null else node (input key name))
+ locks.root
+ ref
+ else ref;
+ has = key: path:
+ if path == [] then true
+ else let next = node (input key (builtins.head path)); in
+ next != null && has next (builtins.tail path);
+in
+ if has locks.root path then "1" else "0"
+"#;
fn bool_setting(settings: &Map<String, Value>, key: &str, default: bool) -> bool {
settings
@@ -24,10 +40,6 @@ fn bool_setting(settings: &Map<String, Value>, key: &str, default: bool) -> bool
.unwrap_or(default)
}
-fn str_setting<'a>(settings: &'a Map<String, Value>, key: &str) -> Option<&'a str> {
- settings.get(key).and_then(Value::as_str)
-}
-
fn patches(settings: &Map<String, Value>) -> Vec<&str> {
settings
.get("patches")
@@ -57,70 +69,6 @@ pub fn derives_flake(source: &Source) -> bool {
|| (bool_setting(settings, "rewriteLock", true) && !source.follows().is_empty()))
}
-/// Shell code running body on the target: as it is for the local target,
-/// and over ssh, via sudo if the target says so, for remote ones. The
-/// definitions in sh::prelude come first.
-pub fn run_shell(context: &Context, body: &str) -> String {
- let command = format!("{}\n{body}", sh::prelude());
- if context.local {
- format!("(\n{command}\n)")
- } else if context.target.sudo {
- format!(
- "{} {} {}",
- q(&context.ssh.to_string_lossy()),
- q(&context.target.host),
- q(&format!("sudo sh -c {}", q(&command)))
- )
- } else {
- format!(
- "{} {} {}",
- q(&context.ssh.to_string_lossy()),
- q(&context.target.host),
- q(&command)
- )
- }
-}
-
-/// Shell code transferring source, a shell word, to path on the target, as
-/// rsync::transfer does without .rsync-filter files.
-fn rsync_text(context: &Context, path: &str, source: &str, checksum: bool) -> String {
- format!(
- r#"{prepare}
-source_path={source}
-if test -d "$source_path"; then
- source_path=$source_path/
-fi
-rsync {checksum} {rsync_path} -e {ssh} --protect-args \
- --itemize-changes \
- --out-format={format} \
- --info=backup,del,copy,misc,mount,name,nonreg,remove,skip,symsafe \
- --omit-dir-times \
- --omit-link-times \
- -rlptD \
- --delete-excluded \
- "$source_path" {destination} >&2
-"#,
- prepare = run_shell(
- context,
- &format!(
- "set -- {} {}\n{}",
- q(path),
- q(&context.name(path)),
- sh::break_hard_links()
- )
- ),
- checksum = if checksum { "--checksum" } else { "" },
- rsync_path = if context.target.sudo {
- "--rsync-path='sudo rsync'"
- } else {
- ""
- },
- ssh = q(&context.ssh.to_string_lossy()),
- format = q("%i %n%L %''b/%''l"),
- destination = q(&context.rsync_destination(path)),
- )
-}
-
fn use_state(sentinel: &str, names: &[&str]) -> String {
let mut call = format!("use_state {}", q(sentinel));
for name in names {
@@ -130,541 +78,6 @@ fn use_state(sentinel: &str, names: &[&str]) -> String {
call
}
-/// Delivers a flake or flake-input source named name to path. Returns the
-/// shell code doing it on the deploying machine, in the source's working
-/// directory.
-pub fn flake_input(context: &Context, name: &str, path: &str, source: &Source) -> String {
- let settings = &source.settings;
- let url = str_setting(settings, "url").unwrap_or_default();
- let input = if source.kind == "flake" {
- None
- } else {
- str_setting(settings, "name")
- };
- let input_path: Vec<&str> = input.map(|i| i.split('/').collect()).unwrap_or_default();
- let fetch_on_target = bool_setting(settings, "fetchOnTarget", false);
- let dir = path.rsplit_once('/').map(|(d, _)| d).unwrap_or("");
- let sentinel = format!("{dir}/.populate");
- let derived = derives_flake(&Source {
- kind: "flake".into(),
- settings: settings.clone(),
- });
- let link_path = if derived {
- format!("{sentinel}/derived/{name}/original")
- } else {
- path.to_string()
- };
- let patches = patches(settings);
-
- // Uploads the patches to .populate/patches/<name>, numbered in order.
- let upload_patches = if patches.is_empty() {
- String::new()
- } else {
- let mut text = String::from("rm -fR \"$PWD\"/patches\nmkdir \"$PWD\"/patches\n");
- for (i, patch) in patches.iter().enumerate() {
- let base = patch.rsplit('/').next().unwrap_or(patch);
- text += &format!(
- "cp {} \"$PWD\"/patches/{}\n",
- q(patch),
- q(&format!("{:04}-{base}", i + 1))
- );
- }
- text += &run_shell(
- context,
- &format!(
- "set -efu\n{}\nmkdir -p {}\n",
- use_state(&sentinel, &[name]),
- q(&format!("{sentinel}/patches"))
- ),
- );
- text.push('\n');
- text += &rsync_text(
- context,
- &format!("{sentinel}/patches/{name}"),
- "\"$PWD\"/patches",
- true,
- );
- text
- };
-
- let is_remote_ref = {
- let scheme: String = url
- .chars()
- .take_while(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '+')
- .collect();
- !scheme.is_empty()
- && url[scheme.len()..].starts_with(':')
- && ![
- "file",
- "flake",
- "git+file",
- "hg+file",
- "path",
- "tarball+file",
- ]
- .contains(&scheme.as_str())
- };
-
- let no_input = format!(
- ">&2 printf 'error: flake %s has no input %s\\n' {} {}\nexit 1\n",
- q(url),
- q(input.unwrap_or(""))
- );
-
- let overridden: Vec<Vec<String>> = Source {
- kind: "flake".into(),
- settings: settings.clone(),
- }
- .follows()
- .into_iter()
- .map(|f| f.path)
- .collect();
- let require_inputs = bool_setting(settings, "requireInputs", true);
-
- // Shell function printing the store path $1, followed by the inputs its
- // flake in the subdirectory $2 pins in its own lock file. Without a
- // lock file, locking would change the flake, so there are no such
- // inputs. Inputs that get drawn from other sources are left out, with
- // all they pin: Nix archives an empty flake in their place, so it
- // doesn't fetch them.
- let closure = |nix: &str, mktemp: &str| -> String {
- let overridden_json = serde_json::to_string(&overridden).expect("serializes");
- let mut overrides = String::new();
- if !overridden.is_empty() {
- overrides += &format!(
- "closure_empty=$({mktemp} -d)\necho '{{ outputs = _: {{}}; }}' > \"$closure_empty\"/flake.nix\n"
- );
- for path in &overridden {
- overrides += &format!(
- "set -- \"$@\" --override-input {} \"path:$closure_empty\"\n",
- q(&path.join("/"))
- );
- }
- }
- let on_failure = if require_inputs {
- format!(
- ">&2 printf 'error: source %s: the inputs its own lock file pins can'\"'\"'t be delivered%s\\n' \\\n {} {}\nexit 1\n",
- q(name),
- q(if input.is_some() {
- ". Set its requireInputs to false to deliver it anyway."
- } else {
- ""
- })
- )
- } else {
- format!(
- r#"# Without --no-update-lock-file, Nix adds what the lock file
-# lacks.
-if closure_archive=$(
- {nix} flake archive --json --no-write-lock-file "$@" "$closure_flake"
-); then
- >&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \
- {name}
- closure_paths "$closure_archive"
-else
- >&2 printf 'warning: source %s: the inputs its own lock file pins can'"'"'t be delivered; delivering it without them\n' \
- {name}
-fi
-"#,
- name = q(name)
- )
- };
- format!(
- r#"
-# Prints the store paths in nix flake archive's output $1.
-closure_paths() {{
- ARCHIVE=$1 OVERRIDDEN={overridden_json} \
- {nix} eval --impure --raw --expr '
- let
- overridden = builtins.fromJSON (builtins.getEnv "OVERRIDDEN");
- # Inputs given by relative paths are part of their parent.
- paths = prefix: node: builtins.concatMap
- (name: let
- input = node.inputs.${{name}};
- path = prefix ++ [ name ];
- in
- if builtins.elem path overridden then []
- else
- (if input ? path then [ input.path ] else []) ++
- paths path input)
- (builtins.attrNames (node.inputs or {{}}));
- in
- builtins.concatStringsSep "\n"
- (paths [] (builtins.fromJSON (builtins.getEnv "ARCHIVE")))
- '
- echo
-}}
-closure() {{
- printf '%s\n' "$1"
- closure_dir=$1${{2:+/$2}}
- if test -e "$closure_dir"/flake.nix && test -e "$closure_dir"/flake.lock; then
- closure_flake=path:$1${{2:+?dir=$2}}
- set --
- {overrides}
- if closure_archive=$(
- {nix} flake archive --json --no-update-lock-file "$@" "$closure_flake"
- ); then
- closure_paths "$closure_archive"
- else
- {on_failure}
- fi
- {cleanup}
- fi
-}}
-"#,
- overridden_json = q(&overridden_json),
- cleanup = if overridden.is_empty() {
- ""
- } else {
- "rm -R \"$closure_empty\""
- },
- )
- };
-
- let inputs_roots = format!("{sentinel}/flake-inputs/{name}");
- let inputs_roots_parent = format!("{sentinel}/flake-inputs");
- let uploads = format!("{sentinel}/flake-uploads");
- let upload = format!("{uploads}/{name}");
- let remove_upload = format!(
- r#"remove_tree {upload}
-# Sources get populated one after another, so no other source uses the
-# directory meanwhile.
-if test -d {uploads} && test -z "$(ls -A {uploads})"; then
- rmdir {uploads}
-fi
-"#,
- upload = q(&upload),
- uploads = q(&uploads)
- );
-
- // Reads the subdirectory containing the flake or input, which may be
- // empty, followed by store paths from stdin. Links the first store
- // path, or the subdirectory therein, at the link path, and registers it
- // and the remaining ones as garbage collector roots. Roots get added
- // before obsolete ones get removed, so the inputs stay rooted.
- let link = format!(
- r#"set -efu
-read -r dir
-read -r path
-{use_state}
-{mkdir_derived}
-make_room_for_link {link_path} {link_name}
-nix-store --realise "$path" --add-root {link_path} --indirect >/dev/null
-# The garbage collector follows the link into the store path.
-if test -n "$dir"; then
- ln -fns "$path/$dir" {link_path}
-fi
-
-# Left behind by an interrupted run, or not needed anymore.
-{remove_upload}
-mkdir -p {roots}
-roots=
-while read -r path; do
- if test -z "$path"; then
- continue
- fi
- root=$(basename "$path")
- if test "$(readlink {roots}/"$root")" != "$path"; then
- nix-store --realise "$path" \
- --add-root {roots}/"$root" --indirect \
- >/dev/null
- fi
- roots="$roots $root"
-done
-ls -A {roots} | while read -r root; do
- case " $roots " in
- *" $root "*) ;;
- *) rm {roots}/"$root" ;;
- esac
-done
-# An input without own inputs leaves the directory empty.
-if test -z "$roots"; then
- rmdir {roots}
- if test -z "$(ls -A {roots_parent})"; then
- rmdir {roots_parent}
- fi
-fi
-"#,
- use_state = use_state(&sentinel, &[name]),
- mkdir_derived = if derived {
- format!(
- "mkdir -p {}",
- q(link_path.rsplit_once('/').map(|(d, _)| d).unwrap_or(""))
- )
- } else {
- String::new()
- },
- link_path = q(&link_path),
- link_name = q(&context.name(&link_path)),
- roots = q(&inputs_roots),
- roots_parent = q(&inputs_roots_parent),
- );
-
- // Reads the selection, as selected produces it, and the store path of
- // the flake itself from stdin, fetches what's selected, unless it's the
- // flake itself, which gets added from the upload directory unless the
- // store has it already, then fetches the inputs its own lock file
- // pins, and links the results.
- let fetch_and_link = format!(
- r#"set -efu
-require_flakes 'fetching flake inputs on the target'
-read -r selected
-read -r self
-read -r expected
-if test -n "$self" && ! nix-store --check-validity "$self" 2>/dev/null; then
- name=$(basename "$self" | cut -c34-)
- mv {upload}/tree {upload}/"$name"
- added=$(nix-store --add-fixed --recursive sha256 {upload}/"$name")
- {remove_upload}
- if test "$added" != "$self"; then
- >&2 printf 'error: flake %s arrived as %s instead of %s\n' \
- {url} "$added" "$self"
- exit 1
- fi
-fi
-# Fetching contacts where the input comes from even if the store has what
-# the lock file pins.
-if test -n "$expected" && ! nix-store --check-validity "$expected" 2>/dev/null; then
- expected=
-fi
-# The store path and the subdirectory, on separate lines. Inputs referring
-# to local paths can only be fetched if the target has them as well.
-fetched=$(
- FLAKE={url} INPUT={input} \
- SELECTED=$selected SELF=$self EXPECTED=$expected \
- nix eval --impure --raw --expr '
- let
- selected = builtins.fromJSON (builtins.getEnv "SELECTED");
- expected = builtins.getEnv "EXPECTED";
- selectedPath =
- if selected.self || expected != "" then null
- else localPath selected.locked;
- path =
- if selected.self then builtins.getEnv "SELF"
- else if expected != "" then expected
- else (builtins.fetchTree selected.locked).outPath;
- dir = path + (if selected.subdir == "" then "" else "/" + selected.subdir);
- lock =
- if builtins.pathExists (dir + "/flake.lock")
- then builtins.fromJSON (builtins.readFile (dir + "/flake.lock"))
- else {{ nodes = {{}}; }};
- localPath = locked:
- if (locked.type or null) == "path" then locked.path
- else let
- match = builtins.match "file:(//)?(.*)" (locked.url or "");
- in
- if match == null then null else builtins.elemAt match 1;
- missing = builtins.filter
- (key: let
- p = localPath (lock.nodes.${{key}}.locked or {{}});
- in
- p != null && builtins.substring 0 1 p == "/" && !builtins.pathExists p)
- (builtins.attrNames lock.nodes);
- key = builtins.head missing;
- in
- if selectedPath != null && builtins.substring 0 1 selectedPath == "/" && !builtins.pathExists selectedPath then
- throw "flake ${{builtins.getEnv "FLAKE"}}: input ${{builtins.getEnv "INPUT"}} refers to ${{selectedPath}}, which does not exist on the target"
- else if missing != [] then
- throw "flake ${{builtins.getEnv "FLAKE"}}: input ${{key}} refers to ${{localPath lock.nodes.${{key}}.locked}}, which does not exist on the target"
- else
- path + "\n" + selected.subdir
- '
-)
-path=$(printf '%s\n' "$fetched" | sed -n 1p)
-subdir=$(printf '%s\n' "$fetched" | sed -n 2p)
-{closure}
-# Not in the pipeline, where failing wouldn't fail populate.
-paths=$(closure "$path" "$subdir")
-printf '%s\n%s\n' "$subdir" "$paths" | {{
- {link}
-}}
-"#,
- upload = q(&upload),
- url = q(url),
- input = q(input.unwrap_or("")),
- closure = closure("nix", "mktemp"),
- );
-
- // Fetches the flake and its inputs, and links the results.
- let archive_and_link = format!(
- r#"set -efu
-require_flakes 'fetching flakes on the target'
-metadata=$(nix flake metadata --json --no-update-lock-file {url})
-# The store path and the subdirectory, on separate lines.
-fetched=$(
- METADATA=$metadata nix eval --impure --raw --expr {expr}
-)
-path=$(printf '%s\n' "$fetched" | sed -n 1p)
-subdir=$(printf '%s\n' "$fetched" | sed -n 2p)
-{closure}
-paths=$(closure "$path" "$subdir")
-printf '%s\n%s\n' "$subdir" "$paths" | {{
- {link}
-}}
-"#,
- url = q(url),
- expr = q(&format!(
- "let\n metadata = builtins.fromJSON (builtins.getEnv \"METADATA\");\nin\n metadata.path + \"\\n\" + ({METADATA_SUBDIR}) metadata\n"
- )),
- closure = closure("nix", "mktemp"),
- );
-
- let nix_ = "nix --extra-experimental-features 'nix-command flakes'";
- let selected_jq = format!("{SUBDIRS}{SELECTED}");
- let self_locked_jq = format!("{SUBDIRS}{SELF_LOCKED}");
-
- if fetch_on_target && input.is_none() && is_remote_ref {
- return format!(
- "{upload_patches}\n{}\n",
- run_shell(context, &archive_and_link)
- );
- }
-
- let self_text = if fetch_on_target {
- let upload_self = if context.local {
- String::new()
- } else {
- format!(
- r#"if ! printf '%s\n' "$self" | {{
- {check}
-}}; then
- {prepare}
- # Without applying .rsync-filter files, which would change the flake.
- {rsync}
-fi
-"#,
- check = run_shell(
- context,
- "read -r self\nnix-store --check-validity \"$self\" 2>/dev/null\n"
- ),
- prepare = run_shell(
- context,
- &format!(
- "set -efu\n{}\n{remove_upload}\nmkdir -p {}\n",
- use_state(&sentinel, &[name]),
- q(&upload)
- )
- ),
- rsync = rsync_text(context, &format!("{upload}/tree"), "\"$self\"", false),
- )
- };
- format!(
- r#"# The flake itself has no lock entry, so the target fetches it by its
-# locked reference, if it can, or else receives it.
-if jq -e {self_is_remote} "$cache" >/dev/null; then
- selected=$(
- jq -c --argjson selected "$selected" {self_locked} \
- "$cache"
- )
-else
- self=$(jq -r .path "$cache")
- {upload_self}
-fi
-"#,
- self_is_remote = q(SELF_IS_REMOTE),
- self_locked = q(&self_locked_jq),
- )
- } else {
- "self=$(jq -r .path \"$cache\")\n".to_string()
- };
-
- let deliver = if fetch_on_target {
- format!(
- "printf '%s\\n%s\\n%s\\n' \"$selected\" \"$self\" \"$expected\" | {{\n {}\n}}\n",
- run_shell(context, &fetch_and_link)
- )
- } else {
- let copy = if context.local {
- String::new()
- } else {
- format!(
- "mkdir -p \"$PWD\"/bin\nln -fns {} \"$PWD\"/bin/ssh\nPATH=$PWD/bin:$PATH \\\n{nix_} copy --to {} $paths >&2\n",
- q(&context.ssh_nix.to_string_lossy()),
- q(&format!("ssh://{}", context.target.host))
- )
- };
- format!(
- r#"path=$self
-if test -z "$path"; then
- if test -n "$expected" &&
- nix-store --check-validity "$expected" 2>/dev/null; then
- path=$expected
- else
- path=$(
- SELECTED=$selected {nix_} eval --impure --raw --expr '
- (builtins.fetchTree
- (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath
- '
- )
- fi
-fi
-subdir=$(printf '%s\n' "$selected" | jq -r .subdir)
-{closure}
-paths=$(closure "$path" "$subdir")
-
-{copy}
-printf '%s\n%s\n' "$subdir" "$paths" | {{
- {link}
-}}
-"#,
- closure = closure(nix_, "mktemp"),
- link = run_shell(context, &link),
- )
- };
-
- format!(
- r#"set -efu
-{upload_patches}
-
-cache=$TMPDIR/flake-metadata.$(
- printf %s {url} |
- sha256sum |
- cut -c-32
-).json
-if ! test -e "$cache"; then
- {{
- {nix_} flake metadata --json --no-update-lock-file {url}
- }} > "$cache"
-fi
-
-if ! selected=$(
- jq -ce --argjson input {input_path} {selected} \
- "$cache" 2>/dev/null
-); then
- {no_input}
-fi
-
-self=
-if printf '%s\n' "$selected" | jq -e .self >/dev/null; then
- {self_text}
-fi
-
-# The store path the lock file pins what's selected to, which is the one of
-# a directory named source with the locked hash. A store that has it needs
-# no fetching, which contacts where the input comes from even then.
-expected=
-hash=$(printf '%s\n' "$selected" | jq -r '.locked.narHash // empty')
-case $hash in
- sha256-*)
- expected=$(
- nix-store --print-fixed-path --recursive sha256 "$(
- printf %s "${{hash#sha256-}}" |
- base64 -d |
- od -An -v -tx1 |
- tr -d ' \n'
- )" source
- )
- ;;
-esac
-
-{deliver}
-"#,
- url = q(url),
- input_path = q(&serde_json::to_string(&input_path).expect("serializes")),
- selected = q(&selected_jq),
- )
-}
-
/// Draws inputs of flakes in sources from other sources, and derives what
/// the target paths of flake and flake-input sources that get patched, get
/// their lock files rewritten, or are materialized get. Sources get
generated by cgit v1.3.1 (git 2.54.0) at 2026-11-15 07:03:39 +0000