aboutsummaryrefslogtreecommitdiffstats
path: root/pkgs/populate/src
diff options
context:
space:
mode:
Diffstat (limited to 'pkgs/populate/src')
-rw-r--r--pkgs/populate/src/flake.rs633
-rw-r--r--pkgs/populate/src/flake_input.rs777
-rw-r--r--pkgs/populate/src/main.rs1
-rw-r--r--pkgs/populate/src/populate.rs7
-rw-r--r--pkgs/populate/src/run.rs23
-rw-r--r--pkgs/populate/src/sh.rs199
6 files changed, 1024 insertions, 616 deletions
diff --git a/pkgs/populate/src/flake.rs b/pkgs/populate/src/flake.rs
index 5538288..f03e443 100644
--- a/pkgs/populate/src/flake.rs
+++ b/pkgs/populate/src/flake.rs
@@ -1,4 +1,4 @@
-//! Flake and flake-input sources, and drawing inputs from sources.
+//! Drawing inputs of flakes in sources from other sources.
//!
//! This builds the same shell code as shell.nix generates, with every value
//! quoted, so both implementations behave alike while the code moves to
@@ -10,12 +10,28 @@ use crate::spec::{Follows, Source};
use serde_json::{Map, Value};
use std::collections::{BTreeMap, BTreeSet};
-pub const LOCK_HAS_INPUT: &str = include_str!("flake/lock-has-input.nix");
-pub const METADATA_SUBDIR: &str = include_str!("flake/metadata-subdir.nix");
-const SUBDIRS: &str = include_str!("flake/subdirs.jq");
-const SELECTED: &str = include_str!("flake/selected.jq");
-const SELF_IS_REMOTE: &str = include_str!("flake/self-is-remote.jq");
-const SELF_LOCKED: &str = include_str!("flake/self-locked.jq");
+/// Nix function returning 1 if the lock file locks has an input at path, a
+/// list of input names, resolving inputs that follow others, and else 0.
+/// Like lockHasInput in flake-jq.nix, for the shell code drawing inputs.
+pub const LOCK_HAS_INPUT: &str = r#"locks: path: let
+ nodes = locks.nodes;
+ input = key: name: (nodes.${key}.inputs or {}).${name} or null;
+ # The node an input refers to, following follows from the root.
+ node = ref:
+ if ref == null then null
+ else if builtins.isList ref then
+ builtins.foldl'
+ (key: name: if key == null then null else node (input key name))
+ locks.root
+ ref
+ else ref;
+ has = key: path:
+ if path == [] then true
+ else let next = node (input key (builtins.head path)); in
+ next != null && has next (builtins.tail path);
+in
+ if has locks.root path then "1" else "0"
+"#;
fn bool_setting(settings: &Map<String, Value>, key: &str, default: bool) -> bool {
settings
@@ -24,10 +40,6 @@ fn bool_setting(settings: &Map<String, Value>, key: &str, default: bool) -> bool
.unwrap_or(default)
}
-fn str_setting<'a>(settings: &'a Map<String, Value>, key: &str) -> Option<&'a str> {
- settings.get(key).and_then(Value::as_str)
-}
-
fn patches(settings: &Map<String, Value>) -> Vec<&str> {
settings
.get("patches")
@@ -57,70 +69,6 @@ pub fn derives_flake(source: &Source) -> bool {
|| (bool_setting(settings, "rewriteLock", true) && !source.follows().is_empty()))
}
-/// Shell code running body on the target: as it is for the local target,
-/// and over ssh, via sudo if the target says so, for remote ones. The
-/// definitions in sh::prelude come first.
-pub fn run_shell(context: &Context, body: &str) -> String {
- let command = format!("{}\n{body}", sh::prelude());
- if context.local {
- format!("(\n{command}\n)")
- } else if context.target.sudo {
- format!(
- "{} {} {}",
- q(&context.ssh.to_string_lossy()),
- q(&context.target.host),
- q(&format!("sudo sh -c {}", q(&command)))
- )
- } else {
- format!(
- "{} {} {}",
- q(&context.ssh.to_string_lossy()),
- q(&context.target.host),
- q(&command)
- )
- }
-}
-
-/// Shell code transferring source, a shell word, to path on the target, as
-/// rsync::transfer does without .rsync-filter files.
-fn rsync_text(context: &Context, path: &str, source: &str, checksum: bool) -> String {
- format!(
- r#"{prepare}
-source_path={source}
-if test -d "$source_path"; then
- source_path=$source_path/
-fi
-rsync {checksum} {rsync_path} -e {ssh} --protect-args \
- --itemize-changes \
- --out-format={format} \
- --info=backup,del,copy,misc,mount,name,nonreg,remove,skip,symsafe \
- --omit-dir-times \
- --omit-link-times \
- -rlptD \
- --delete-excluded \
- "$source_path" {destination} >&2
-"#,
- prepare = run_shell(
- context,
- &format!(
- "set -- {} {}\n{}",
- q(path),
- q(&context.name(path)),
- sh::break_hard_links()
- )
- ),
- checksum = if checksum { "--checksum" } else { "" },
- rsync_path = if context.target.sudo {
- "--rsync-path='sudo rsync'"
- } else {
- ""
- },
- ssh = q(&context.ssh.to_string_lossy()),
- format = q("%i %n%L %''b/%''l"),
- destination = q(&context.rsync_destination(path)),
- )
-}
-
fn use_state(sentinel: &str, names: &[&str]) -> String {
let mut call = format!("use_state {}", q(sentinel));
for name in names {
@@ -130,541 +78,6 @@ fn use_state(sentinel: &str, names: &[&str]) -> String {
call
}
-/// Delivers a flake or flake-input source named name to path. Returns the
-/// shell code doing it on the deploying machine, in the source's working
-/// directory.
-pub fn flake_input(context: &Context, name: &str, path: &str, source: &Source) -> String {
- let settings = &source.settings;
- let url = str_setting(settings, "url").unwrap_or_default();
- let input = if source.kind == "flake" {
- None
- } else {
- str_setting(settings, "name")
- };
- let input_path: Vec<&str> = input.map(|i| i.split('/').collect()).unwrap_or_default();
- let fetch_on_target = bool_setting(settings, "fetchOnTarget", false);
- let dir = path.rsplit_once('/').map(|(d, _)| d).unwrap_or("");
- let sentinel = format!("{dir}/.populate");
- let derived = derives_flake(&Source {
- kind: "flake".into(),
- settings: settings.clone(),
- });
- let link_path = if derived {
- format!("{sentinel}/derived/{name}/original")
- } else {
- path.to_string()
- };
- let patches = patches(settings);
-
- // Uploads the patches to .populate/patches/<name>, numbered in order.
- let upload_patches = if patches.is_empty() {
- String::new()
- } else {
- let mut text = String::from("rm -fR \"$PWD\"/patches\nmkdir \"$PWD\"/patches\n");
- for (i, patch) in patches.iter().enumerate() {
- let base = patch.rsplit('/').next().unwrap_or(patch);
- text += &format!(
- "cp {} \"$PWD\"/patches/{}\n",
- q(patch),
- q(&format!("{:04}-{base}", i + 1))
- );
- }
- text += &run_shell(
- context,
- &format!(
- "set -efu\n{}\nmkdir -p {}\n",
- use_state(&sentinel, &[name]),
- q(&format!("{sentinel}/patches"))
- ),
- );
- text.push('\n');
- text += &rsync_text(
- context,
- &format!("{sentinel}/patches/{name}"),
- "\"$PWD\"/patches",
- true,
- );
- text
- };
-
- let is_remote_ref = {
- let scheme: String = url
- .chars()
- .take_while(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '+')
- .collect();
- !scheme.is_empty()
- && url[scheme.len()..].starts_with(':')
- && ![
- "file",
- "flake",
- "git+file",
- "hg+file",
- "path",
- "tarball+file",
- ]
- .contains(&scheme.as_str())
- };
-
- let no_input = format!(
- ">&2 printf 'error: flake %s has no input %s\\n' {} {}\nexit 1\n",
- q(url),
- q(input.unwrap_or(""))
- );
-
- let overridden: Vec<Vec<String>> = Source {
- kind: "flake".into(),
- settings: settings.clone(),
- }
- .follows()
- .into_iter()
- .map(|f| f.path)
- .collect();
- let require_inputs = bool_setting(settings, "requireInputs", true);
-
- // Shell function printing the store path $1, followed by the inputs its
- // flake in the subdirectory $2 pins in its own lock file. Without a
- // lock file, locking would change the flake, so there are no such
- // inputs. Inputs that get drawn from other sources are left out, with
- // all they pin: Nix archives an empty flake in their place, so it
- // doesn't fetch them.
- let closure = |nix: &str, mktemp: &str| -> String {
- let overridden_json = serde_json::to_string(&overridden).expect("serializes");
- let mut overrides = String::new();
- if !overridden.is_empty() {
- overrides += &format!(
- "closure_empty=$({mktemp} -d)\necho '{{ outputs = _: {{}}; }}' > \"$closure_empty\"/flake.nix\n"
- );
- for path in &overridden {
- overrides += &format!(
- "set -- \"$@\" --override-input {} \"path:$closure_empty\"\n",
- q(&path.join("/"))
- );
- }
- }
- let on_failure = if require_inputs {
- format!(
- ">&2 printf 'error: source %s: the inputs its own lock file pins can'\"'\"'t be delivered%s\\n' \\\n {} {}\nexit 1\n",
- q(name),
- q(if input.is_some() {
- ". Set its requireInputs to false to deliver it anyway."
- } else {
- ""
- })
- )
- } else {
- format!(
- r#"# Without --no-update-lock-file, Nix adds what the lock file
-# lacks.
-if closure_archive=$(
- {nix} flake archive --json --no-write-lock-file "$@" "$closure_flake"
-); then
- >&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \
- {name}
- closure_paths "$closure_archive"
-else
- >&2 printf 'warning: source %s: the inputs its own lock file pins can'"'"'t be delivered; delivering it without them\n' \
- {name}
-fi
-"#,
- name = q(name)
- )
- };
- format!(
- r#"
-# Prints the store paths in nix flake archive's output $1.
-closure_paths() {{
- ARCHIVE=$1 OVERRIDDEN={overridden_json} \
- {nix} eval --impure --raw --expr '
- let
- overridden = builtins.fromJSON (builtins.getEnv "OVERRIDDEN");
- # Inputs given by relative paths are part of their parent.
- paths = prefix: node: builtins.concatMap
- (name: let
- input = node.inputs.${{name}};
- path = prefix ++ [ name ];
- in
- if builtins.elem path overridden then []
- else
- (if input ? path then [ input.path ] else []) ++
- paths path input)
- (builtins.attrNames (node.inputs or {{}}));
- in
- builtins.concatStringsSep "\n"
- (paths [] (builtins.fromJSON (builtins.getEnv "ARCHIVE")))
- '
- echo
-}}
-closure() {{
- printf '%s\n' "$1"
- closure_dir=$1${{2:+/$2}}
- if test -e "$closure_dir"/flake.nix && test -e "$closure_dir"/flake.lock; then
- closure_flake=path:$1${{2:+?dir=$2}}
- set --
- {overrides}
- if closure_archive=$(
- {nix} flake archive --json --no-update-lock-file "$@" "$closure_flake"
- ); then
- closure_paths "$closure_archive"
- else
- {on_failure}
- fi
- {cleanup}
- fi
-}}
-"#,
- overridden_json = q(&overridden_json),
- cleanup = if overridden.is_empty() {
- ""
- } else {
- "rm -R \"$closure_empty\""
- },
- )
- };
-
- let inputs_roots = format!("{sentinel}/flake-inputs/{name}");
- let inputs_roots_parent = format!("{sentinel}/flake-inputs");
- let uploads = format!("{sentinel}/flake-uploads");
- let upload = format!("{uploads}/{name}");
- let remove_upload = format!(
- r#"remove_tree {upload}
-# Sources get populated one after another, so no other source uses the
-# directory meanwhile.
-if test -d {uploads} && test -z "$(ls -A {uploads})"; then
- rmdir {uploads}
-fi
-"#,
- upload = q(&upload),
- uploads = q(&uploads)
- );
-
- // Reads the subdirectory containing the flake or input, which may be
- // empty, followed by store paths from stdin. Links the first store
- // path, or the subdirectory therein, at the link path, and registers it
- // and the remaining ones as garbage collector roots. Roots get added
- // before obsolete ones get removed, so the inputs stay rooted.
- let link = format!(
- r#"set -efu
-read -r dir
-read -r path
-{use_state}
-{mkdir_derived}
-make_room_for_link {link_path} {link_name}
-nix-store --realise "$path" --add-root {link_path} --indirect >/dev/null
-# The garbage collector follows the link into the store path.
-if test -n "$dir"; then
- ln -fns "$path/$dir" {link_path}
-fi
-
-# Left behind by an interrupted run, or not needed anymore.
-{remove_upload}
-mkdir -p {roots}
-roots=
-while read -r path; do
- if test -z "$path"; then
- continue
- fi
- root=$(basename "$path")
- if test "$(readlink {roots}/"$root")" != "$path"; then
- nix-store --realise "$path" \
- --add-root {roots}/"$root" --indirect \
- >/dev/null
- fi
- roots="$roots $root"
-done
-ls -A {roots} | while read -r root; do
- case " $roots " in
- *" $root "*) ;;
- *) rm {roots}/"$root" ;;
- esac
-done
-# An input without own inputs leaves the directory empty.
-if test -z "$roots"; then
- rmdir {roots}
- if test -z "$(ls -A {roots_parent})"; then
- rmdir {roots_parent}
- fi
-fi
-"#,
- use_state = use_state(&sentinel, &[name]),
- mkdir_derived = if derived {
- format!(
- "mkdir -p {}",
- q(link_path.rsplit_once('/').map(|(d, _)| d).unwrap_or(""))
- )
- } else {
- String::new()
- },
- link_path = q(&link_path),
- link_name = q(&context.name(&link_path)),
- roots = q(&inputs_roots),
- roots_parent = q(&inputs_roots_parent),
- );
-
- // Reads the selection, as selected produces it, and the store path of
- // the flake itself from stdin, fetches what's selected, unless it's the
- // flake itself, which gets added from the upload directory unless the
- // store has it already, then fetches the inputs its own lock file
- // pins, and links the results.
- let fetch_and_link = format!(
- r#"set -efu
-require_flakes 'fetching flake inputs on the target'
-read -r selected
-read -r self
-read -r expected
-if test -n "$self" && ! nix-store --check-validity "$self" 2>/dev/null; then
- name=$(basename "$self" | cut -c34-)
- mv {upload}/tree {upload}/"$name"
- added=$(nix-store --add-fixed --recursive sha256 {upload}/"$name")
- {remove_upload}
- if test "$added" != "$self"; then
- >&2 printf 'error: flake %s arrived as %s instead of %s\n' \
- {url} "$added" "$self"
- exit 1
- fi
-fi
-# Fetching contacts where the input comes from even if the store has what
-# the lock file pins.
-if test -n "$expected" && ! nix-store --check-validity "$expected" 2>/dev/null; then
- expected=
-fi
-# The store path and the subdirectory, on separate lines. Inputs referring
-# to local paths can only be fetched if the target has them as well.
-fetched=$(
- FLAKE={url} INPUT={input} \
- SELECTED=$selected SELF=$self EXPECTED=$expected \
- nix eval --impure --raw --expr '
- let
- selected = builtins.fromJSON (builtins.getEnv "SELECTED");
- expected = builtins.getEnv "EXPECTED";
- selectedPath =
- if selected.self || expected != "" then null
- else localPath selected.locked;
- path =
- if selected.self then builtins.getEnv "SELF"
- else if expected != "" then expected
- else (builtins.fetchTree selected.locked).outPath;
- dir = path + (if selected.subdir == "" then "" else "/" + selected.subdir);
- lock =
- if builtins.pathExists (dir + "/flake.lock")
- then builtins.fromJSON (builtins.readFile (dir + "/flake.lock"))
- else {{ nodes = {{}}; }};
- localPath = locked:
- if (locked.type or null) == "path" then locked.path
- else let
- match = builtins.match "file:(//)?(.*)" (locked.url or "");
- in
- if match == null then null else builtins.elemAt match 1;
- missing = builtins.filter
- (key: let
- p = localPath (lock.nodes.${{key}}.locked or {{}});
- in
- p != null && builtins.substring 0 1 p == "/" && !builtins.pathExists p)
- (builtins.attrNames lock.nodes);
- key = builtins.head missing;
- in
- if selectedPath != null && builtins.substring 0 1 selectedPath == "/" && !builtins.pathExists selectedPath then
- throw "flake ${{builtins.getEnv "FLAKE"}}: input ${{builtins.getEnv "INPUT"}} refers to ${{selectedPath}}, which does not exist on the target"
- else if missing != [] then
- throw "flake ${{builtins.getEnv "FLAKE"}}: input ${{key}} refers to ${{localPath lock.nodes.${{key}}.locked}}, which does not exist on the target"
- else
- path + "\n" + selected.subdir
- '
-)
-path=$(printf '%s\n' "$fetched" | sed -n 1p)
-subdir=$(printf '%s\n' "$fetched" | sed -n 2p)
-{closure}
-# Not in the pipeline, where failing wouldn't fail populate.
-paths=$(closure "$path" "$subdir")
-printf '%s\n%s\n' "$subdir" "$paths" | {{
- {link}
-}}
-"#,
- upload = q(&upload),
- url = q(url),
- input = q(input.unwrap_or("")),
- closure = closure("nix", "mktemp"),
- );
-
- // Fetches the flake and its inputs, and links the results.
- let archive_and_link = format!(
- r#"set -efu
-require_flakes 'fetching flakes on the target'
-metadata=$(nix flake metadata --json --no-update-lock-file {url})
-# The store path and the subdirectory, on separate lines.
-fetched=$(
- METADATA=$metadata nix eval --impure --raw --expr {expr}
-)
-path=$(printf '%s\n' "$fetched" | sed -n 1p)
-subdir=$(printf '%s\n' "$fetched" | sed -n 2p)
-{closure}
-paths=$(closure "$path" "$subdir")
-printf '%s\n%s\n' "$subdir" "$paths" | {{
- {link}
-}}
-"#,
- url = q(url),
- expr = q(&format!(
- "let\n metadata = builtins.fromJSON (builtins.getEnv \"METADATA\");\nin\n metadata.path + \"\\n\" + ({METADATA_SUBDIR}) metadata\n"
- )),
- closure = closure("nix", "mktemp"),
- );
-
- let nix_ = "nix --extra-experimental-features 'nix-command flakes'";
- let selected_jq = format!("{SUBDIRS}{SELECTED}");
- let self_locked_jq = format!("{SUBDIRS}{SELF_LOCKED}");
-
- if fetch_on_target && input.is_none() && is_remote_ref {
- return format!(
- "{upload_patches}\n{}\n",
- run_shell(context, &archive_and_link)
- );
- }
-
- let self_text = if fetch_on_target {
- let upload_self = if context.local {
- String::new()
- } else {
- format!(
- r#"if ! printf '%s\n' "$self" | {{
- {check}
-}}; then
- {prepare}
- # Without applying .rsync-filter files, which would change the flake.
- {rsync}
-fi
-"#,
- check = run_shell(
- context,
- "read -r self\nnix-store --check-validity \"$self\" 2>/dev/null\n"
- ),
- prepare = run_shell(
- context,
- &format!(
- "set -efu\n{}\n{remove_upload}\nmkdir -p {}\n",
- use_state(&sentinel, &[name]),
- q(&upload)
- )
- ),
- rsync = rsync_text(context, &format!("{upload}/tree"), "\"$self\"", false),
- )
- };
- format!(
- r#"# The flake itself has no lock entry, so the target fetches it by its
-# locked reference, if it can, or else receives it.
-if jq -e {self_is_remote} "$cache" >/dev/null; then
- selected=$(
- jq -c --argjson selected "$selected" {self_locked} \
- "$cache"
- )
-else
- self=$(jq -r .path "$cache")
- {upload_self}
-fi
-"#,
- self_is_remote = q(SELF_IS_REMOTE),
- self_locked = q(&self_locked_jq),
- )
- } else {
- "self=$(jq -r .path \"$cache\")\n".to_string()
- };
-
- let deliver = if fetch_on_target {
- format!(
- "printf '%s\\n%s\\n%s\\n' \"$selected\" \"$self\" \"$expected\" | {{\n {}\n}}\n",
- run_shell(context, &fetch_and_link)
- )
- } else {
- let copy = if context.local {
- String::new()
- } else {
- format!(
- "mkdir -p \"$PWD\"/bin\nln -fns {} \"$PWD\"/bin/ssh\nPATH=$PWD/bin:$PATH \\\n{nix_} copy --to {} $paths >&2\n",
- q(&context.ssh_nix.to_string_lossy()),
- q(&format!("ssh://{}", context.target.host))
- )
- };
- format!(
- r#"path=$self
-if test -z "$path"; then
- if test -n "$expected" &&
- nix-store --check-validity "$expected" 2>/dev/null; then
- path=$expected
- else
- path=$(
- SELECTED=$selected {nix_} eval --impure --raw --expr '
- (builtins.fetchTree
- (builtins.fromJSON (builtins.getEnv "SELECTED")).locked).outPath
- '
- )
- fi
-fi
-subdir=$(printf '%s\n' "$selected" | jq -r .subdir)
-{closure}
-paths=$(closure "$path" "$subdir")
-
-{copy}
-printf '%s\n%s\n' "$subdir" "$paths" | {{
- {link}
-}}
-"#,
- closure = closure(nix_, "mktemp"),
- link = run_shell(context, &link),
- )
- };
-
- format!(
- r#"set -efu
-{upload_patches}
-
-cache=$TMPDIR/flake-metadata.$(
- printf %s {url} |
- sha256sum |
- cut -c-32
-).json
-if ! test -e "$cache"; then
- {{
- {nix_} flake metadata --json --no-update-lock-file {url}
- }} > "$cache"
-fi
-
-if ! selected=$(
- jq -ce --argjson input {input_path} {selected} \
- "$cache" 2>/dev/null
-); then
- {no_input}
-fi
-
-self=
-if printf '%s\n' "$selected" | jq -e .self >/dev/null; then
- {self_text}
-fi
-
-# The store path the lock file pins what's selected to, which is the one of
-# a directory named source with the locked hash. A store that has it needs
-# no fetching, which contacts where the input comes from even then.
-expected=
-hash=$(printf '%s\n' "$selected" | jq -r '.locked.narHash // empty')
-case $hash in
- sha256-*)
- expected=$(
- nix-store --print-fixed-path --recursive sha256 "$(
- printf %s "${{hash#sha256-}}" |
- base64 -d |
- od -An -v -tx1 |
- tr -d ' \n'
- )" source
- )
- ;;
-esac
-
-{deliver}
-"#,
- url = q(url),
- input_path = q(&serde_json::to_string(&input_path).expect("serializes")),
- selected = q(&selected_jq),
- )
-}
-
/// Draws inputs of flakes in sources from other sources, and derives what
/// the target paths of flake and flake-input sources that get patched, get
/// their lock files rewritten, or are materialized get. Sources get
diff --git a/pkgs/populate/src/flake_input.rs b/pkgs/populate/src/flake_input.rs
new file mode 100644
index 0000000..0084689
--- /dev/null
+++ b/pkgs/populate/src/flake_input.rs
@@ -0,0 +1,777 @@
+//! Flake and flake-input sources: delivering a flake, or one input of a
+//! flake as its lock file pins it, as a link into the store, with garbage
+//! collector roots for its own inputs.
+
+use crate::flake::derives_flake;
+use crate::rsync::{self, Config};
+use crate::run::{self, Context, Failure, Result};
+use crate::sh;
+use crate::spec::Source;
+use serde_json::{Map, Value};
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+
+/// What a flake-input source delivers: the flake itself, or an input
+/// relative to it, which the target can't fetch on its own, or what
+/// fetchTree takes as locked, and the subdirectory containing the flake.
+#[derive(Clone, Debug, PartialEq)]
+pub struct Selection {
+ pub is_self: bool,
+ pub locked: Option<Value>,
+ pub subdir: String,
+}
+
+impl Selection {
+ fn to_json(&self) -> String {
+ serde_json::json!({
+ "self": self.is_self,
+ "locked": self.locked,
+ "subdir": self.subdir,
+ })
+ .to_string()
+ }
+}
+
+/// The subdirectory a flake reference or lock entry names, as attribute
+/// dir or as query parameter dir of its URL, depending on the version of
+/// Nix. fetchTree supports neither.
+fn subdir(reference: &Value) -> String {
+ if let Some(dir) = reference.get("dir").and_then(Value::as_str) {
+ return dir.to_string();
+ }
+ let url = reference.get("url").and_then(Value::as_str).unwrap_or("");
+ query_dir(url)
+ .map(|(_, dir, _)| dir.to_string())
+ .unwrap_or_default()
+}
+
+/// The first query parameter dir in url: where its separator is, its
+/// value, and where it ends, including a following &.
+fn query_dir(url: &str) -> Option<(usize, &str, usize)> {
+ let bytes = url.as_bytes();
+ (0..bytes.len()).find_map(|i| {
+ if (bytes[i] == b'?' || bytes[i] == b'&') && url[i + 1..].starts_with("dir=") {
+ let start = i + 5;
+ let len = url[start..].find('&').unwrap_or(url.len() - start);
+ let end = start + len;
+ let end_with_amp = if url[end..].starts_with('&') {
+ end + 1
+ } else {
+ end
+ };
+ Some((i, &url[start..end], end_with_amp))
+ } else {
+ None
+ }
+ })
+}
+
+/// The reference without its subdirectory, for fetchTree.
+fn without_subdir(reference: &Value) -> Value {
+ let mut reference = reference.clone();
+ if let Value::Object(object) = &mut reference {
+ object.remove("__final");
+ object.remove("dir");
+ if let Some(Value::String(url)) = object.get_mut("url") {
+ if let Some((i, _, end)) = query_dir(url) {
+ let separator = url[i..i + 1].to_string();
+ url.replace_range(i..end, &separator);
+ }
+ if url.ends_with('?') || url.ends_with('&') {
+ url.pop();
+ }
+ }
+ }
+ reference
+}
+
+/// The subdirectory a node's locked or original reference names.
+fn subdir_of(node: &Value) -> String {
+ ["locked", "original"]
+ .iter()
+ .filter_map(|key| node.get(key))
+ .filter(|reference| !reference.is_null())
+ .map(subdir)
+ .find(|dir| !dir.is_empty())
+ .unwrap_or_default()
+}
+
+/// Selects the flake or input in nix flake metadata's output by the path
+/// of input names input, following the flake's lock file, follows
+/// included. An empty follows path removes an input. Errors say what's
+/// missing.
+pub fn select(metadata: &Value, input: &[&str]) -> std::result::Result<Selection, String> {
+ let locks = &metadata["locks"];
+ let root = locks["root"].as_str().unwrap_or("root");
+ let nodes = &locks["nodes"];
+ fn reference<'a>(
+ nodes: &'a Value,
+ root: &'a str,
+ path: &[&str],
+ depth: usize,
+ ) -> std::result::Result<String, String> {
+ if depth > 64 {
+ return Err("follows nest too deep".to_string());
+ }
+ let mut key = root.to_string();
+ for name in path {
+ let next = &nodes[&key]["inputs"][name];
+ key = match next {
+ Value::String(next) => next.clone(),
+ Value::Array(follows) if !follows.is_empty() => {
+ let follows: Vec<&str> = follows.iter().filter_map(Value::as_str).collect();
+ reference(nodes, root, &follows, depth + 1)?
+ }
+ _ => return Err(format!("no input {name}")),
+ };
+ }
+ Ok(key)
+ }
+ fn join(a: &str, b: &str) -> String {
+ match (a.is_empty(), b.is_empty()) {
+ (true, _) => b.to_string(),
+ (_, true) => a.to_string(),
+ _ => format!("{a}/{b}"),
+ }
+ }
+ fn resolve(
+ metadata: &Value,
+ nodes: &Value,
+ root: &str,
+ key: &str,
+ depth: usize,
+ ) -> std::result::Result<Selection, String> {
+ let node = &nodes[key];
+ if depth > 64 {
+ return Err("parents nest too deep".to_string());
+ }
+ if key == root {
+ return Ok(Selection {
+ is_self: true,
+ locked: None,
+ subdir: subdir_of(metadata),
+ });
+ }
+ if let Some(parent) = node.get("parent").and_then(Value::as_array) {
+ let parent: Vec<&str> = parent.iter().filter_map(Value::as_str).collect();
+ let base = resolve(
+ metadata,
+ nodes,
+ root,
+ &reference(nodes, root, &parent, 0)?,
+ depth + 1,
+ )?;
+ let path = node["locked"]["path"].as_str().unwrap_or("");
+ let path = path.strip_prefix("./").unwrap_or(path);
+ let path = path.strip_suffix('/').unwrap_or(path);
+ let path = if path == "." { "" } else { path };
+ let subdir = join(&join(&base.subdir, path), &subdir_of(node));
+ return Ok(Selection { subdir, ..base });
+ }
+ Ok(Selection {
+ is_self: false,
+ locked: Some(without_subdir(&node["locked"])),
+ subdir: subdir_of(node),
+ })
+ }
+ let key = reference(nodes, root, input, 0)?;
+ resolve(metadata, nodes, root, &key, 0)
+}
+
+/// Whether the target can fetch the flake itself, i.e. whether its locked
+/// reference doesn't refer to the deploying machine.
+pub fn self_is_remote(metadata: &Value) -> bool {
+ let locked = &metadata["locked"];
+ locked["type"].as_str() != Some("path")
+ && !locked["url"].as_str().unwrap_or("").starts_with("file:")
+}
+
+/// The store paths in nix flake archive's output, but not those of inputs
+/// at the paths overridden, nor what's below them. Inputs given by
+/// relative paths are part of their parent.
+pub fn archived_paths(archive: &Value, overridden: &[Vec<String>]) -> Vec<String> {
+ fn walk(node: &Value, prefix: &[String], overridden: &[Vec<String>], out: &mut Vec<String>) {
+ let Some(inputs) = node.get("inputs").and_then(Value::as_object) else {
+ return;
+ };
+ for (name, input) in inputs {
+ let mut path = prefix.to_vec();
+ path.push(name.clone());
+ if overridden.contains(&path) {
+ continue;
+ }
+ if let Some(store_path) = input.get("path").and_then(Value::as_str) {
+ out.push(store_path.to_string());
+ }
+ walk(input, &path, overridden, out);
+ }
+ }
+ let mut out = Vec::new();
+ walk(archive, &[], overridden, &mut out);
+ out
+}
+
+/// The hexadecimal SHA-256 hash in a narHash like sha256-<base64>.
+fn nar_hash_hex(hash: &str) -> Option<String> {
+ let encoded = hash.strip_prefix("sha256-")?;
+ let alphabet = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+ let mut bits = 0u32;
+ let mut count = 0;
+ let mut out = String::new();
+ for byte in encoded.bytes().filter(|b| *b != b'=') {
+ let value = alphabet.iter().position(|a| *a == byte)? as u32;
+ bits = (bits << 6) | value;
+ count += 6;
+ if count >= 8 {
+ count -= 8;
+ out += &format!("{:02x}", (bits >> count) & 0xff);
+ }
+ }
+ Some(out)
+}
+
+/// A Nix command with flakes enabled.
+fn nix(args: &[&str]) -> Command {
+ let mut command = Command::new("nix");
+ command
+ .args(["--extra-experimental-features", "nix-command flakes"])
+ .args(args);
+ command
+}
+
+/// Whether the store on the deploying machine has path.
+fn valid_here(path: &str) -> bool {
+ Command::new("nix-store")
+ .args(["--check-validity", path])
+ .stderr(Stdio::null())
+ .status()
+ .is_ok_and(|status| status.success())
+}
+
+/// The shell code of snippet for the target, with the definitions it uses,
+/// enabling flakes for what if given.
+fn on_target(snippet: &str, flakes: Option<&str>) -> String {
+ let require = flakes
+ .map(|what| format!("require_flakes {}\n", run::quote(what)))
+ .unwrap_or_default();
+ format!(
+ "{}{}{require}{snippet}",
+ sh::prelude(),
+ sh::flake::REMOVE_UPLOAD
+ )
+}
+
+fn setting<'a>(settings: &'a Map<String, Value>, key: &str) -> Option<&'a Value> {
+ settings.get(key)
+}
+
+fn flag(settings: &Map<String, Value>, key: &str, default: bool) -> bool {
+ setting(settings, key)
+ .and_then(Value::as_bool)
+ .unwrap_or(default)
+}
+
+/// Delivers the flake or flake-input source named name to path, preparing
+/// what it needs on the deploying machine in workdir.
+pub fn deliver(
+ context: &Context,
+ name: &str,
+ path: &str,
+ source: &Source,
+ workdir: &Path,
+) -> Result<()> {
+ let settings = &source.settings;
+ let url = setting(settings, "url")
+ .and_then(Value::as_str)
+ .unwrap_or_default();
+ let input = if source.kind == "flake" {
+ None
+ } else {
+ setting(settings, "name").and_then(Value::as_str)
+ };
+ let input_path: Vec<&str> = input.map(|i| i.split('/').collect()).unwrap_or_default();
+ let fetch_on_target = flag(settings, "fetchOnTarget", false);
+ let dir = path.rsplit_once('/').map(|(d, _)| d).unwrap_or("");
+ let sentinel = format!("{dir}/.populate");
+ let as_flake = Source {
+ kind: "flake".into(),
+ settings: settings.clone(),
+ };
+ // A source that gets patched, gets its lock file rewritten, or is
+ // materialized gets linked in .populate/derived/<name> instead of at
+ // the target path, and the flakes step derives what the target path
+ // gets.
+ let derived = derives_flake(&as_flake);
+ let link_path = if derived {
+ format!("{sentinel}/derived/{name}/original")
+ } else {
+ path.to_string()
+ };
+ let overridden: Vec<Vec<String>> = as_flake.follows().into_iter().map(|f| f.path).collect();
+ let require_inputs = flag(settings, "requireInputs", true);
+ let hint = if input.is_some() {
+ ". Set its requireInputs to false to deliver it anyway."
+ } else {
+ ""
+ };
+ let failure = |message: String| Failure::Message(message);
+
+ upload_patches(context, name, &sentinel, settings, workdir)?;
+
+ // The store paths of the flake or input, followed by the inputs its
+ // flake in subdir pins in its own lock file, archived where script
+ // runs.
+ let closure = |store_path: &str, subdir: &str, on: &dyn Fn(&str, &[&str]) -> Result<String>| {
+ let overrides: Vec<String> = overridden.iter().map(|p| p.join("/")).collect();
+ let mut args = vec![
+ store_path,
+ subdir,
+ if require_inputs { "1" } else { "" },
+ name,
+ hint,
+ ];
+ args.extend(overrides.iter().map(String::as_str));
+ let archive = on(sh::flake::ARCHIVE, &args)?;
+ let mut paths = vec![store_path.to_string()];
+ if !archive.trim().is_empty() {
+ let archive: Value = serde_json::from_str(&archive).map_err(|e| {
+ failure(format!(
+ "error: can't read what nix flake archive prints: {e}"
+ ))
+ })?;
+ paths.extend(archived_paths(&archive, &overridden));
+ }
+ Ok::<_, Failure>(paths)
+ };
+ let link = |subdir: &str, paths: &[String]| -> Result<()> {
+ let link_name = context.name(&link_path);
+ let mut args = vec![
+ sentinel.as_str(),
+ name,
+ &link_path,
+ &link_name,
+ if derived { "1" } else { "" },
+ subdir,
+ ];
+ args.extend(paths.iter().map(String::as_str));
+ context.run(&on_target(sh::flake::LINK, None), &args)
+ };
+ let on_target_output = |what: &'static str| {
+ move |snippet: &str, args: &[&str]| context.output(&on_target(snippet, Some(what)), args)
+ };
+
+ let is_remote_ref = {
+ let scheme: String = url
+ .chars()
+ .take_while(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == '+')
+ .collect();
+ !scheme.is_empty()
+ && url[scheme.len()..].starts_with(':')
+ && ![
+ "file",
+ "flake",
+ "git+file",
+ "hg+file",
+ "path",
+ "tarball+file",
+ ]
+ .contains(&scheme.as_str())
+ };
+
+ // The target fetches the flake itself on its own.
+ if fetch_on_target && input.is_none() && is_remote_ref {
+ let fetching = "fetching flakes on the target";
+ let metadata = context.output(
+ &on_target(
+ "nix flake metadata --json --no-update-lock-file \"$1\"\n",
+ Some(fetching),
+ ),
+ &[url],
+ )?;
+ let metadata: Value = serde_json::from_str(&metadata)
+ .map_err(|e| failure(format!("error: can't read flake metadata of {url}: {e}")))?;
+ let store_path = metadata["path"].as_str().unwrap_or_default();
+ let subdir = subdir_of(&metadata);
+ let paths = closure(store_path, &subdir, &on_target_output(fetching))?;
+ return link(&subdir, &paths);
+ }
+
+ let metadata = context.flake_metadata(url, || {
+ run::output(&mut nix(&[
+ "flake",
+ "metadata",
+ "--json",
+ "--no-update-lock-file",
+ url,
+ ]))
+ })?;
+ let no_input = || {
+ failure(format!(
+ "error: flake {url} has no input {}",
+ input.unwrap_or("")
+ ))
+ };
+ let mut selected = select(&metadata, &input_path).map_err(|_| no_input())?;
+
+ let mut self_path = String::new();
+ if selected.is_self {
+ if fetch_on_target && self_is_remote(&metadata) {
+ // The flake itself has no lock entry, so the target fetches it
+ // by its locked reference.
+ selected = Selection {
+ is_self: false,
+ locked: Some(without_subdir(&metadata["locked"])),
+ ..selected
+ };
+ } else {
+ self_path = metadata["path"].as_str().unwrap_or_default().to_string();
+ if fetch_on_target && !context.local {
+ upload_self(context, name, &sentinel, &self_path)?;
+ }
+ }
+ }
+
+ // The store path the lock file pins what's selected to, which is the
+ // one of a directory named source with the locked hash. A store that
+ // has it needs no fetching, which contacts where the input comes from
+ // even then.
+ let mut expected = String::new();
+ let nar_hash = selected
+ .locked
+ .as_ref()
+ .and_then(|locked| locked.get("narHash"))
+ .and_then(Value::as_str)
+ .and_then(nar_hash_hex);
+ if let Some(hex) = nar_hash {
+ expected = run::output(Command::new("nix-store").args([
+ "--print-fixed-path",
+ "--recursive",
+ "sha256",
+ &hex,
+ "source",
+ ]))?
+ .trim_end()
+ .to_string();
+ }
+
+ if fetch_on_target {
+ let fetching = "fetching flake inputs on the target";
+ let fetched = context.output(
+ &on_target(sh::flake::FETCH, Some(fetching)),
+ &[
+ &sentinel,
+ name,
+ url,
+ input.unwrap_or(""),
+ &selected.to_json(),
+ &self_path,
+ &expected,
+ ],
+ )?;
+ let (store_path, subdir) = fetched.split_once('\n').unwrap_or((&fetched, ""));
+ let paths = closure(store_path, subdir, &on_target_output(fetching))?;
+ return link(subdir, &paths);
+ }
+
+ let store_path = if !self_path.is_empty() {
+ self_path
+ } else if !expected.is_empty() && valid_here(&expected) {
+ expected
+ } else {
+ let locked = selected.locked.clone().unwrap_or(Value::Null).to_string();
+ run::output(
+ nix(&[
+ "eval",
+ "--impure",
+ "--raw",
+ "--expr",
+ "(builtins.fetchTree (builtins.fromJSON (builtins.getEnv \"LOCKED\"))).outPath",
+ ])
+ .env("LOCKED", locked),
+ )?
+ };
+ let here = |snippet: &str, args: &[&str]| {
+ let mut command = run::local_shell(snippet, args, workdir);
+ command.env("NIX_CONFIG", nix_config());
+ run::output(&mut command)
+ };
+ let paths = closure(&store_path, &selected.subdir, &here)?;
+ if !context.local {
+ copy(context, workdir, &paths)?;
+ }
+ link(&selected.subdir, &paths)
+}
+
+/// NIX_CONFIG with flakes enabled.
+fn nix_config() -> String {
+ let config = std::env::var("NIX_CONFIG").unwrap_or_default();
+ format!("{config}\nextra-experimental-features = nix-command flakes")
+}
+
+/// Copies store paths to the target with nix copy, which runs ssh from
+/// PATH, so that's the ssh for Nix's ssh:// store there.
+fn copy(context: &Context, workdir: &Path, paths: &[String]) -> Result<()> {
+ let bin = workdir.join("bin");
+ let link = bin.join("ssh");
+ let prepare = || -> std::io::Result<()> {
+ fs::create_dir_all(&bin)?;
+ if fs::symlink_metadata(&link).is_ok() {
+ fs::remove_file(&link)?;
+ }
+ std::os::unix::fs::symlink(&context.ssh_nix, &link)
+ };
+ prepare()
+ .map_err(|e| Failure::Message(format!("error: can't prepare {}: {e}", bin.display())))?;
+ let path = std::env::var_os("PATH").unwrap_or_default();
+ let path =
+ std::env::join_paths(std::iter::once(bin.clone()).chain(std::env::split_paths(&path)))
+ .map_err(|e| Failure::Message(format!("error: can't extend PATH: {e}")))?;
+ let mut command = nix(&["copy", "--to", &format!("ssh://{}", context.target.host)]);
+ command.args(paths).env("PATH", path).stdout(run::stderr());
+ run::status(&mut command)
+}
+
+/// Uploads the source's patches to .populate/patches/<name>, numbered in
+/// order.
+fn upload_patches(
+ context: &Context,
+ name: &str,
+ sentinel: &str,
+ settings: &Map<String, Value>,
+ workdir: &Path,
+) -> Result<()> {
+ let patches: Vec<&str> = settings
+ .get("patches")
+ .and_then(Value::as_array)
+ .into_iter()
+ .flatten()
+ .filter_map(Value::as_str)
+ .collect();
+ if patches.is_empty() {
+ return Ok(());
+ }
+ let dir = workdir.join("patches");
+ let prepare = || -> std::io::Result<()> {
+ if dir.exists() {
+ fs::remove_dir_all(&dir)?;
+ }
+ fs::create_dir(&dir)?;
+ for (i, patch) in patches.iter().enumerate() {
+ let base = Path::new(patch)
+ .file_name()
+ .unwrap_or_default()
+ .to_string_lossy();
+ fs::copy(patch, dir.join(format!("{:04}-{base}", i + 1)))?;
+ }
+ Ok(())
+ };
+ prepare().map_err(|e| Failure::Message(format!("error: can't copy patches: {e}")))?;
+ context.run(&on_target(sh::flake::PREPARE, None), &[sentinel, name, "1"])?;
+ let config = Config {
+ filter_files: false,
+ use_checksum: true,
+ ..Config::default()
+ };
+ rsync::transfer(
+ context,
+ &config,
+ &dir.to_string_lossy(),
+ &format!("{sentinel}/patches/{name}"),
+ )
+}
+
+/// Uploads the flake itself to a target that can't fetch it, unless the
+/// target's store has it already.
+fn upload_self(context: &Context, name: &str, sentinel: &str, self_path: &str) -> Result<()> {
+ if context.run(sh::flake::VALID, &[self_path]).is_ok() {
+ return Ok(());
+ }
+ run::check_signal()?;
+ context.run(&on_target(sh::flake::PREPARE, None), &[sentinel, name, ""])?;
+ // Without applying .rsync-filter files, which would change the flake.
+ let config = Config {
+ filter_files: false,
+ ..Config::default()
+ };
+ let upload: PathBuf = Path::new(sentinel)
+ .join("flake-uploads")
+ .join(name)
+ .join("tree");
+ rsync::transfer(context, &config, self_path, &upload.to_string_lossy())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ /// A flake with an input dep, which has an input inner following the
+ /// flake's input inner, and an input sub, a flake in a subdirectory.
+ fn metadata(locked: Value, original: Value) -> Value {
+ json!({
+ "locked": { "type": "path", "path": "/src/top", "narHash": "sha256-top" },
+ "path": "/nix/store/top-source",
+ "locks": {
+ "root": "root",
+ "version": 7,
+ "nodes": {
+ "root": { "inputs": { "dep": "dep", "inner": "inner", "sub": "sub" } },
+ "dep": {
+ "inputs": { "inner": [ "inner" ] },
+ "locked": { "type": "tarball", "url": "http://x/dep.tar.gz", "narHash": "sha256-dep" }
+ },
+ "inner": {
+ "locked": { "type": "tarball", "url": "http://x/inner.tar.gz", "narHash": "sha256-inner" }
+ },
+ "sub": { "locked": locked, "original": original }
+ }
+ }
+ })
+ }
+
+ fn attribute() -> Value {
+ metadata(
+ json!({ "type": "tarball", "url": "http://x/mono.tar.gz", "dir": "sub", "narHash": "sha256-mono" }),
+ json!({ "type": "tarball", "url": "http://x/mono.tar.gz", "dir": "sub" }),
+ )
+ }
+
+ fn mono() -> Selection {
+ Selection {
+ is_self: false,
+ locked: Some(
+ json!({ "type": "tarball", "url": "http://x/mono.tar.gz", "narHash": "sha256-mono" }),
+ ),
+ subdir: "sub".into(),
+ }
+ }
+
+ #[test]
+ fn selection() {
+ let url = metadata(
+ json!({ "type": "tarball", "url": "http://x/mono.tar.gz?dir=sub", "narHash": "sha256-mono" }),
+ json!({ "type": "tarball", "url": "http://x/mono.tar.gz?dir=sub" }),
+ );
+ assert_eq!(select(&attribute(), &["sub"]), Ok(mono()));
+ assert_eq!(select(&url, &["sub"]), Ok(mono()));
+ assert_eq!(
+ select(&attribute(), &[]),
+ Ok(Selection {
+ is_self: true,
+ locked: None,
+ subdir: String::new()
+ })
+ );
+ assert_eq!(
+ select(&attribute(), &["dep", "inner"]).map(|s| s.locked),
+ Ok(Some(
+ json!({ "type": "tarball", "url": "http://x/inner.tar.gz", "narHash": "sha256-inner" })
+ ))
+ );
+ assert_eq!(select(&attribute(), &["nope"]), Err("no input nope".into()));
+ }
+
+ #[test]
+ fn removed_and_cyclic_follows() {
+ let mut removed = attribute();
+ removed["locks"]["nodes"]["dep"]["inputs"]["inner"] = json!([]);
+ assert_eq!(
+ select(&removed, &["dep", "inner"]),
+ Err("no input inner".into())
+ );
+ let mut cyclic = attribute();
+ cyclic["locks"]["nodes"]["dep"]["inputs"]["inner"] = json!(["inner", "x"]);
+ cyclic["locks"]["nodes"]["inner"]["inputs"] = json!({ "x": ["dep", "inner"] });
+ assert_eq!(
+ select(&cyclic, &["dep", "inner"]),
+ Err("follows nest too deep".into())
+ );
+ }
+
+ #[test]
+ fn relative_path() {
+ let mut relative = attribute();
+ relative["locked"]["dir"] = json!("sub");
+ relative["locks"]["nodes"]["root"]["inputs"]["lib"] = json!("lib");
+ relative["locks"]["nodes"]["lib"] =
+ json!({ "locked": { "type": "path", "path": "./lib" }, "parent": [] });
+ assert_eq!(
+ select(&relative, &["lib"]),
+ Ok(Selection {
+ is_self: true,
+ locked: None,
+ subdir: "sub/lib".into()
+ })
+ );
+ }
+
+ #[test]
+ fn remote() {
+ let remote = |locked: Value| self_is_remote(&json!({ "locked": locked }));
+ assert!(!remote(json!({ "type": "path", "path": "/src" })));
+ assert!(!remote(json!({ "type": "git", "url": "file:///src" })));
+ assert!(remote(json!({ "type": "git", "url": "https://x/y" })));
+ assert!(remote(
+ json!({ "type": "github", "owner": "o", "repo": "r" })
+ ));
+ }
+
+ #[test]
+ fn locked_self() {
+ let locked = json!({ "__final": true, "type": "tarball", "url": "http://x/mono.tar.gz?dir=sub", "narHash": "sha256-mono" });
+ assert_eq!(Some(without_subdir(&locked)), mono().locked);
+ let url = json!({ "url": "http://x/y?rev=1&dir=sub&z=2" });
+ assert_eq!(
+ without_subdir(&url),
+ json!({ "url": "http://x/y?rev=1&z=2" })
+ );
+ }
+
+ #[test]
+ fn subdirs() {
+ let of = |m: Value| subdir_of(&m);
+ assert_eq!(
+ of(json!({
+ "locked": { "type": "tarball", "url": "http://x/mono.tar.gz?dir=sub&narHash=sha256-mono" },
+ "original": { "type": "tarball", "url": "http://x/mono.tar.gz?dir=sub" }
+ })),
+ "sub"
+ );
+ assert_eq!(
+ of(json!({
+ "locked": { "type": "tarball", "url": "http://x/mono.tar.gz" },
+ "original": { "type": "tarball", "url": "http://x/mono.tar.gz?rev=1&dir=sub" }
+ })),
+ "sub"
+ );
+ assert_eq!(
+ of(json!({ "locked": { "type": "path", "path": "/src/top" } })),
+ ""
+ );
+ }
+
+ #[test]
+ fn archive() {
+ let archive = json!({
+ "path": "/nix/store/top",
+ "inputs": {
+ "a": { "path": "/nix/store/a", "inputs": { "b": { "path": "/nix/store/b" } } },
+ "c": { "path": "/nix/store/c", "inputs": { "d": { "path": "/nix/store/d" } } },
+ "rel": { "inputs": {} }
+ }
+ });
+ assert_eq!(
+ archived_paths(&archive, &[vec!["c".into()]]),
+ vec!["/nix/store/a", "/nix/store/b"]
+ );
+ }
+
+ #[test]
+ fn nar_hashes() {
+ assert_eq!(
+ nar_hash_hex("sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=").as_deref(),
+ Some("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+ );
+ }
+}
diff --git a/pkgs/populate/src/main.rs b/pkgs/populate/src/main.rs
index c1e389a..17d9678 100644
--- a/pkgs/populate/src/main.rs
+++ b/pkgs/populate/src/main.rs
@@ -3,6 +3,7 @@
mod check;
mod flake;
+mod flake_input;
mod populate;
mod rsync;
mod run;
diff --git a/pkgs/populate/src/populate.rs b/pkgs/populate/src/populate.rs
index ef806b0..ad0ec35 100644
--- a/pkgs/populate/src/populate.rs
+++ b/pkgs/populate/src/populate.rs
@@ -1,6 +1,7 @@
//! Delivering the sources of a spec.
use crate::flake;
+use crate::flake_input;
use crate::rsync::{self, Config};
use crate::run::{self, Context, Failure, Result};
use crate::sh;
@@ -132,11 +133,7 @@ fn deliver(
};
rsync::transfer(context, &config, &out.to_string_lossy(), path)
}
- "flake" | "flake-input" => run::status(&mut run::local_shell(
- &flake::flake_input(context, name, path, source),
- &[],
- &workdir,
- )),
+ "flake" | "flake-input" => flake_input::deliver(context, name, path, source, &workdir),
"git" => {
let flag = |key| match settings.get(key).and_then(Value::as_bool) {
Some(true) => "1",
diff --git a/pkgs/populate/src/run.rs b/pkgs/populate/src/run.rs
index 7525cc0..528977c 100644
--- a/pkgs/populate/src/run.rs
+++ b/pkgs/populate/src/run.rs
@@ -102,6 +102,9 @@ pub struct Context {
/// because Nix waits for its LocalCommand, which may not run over a
/// multiplexed connection.
pub ssh_nix: PathBuf,
+ /// What nix flake metadata prints for flakes, by reference, so it's
+ /// obtained only once, even if several sources use the same flake.
+ metadata: std::cell::RefCell<std::collections::BTreeMap<String, serde_json::Value>>,
}
impl Context {
@@ -127,6 +130,7 @@ impl Context {
tmpdir,
ssh,
ssh_nix,
+ metadata: Default::default(),
};
context.write_ssh(&context.ssh, &context.ssh_options(true))?;
context.write_ssh(&context.ssh_nix, &context.ssh_options(false))?;
@@ -231,6 +235,25 @@ impl Context {
status(&mut command)
}
+ /// What nix flake metadata prints for the flake url, as get gets it,
+ /// once.
+ pub fn flake_metadata(
+ &self,
+ url: &str,
+ get: impl FnOnce() -> Result<String>,
+ ) -> Result<serde_json::Value> {
+ if let Some(metadata) = self.metadata.borrow().get(url) {
+ return Ok(metadata.clone());
+ }
+ let metadata: serde_json::Value = serde_json::from_str(&get()?).map_err(|e| {
+ Failure::Message(format!("error: can't read flake metadata of {url}: {e}"))
+ })?;
+ self.metadata
+ .borrow_mut()
+ .insert(url.to_string(), metadata.clone());
+ Ok(metadata)
+ }
+
/// Runs shell code on the target, returning its output.
pub fn output(&self, script: &str, args: &[&str]) -> Result<String> {
output(&mut self.shell(script, args))
diff --git a/pkgs/populate/src/sh.rs b/pkgs/populate/src/sh.rs
index 5605699..1a2d2c7 100644
--- a/pkgs/populate/src/sh.rs
+++ b/pkgs/populate/src/sh.rs
@@ -471,7 +471,8 @@ while read -r age_path; do
mkdir -p "$(dirname "$pass_out")"
if ! passage show "$pass_name" > "$pass_out" 2>&1; then
echo ++ secrets "$host" show "$pass_name" >&2
- jq -Rr 'select(test("^[^[]"))' "$pass_out" >&2
+ # What passage says, without age's progress lines.
+ grep '^[^[]' "$pass_out" >&2 || :
exit 1
fi
touch -r "$age_path" "$pass_out"
@@ -533,3 +534,199 @@ pub fn prelude() -> String {
]
.concat()
}
+
+/// Shell code for flake and flake-input sources on the target. Each gets
+/// SENTINEL and NAME, the source's name, first, and the definitions of
+/// prelude.
+pub mod flake {
+ /// Defines remove_upload, which removes the directory receiving the
+ /// flake itself, and the directory of uploads once it's empty.
+ /// Sources get populated one after another, so no other source uses
+ /// that meanwhile.
+ pub const REMOVE_UPLOAD: &str = r#"
+remove_upload() {
+ remove_tree "$sentinel"/flake-uploads/"$name"
+ if test -d "$sentinel"/flake-uploads &&
+ test -z "$(ls -A "$sentinel"/flake-uploads)"; then
+ rmdir "$sentinel"/flake-uploads
+ fi
+}
+"#;
+
+ /// prepare_upload SENTINEL NAME: makes the directory receiving the
+ /// flake itself, or the patches if PATCHES isn't empty.
+ pub const PREPARE: &str = r#"
+sentinel=$1 name=$2 patches=$3
+use_state "$sentinel" "$name"
+if test -n "$patches"; then
+ mkdir -p "$sentinel"/patches
+else
+ remove_upload
+ mkdir -p "$sentinel"/flake-uploads/"$name"
+fi
+"#;
+
+ /// valid PATH: succeeds if the store has PATH.
+ pub const VALID: &str = r#"
+nix-store --check-validity "$1" 2>/dev/null
+"#;
+
+ /// archive PATH SUBDIR REQUIRE NAME HINT OVERRIDDEN...: prints what
+ /// nix flake archive prints for the flake in SUBDIR of the store path
+ /// PATH, with an empty flake in place of each input at an OVERRIDDEN
+ /// path, so Nix doesn't fetch them, or nothing if the flake has no
+ /// lock file, since locking would change it. If its inputs can't be
+ /// archived, it fails if REQUIRE isn't empty, saying so for the source
+ /// NAME, with HINT appended, or else archives what Nix locks now, or
+ /// prints nothing, with a warning.
+ pub const ARCHIVE: &str = r#"
+path=$1 subdir=$2 require=$3 name=$4 hint=$5
+shift 5
+dir=$path${subdir:+/$subdir}
+if test -e "$dir"/flake.nix && test -e "$dir"/flake.lock; then
+ flake=path:$path${subdir:+?dir=$subdir}
+ empty=
+ if test $# -gt 0; then
+ empty=$(mktemp -d)
+ echo '{ outputs = _: {}; }' > "$empty"/flake.nix
+ n=$#
+ while test "$n" -gt 0; do
+ set -- "$@" --override-input "$1" "path:$empty"
+ shift
+ n=$((n - 1))
+ done
+ fi
+ if nix flake archive --json --no-update-lock-file "$@" "$flake"; then
+ :
+ elif test -n "$require"; then
+ >&2 printf 'error: source %s: the inputs its own lock file pins can'"'"'t be delivered%s\n' \
+ "$name" "$hint"
+ exit 1
+ # Without --no-update-lock-file, Nix adds what the lock file lacks.
+ elif nix flake archive --json --no-write-lock-file "$@" "$flake"; then
+ >&2 printf 'warning: source %s: its own lock file needs to be updated; delivering it with the inputs Nix locks now, which evaluating it may not use\n' \
+ "$name"
+ else
+ >&2 printf 'warning: source %s: the inputs its own lock file pins can'"'"'t be delivered; delivering it without them\n' \
+ "$name"
+ fi
+ if test -n "$empty"; then
+ rm -R "$empty"
+ fi
+fi
+"#;
+
+ /// link SENTINEL NAME LINK_PATH LINK_NAME DERIVED SUBDIR PATH ROOT...:
+ /// links the store path PATH, or SUBDIR therein, at LINK_PATH, and
+ /// registers it and the store paths ROOT as garbage collector roots of
+ /// the source NAME. Roots get added before obsolete ones get removed,
+ /// so the inputs stay rooted. LINK_NAME is LINK_PATH's name in
+ /// messages, and DERIVED isn't empty if LINK_PATH is in the source's
+ /// derived state.
+ pub const LINK: &str = r#"
+sentinel=$1 name=$2 link_path=$3 link_name=$4 derived=$5 subdir=$6 path=$7
+shift 7
+roots_dir=$sentinel/flake-inputs/$name
+use_state "$sentinel" "$name"
+if test -n "$derived"; then
+ mkdir -p "$(dirname "$link_path")"
+fi
+make_room_for_link "$link_path" "$link_name"
+nix-store --realise "$path" --add-root "$link_path" --indirect >/dev/null
+# The garbage collector follows the link into the store path.
+if test -n "$subdir"; then
+ ln -fns "$path/$subdir" "$link_path"
+fi
+# Left behind by an interrupted run, or not needed anymore.
+remove_upload
+mkdir -p "$roots_dir"
+roots=
+for path; do
+ root=$(basename "$path")
+ if test "$(readlink "$roots_dir"/"$root")" != "$path"; then
+ nix-store --realise "$path" --add-root "$roots_dir"/"$root" --indirect >/dev/null
+ fi
+ roots="$roots $root"
+done
+ls -A "$roots_dir" | while read -r root; do
+ case " $roots " in
+ *" $root "*) ;;
+ *) rm "$roots_dir"/"$root" ;;
+ esac
+done
+# An input without own inputs leaves the directory empty.
+if test -z "$roots"; then
+ rmdir "$roots_dir"
+ if test -z "$(ls -A "$sentinel"/flake-inputs)"; then
+ rmdir "$sentinel"/flake-inputs
+ fi
+fi
+"#;
+
+ /// fetch SENTINEL NAME URL INPUT SELECTED SELF EXPECTED: prints the
+ /// store path of what the selection SELECTED, as JSON, names, and the
+ /// subdirectory therein, on separate lines. That's SELF, the flake
+ /// itself, which gets added from the upload directory unless the store
+ /// has it already, or EXPECTED, the store path the lock file pins it
+ /// to, if the store has it, or else what fetching it gets. Inputs
+ /// referring to local paths can only be fetched if the target has them
+ /// as well. URL and INPUT name the flake and the input in messages.
+ pub const FETCH: &str = r#"
+sentinel=$1 name=$2 url=$3 input=$4 selected=$5 self=$6 expected=$7
+upload=$sentinel/flake-uploads/$name
+if test -n "$self" && ! nix-store --check-validity "$self" 2>/dev/null; then
+ store_name=$(basename "$self" | cut -c34-)
+ mv "$upload"/tree "$upload"/"$store_name"
+ added=$(nix-store --add-fixed --recursive sha256 "$upload"/"$store_name")
+ remove_upload
+ if test "$added" != "$self"; then
+ >&2 printf 'error: flake %s arrived as %s instead of %s\n' \
+ "$url" "$added" "$self"
+ exit 1
+ fi
+fi
+# Fetching contacts where the input comes from even if the store has what
+# the lock file pins.
+if test -n "$expected" && ! nix-store --check-validity "$expected" 2>/dev/null; then
+ expected=
+fi
+FLAKE=$url INPUT=$input SELECTED=$selected SELF=$self EXPECTED=$expected \
+nix eval --impure --raw --expr '
+ let
+ selected = builtins.fromJSON (builtins.getEnv "SELECTED");
+ expected = builtins.getEnv "EXPECTED";
+ selectedPath =
+ if selected.self || expected != "" then null
+ else localPath selected.locked;
+ path =
+ if selected.self then builtins.getEnv "SELF"
+ else if expected != "" then expected
+ else (builtins.fetchTree selected.locked).outPath;
+ dir = path + (if selected.subdir == "" then "" else "/" + selected.subdir);
+ lock =
+ if builtins.pathExists (dir + "/flake.lock")
+ then builtins.fromJSON (builtins.readFile (dir + "/flake.lock"))
+ else { nodes = {}; };
+ localPath = locked:
+ if (locked.type or null) == "path" then locked.path
+ else let
+ match = builtins.match "file:(//)?(.*)" (locked.url or "");
+ in
+ if match == null then null else builtins.elemAt match 1;
+ missing = builtins.filter
+ (key: let
+ p = localPath (lock.nodes.${key}.locked or {});
+ in
+ p != null && builtins.substring 0 1 p == "/" && !builtins.pathExists p)
+ (builtins.attrNames lock.nodes);
+ key = builtins.head missing;
+ in
+ if selectedPath != null && builtins.substring 0 1 selectedPath == "/" && !builtins.pathExists selectedPath then
+ throw "flake ${builtins.getEnv "FLAKE"}: input ${builtins.getEnv "INPUT"} refers to ${selectedPath}, which does not exist on the target"
+ else if missing != [] then
+ throw "flake ${builtins.getEnv "FLAKE"}: input ${key} refers to ${localPath lock.nodes.${key}.locked}, which does not exist on the target"
+ else
+ path + "\n" + selected.subdir
+'
+"#;
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-11-14 16:09:23 +0000