diff options
Diffstat (limited to 'pkgs/populate/src/flake.rs')
| -rw-r--r-- | pkgs/populate/src/flake.rs | 1351 |
1 files changed, 752 insertions, 599 deletions
diff --git a/pkgs/populate/src/flake.rs b/pkgs/populate/src/flake.rs index f03e443..b6b0790 100644 --- a/pkgs/populate/src/flake.rs +++ b/pkgs/populate/src/flake.rs @@ -1,37 +1,20 @@ -//! Drawing inputs of flakes in sources from other sources. +//! Drawing inputs of flakes in sources from other sources, and deriving +//! what flake and flake-input sources that get patched, get their lock +//! files rewritten, or are materialized deliver. //! -//! This builds the same shell code as shell.nix generates, with every value -//! quoted, so both implementations behave alike while the code moves to -//! Rust piece by piece. +//! Populate decides what to do. The target runs the fixed snippets of +//! sh::draw, which send back what populate needs to know: where paths +//! resolve to, the lock files of flakes, and what tells whether a derived +//! copy needs to be made again. -use crate::run::{quote as q, Context}; +use crate::run::{Context, Failure, Result}; use crate::sh; use crate::spec::{Follows, Source}; use serde_json::{Map, Value}; use std::collections::{BTreeMap, BTreeSet}; -/// Nix function returning 1 if the lock file locks has an input at path, a -/// list of input names, resolving inputs that follow others, and else 0. -/// Like lockHasInput in flake-jq.nix, for the shell code drawing inputs. -pub const LOCK_HAS_INPUT: &str = r#"locks: path: let - nodes = locks.nodes; - input = key: name: (nodes.${key}.inputs or {}).${name} or null; - # The node an input refers to, following follows from the root. - node = ref: - if ref == null then null - else if builtins.isList ref then - builtins.foldl' - (key: name: if key == null then null else node (input key name)) - locks.root - ref - else ref; - has = key: path: - if path == [] then true - else let next = node (input key (builtins.head path)); in - next != null && has next (builtins.tail path); -in - if has locks.root path then "1" else "0" -"#; +/// What needs flakes, for the error if Nix on the target lacks them. +const DRAWING: &str = "drawing flake inputs from sources"; fn bool_setting(settings: &Map<String, Value>, key: &str, default: bool) -> bool { settings @@ -69,630 +52,676 @@ pub fn derives_flake(source: &Source) -> bool { || (bool_setting(settings, "rewriteLock", true) && !source.follows().is_empty())) } -fn use_state(sentinel: &str, names: &[&str]) -> String { - let mut call = format!("use_state {}", q(sentinel)); - for name in names { - call.push(' '); - call.push_str(&q(name)); +/// A path on the target, where it resolves to, and what's there. +#[derive(Debug, Default)] +struct Resolved { + path: String, + dir: bool, + git: bool, + submodules: bool, + flake: bool, +} + +/// What a path is in a URL: with every byte percent-encoded that isn't a +/// letter, a digit, or one of -._~/. Nix decodes that, and otherwise +/// takes, e.g., ? and # for delimiters, and refuses blanks. +fn url_path(path: &str) -> String { + path.bytes() + .map(|b| { + if b.is_ascii_alphanumeric() || b"-._~/".contains(&b) { + (b as char).to_string() + } else { + format!("%{b:02x}") + } + }) + .collect() +} + +/// Source names become input names in generated flakes, where Nix accepts +/// fewer names than populate. Others get replaced by one derived from +/// them. +fn input_name(name: &str) -> String { + let ok = name.chars().next().is_some_and(|c| c.is_ascii_alphabetic()) + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + && name != "self"; + if ok { + name.to_string() + } else { + format!("source-{}", &sha256_hex(name.as_bytes())[..16]) } - call } -/// Draws inputs of flakes in sources from other sources, and derives what -/// the target paths of flake and flake-input sources that get patched, get -/// their lock files rewritten, or are materialized get. Sources get -/// processed in the order they draw inputs from each other. Returns the -/// shell code to run on the target, which also removes state that sources -/// don't need anymore. -/// -/// Inputs get drawn by rewriting the lock file of the flake in the source -/// with nix flake lock --override-input: in place for file and git sources, -/// and in a copy for flake and flake-input sources. Sources with -/// rewriteLock set to false, and symlink sources, get a generated flake in -/// .populate/flakes/<name> instead, which re-exports the outputs of the -/// flake in the source, with those inputs drawn from the other sources. -/// Lock files get rewritten or generated on every run, so they match the -/// sources. -pub fn flakes(context: &Context, dir: &str, sources: &BTreeMap<String, Source>) -> String { - let sentinel = format!("{dir}/.populate"); - let flakes_dir = format!("{sentinel}/flakes"); - let derived_dir = format!("{sentinel}/derived"); - let patches_dir = format!("{sentinel}/patches"); - let follows: BTreeMap<&str, Vec<Follows>> = sources - .iter() - .map(|(n, s)| (n.as_str(), s.follows())) - .collect(); - let drawing: BTreeSet<&str> = follows - .iter() - .filter(|(_, f)| !f.is_empty()) - .map(|(n, _)| *n) - .collect(); - let generated: BTreeSet<&str> = sources - .iter() - .filter(|(_, s)| has_generated_flake(s)) - .map(|(n, _)| n.as_str()) - .collect(); - let derived: BTreeSet<&str> = sources - .iter() - .filter(|(_, s)| derives_flake(s)) - .map(|(n, _)| n.as_str()) - .collect(); - let patched: BTreeSet<&str> = sources - .iter() - .filter(|(_, s)| !patches(&s.settings).is_empty()) - .map(|(n, _)| n.as_str()) - .collect(); - let names: BTreeSet<&str> = drawing.union(&derived).copied().collect(); +/// The follows of a source that draws the input at path on the way to it, +/// if any: the one with the longest path that path starts with. +fn parent_of<'a>(follows: &'a [Follows], path: &[String]) -> Option<&'a Follows> { + (1..path.len()) + .rev() + .find_map(|n| follows.iter().find(|f| f.path == path[..n])) +} - // Source names become input names in generated flakes, where Nix - // accepts fewer names than populate. Others get replaced by one - // derived from them. - let input_name = |name: &str| -> String { - let ok = name.chars().next().is_some_and(|c| c.is_ascii_alphabetic()) - && name - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') - && name != "self"; - if ok { - name.to_string() - } else { - format!("source-{}", &sha256_hex(name.as_bytes())[..16]) - } - }; - let drawn_from = - |name: &str| -> Vec<String> { follows[name].iter().map(|f| f.source.clone()).collect() }; - let sorted = toposort(&names, &drawn_from); +/// Where the input at path of the flake in the source named name is in the +/// flake that the source's inputs get drawn into: the flake itself, or its +/// generated flake. On the way, inputs that get drawn from sources are the +/// flakes of those sources, where drawing went on, so the path continues +/// there. A generated flake has its source and the sources that one draws +/// from as inputs, and nothing else, so the path names them there. +fn drawn_path( + name: &str, + path: &[String], + follows: &BTreeMap<&str, Vec<Follows>>, + generated: &BTreeSet<&str>, +) -> Vec<String> { + let parent = parent_of(&follows[name], path); + let rest = |p: &Follows| drawn_path(&p.source, &path[p.path.len()..], follows, generated); + match (generated.contains(name), parent) { + (true, None) => [vec![input_name(name)], path.to_vec()].concat(), + (true, Some(p)) => [vec![input_name(&p.source)], rest(p)].concat(), + (false, None) => path.to_vec(), + (false, Some(p)) => [p.path.clone(), rest(p)].concat(), + } +} - // Shell word for the URL of the flake that inputs drawn from a source - // use: its generated flake, if it has one, so overrides don't need to - // be repeated, or else the flake in the source. Nix refuses path: URLs - // with symlinks on the way, which the target directory may be, so paths - // get resolved. - let url = |name: &str| -> String { - if generated.contains(name) { - format!( - "\"path:$(url_path \"$(readlink -f {})\")\"", - q(&format!("{flakes_dir}/{name}")) - ) - } else { - format!("\"$(flake_url {})\"", q(name)) - } - }; +/// The inputs that drawing the input at path of the flake in the source +/// named name needs to exist, as (name, path) of the source whose flake +/// has the input at path. +fn input_checks( + name: &str, + path: &[String], + follows: &BTreeMap<&str, Vec<Follows>>, +) -> Vec<(String, Vec<String>)> { + match parent_of(&follows[name], path) { + None => vec![(name.to_string(), path.to_vec())], + Some(p) => [ + vec![(name.to_string(), p.path.clone())], + input_checks(&p.source, &path[p.path.len()..], follows), + ] + .concat(), + } +} - // Where the input at path of the flake in the source named name is in - // the flake that the source's inputs get drawn into: the flake itself, - // or its generated flake. On the way, inputs that get drawn from - // sources are the flakes of those sources, where drawing went on, so - // the path continues there. A generated flake has its source and the - // sources that one draws from as inputs, and nothing else, so the path - // names them there. - fn parent_of<'a>(follows: &'a [Follows], path: &[String]) -> Option<&'a Follows> { - (1..path.len()) - .rev() - .find_map(|n| follows.iter().find(|f| f.path == path[..n])) +/// Whether the lock file locks, as nix flake metadata prints it, has an +/// input at path, a list of input names, resolving inputs that follow +/// others. +fn lock_has_input(locks: &Value, path: &[String]) -> bool { + let nodes = &locks["nodes"]; + let Some(root) = locks["root"].as_str() else { + return false; + }; + let input = |key: &str, name: &str| -> Option<&Value> { + Some(&nodes[key]["inputs"][name]).filter(|input| !input.is_null()) + }; + // The node an input refers to, following follows from the root. + fn node<'a>( + reference: Option<&'a Value>, + root: &'a str, + input: &dyn Fn(&str, &str) -> Option<&'a Value>, + depth: usize, + ) -> Option<&'a str> { + if depth > 64 { + return None; + } + match reference? { + Value::String(key) => Some(key), + Value::Array(names) => names.iter().try_fold(root, |key, name| { + node(input(key, name.as_str()?), root, input, depth + 1) + }), + _ => None, + } } - fn drawn_path( - name: &str, - path: &[String], - follows: &BTreeMap<&str, Vec<Follows>>, - generated: &BTreeSet<&str>, - input_name: &dyn Fn(&str) -> String, - ) -> Vec<String> { - let parent = parent_of(&follows[name], path); - match (generated.contains(name), parent) { - (true, None) => [vec![input_name(name)], path.to_vec()].concat(), - (true, Some(p)) => [ - vec![input_name(&p.source)], - drawn_path( - &p.source, - &path[p.path.len()..], - follows, - generated, - input_name, - ), - ] - .concat(), - (false, None) => path.to_vec(), - (false, Some(p)) => [ - p.path.clone(), - drawn_path( - &p.source, - &path[p.path.len()..], - follows, - generated, - input_name, - ), - ] - .concat(), + let mut key = root; + for name in path { + match node(input(key, name), root, &input, 0) { + Some(next) => key = next, + None => return false, } } - let drawn_path = - |name: &str, path: &[String]| drawn_path(name, path, &follows, &generated, &input_name); + true +} - // The inputs that drawing the input at path of the flake in the source - // named name needs to exist, as (name, path) of the source whose flake - // has the input at path. - fn input_checks( - name: &str, - path: &[String], - follows: &BTreeMap<&str, Vec<Follows>>, - ) -> Vec<(String, Vec<String>)> { - match parent_of(&follows[name], path) { - None => vec![(name.to_string(), path.to_vec())], - Some(p) => [ - vec![(name.to_string(), p.path.clone())], - input_checks(&p.source, &path[p.path.len()..], follows), - ] - .concat(), - } +/// The store path a path below the store is in, and the rest of the path. +fn split_store_path(path: &str) -> (&str, &str) { + let end = path + .match_indices('/') + .nth(3) + .map(|(i, _)| i) + .unwrap_or(path.len()); + let rest = &path[end..]; + (&path[..end], rest.strip_prefix('/').unwrap_or(rest)) +} + +/// The sources, and what drawing inputs and deriving copies needs to know +/// about them. +struct Flakes<'a> { + context: &'a Context, + dir: &'a str, + sources: &'a BTreeMap<String, Source>, + follows: BTreeMap<&'a str, Vec<Follows>>, + /// The sources drawing inputs from others. + drawing: BTreeSet<&'a str>, + /// The sources with a generated flake. + generated: BTreeSet<&'a str>, + /// The sources with derived state. + derived: BTreeSet<&'a str>, +} + +impl<'a> Flakes<'a> { + fn sentinel(&self) -> String { + format!("{}/.populate", self.dir) } - // Sets the positional parameters to nix flake lock's arguments for - // drawing the inputs of the flake in source name from other sources. - let override_args = |name: &str| -> String { - let mut text = String::from("set --\n"); - for f in &follows[name] { - text += &format!( - "set -- \"$@\" --override-input {} {}\n", - q(&drawn_path(name, &f.path).join("/")), - url(&f.source) - ); - } - text - }; + fn state(&self, kind: &str, name: &str) -> String { + format!("{}/{kind}/{name}", self.sentinel()) + } - // Rewrites the lock file of the flake in a file or git source in place. - let rewrite = |name: &str| -> String { - format!( - "(\n{}own_lock \"$(readlink -f {})\"\nnix flake lock \"$@\" \"$(flake_url {})\" >&2\n)\n", - override_args(name), - q(&format!("{dir}/{name}")), - q(name) - ) - }; + fn path(&self, name: &str) -> String { + format!("{}/{name}", self.dir) + } - // Derives what the target path of a flake or flake-input source gets - // from what got delivered to .populate/derived/<name>/original: a copy, - // which gets patched and gets its lock file rewritten. Unless - // materialized, the copy gets added to the store, and the target path - // becomes a link to it. Copies get made again only if what they derive - // from changes: materialized ones if the delivered store path, the - // patches, or what inputs get drawn from which sources changes, and - // others also if the sources they draw inputs from change. - let derive = |name: &str| -> String { - let source = &sources[name]; - let settings = &source.settings; - let materialize = bool_setting(settings, "materialize", false); - let rewrite_lock = bool_setting(settings, "rewriteLock", true) && drawing.contains(name); - let target_path = format!("{dir}/{name}"); - let dir_ = format!("{derived_dir}/{name}"); - let has_patches = !patches(settings).is_empty(); - let follows_json: Vec<Value> = follows[name] - .iter() - .map(|f| serde_json::json!({ "path": f.path, "source": f.source })) - .collect(); - // Like Nix' toJSON: attributes sorted by name. - let spec = serde_json::to_string(&serde_json::json!({ - "follows": follows_json, - "materialize": materialize, - "rewriteLock": rewrite_lock, - })) - .expect("serializes"); - let patch_dir = format!("{patches_dir}/{name}"); - let apply_patches = if has_patches { - format!( - r#"ls {pd} | while read -r patch; do - if ! patch -p1 -s --batch --no-backup-if-mismatch \ - -i {pd}/"$patch" >&2; then - >&2 printf 'error: patch %s does not apply to source %s\n' \ - "$(printf %s "$patch" | cut -c6-)" {name} - exit 1 - fi -done -"#, - pd = q(&patch_dir), - name = q(name) - ) + /// The shell code of snippet for the target, with the definitions it + /// uses, enabling flakes if flakes is set. + fn on_target(snippet: &str, flakes: bool) -> String { + let require = if flakes { + format!("require_flakes {}\n", crate::run::quote(DRAWING)) } else { String::new() }; - let remove_work = format!("remove_tree {}", q(&format!("{dir_}/work"))); - let key_patches = if has_patches { - format!( - "ls {pd} | while read -r patch; do\n sha256sum {pd}/\"$patch\"\ndone\n", - pd = q(&patch_dir) - ) - } else { - String::new() - }; - let key_drawn = if rewrite_lock && !materialize { - let mut text = override_args(name); - text += "printf '%s\\n' \"$@\"\n# Sources outside the store can change without changing their\n# path.\n"; - let mut seen = BTreeSet::new(); - for source in drawn_from(name) { - if !seen.insert(source.clone()) { - continue; - } - let source = source.as_str(); - let p = if generated.contains(source) { - format!("{flakes_dir}/{source}") - } else { - format!("{dir}/{source}") - }; - text += &format!( - "drawn=$(readlink -f {})\ncase $drawn in\n /nix/store/*) ;;\n *) nix-hash --type sha256 \"$drawn\" ;;\nesac\n", - q(&p) - ); - } - text - } else { - String::new() - }; - let body = if materialize { - let relock = if rewrite_lock { - format!( - "{}own_lock {}\nnix flake lock \"$@\" \"$(flake_url {})\" >&2\n", - override_args(name), - q(&target_path), - q(name) - ) - } else { - String::new() - }; - format!( - r#"if ! test -e {key} || - test "$(cat {key})" != "$key" || - test -L {tp} || - ! test -d {tp}; then - cp -R "$original" {work} - chmod -R u+w {work} - ( - cd {work} - {apply_patches} - ) - remove_foreign_link {tp} {tp_name} - rm -fR {tp} - mv {work} {tp} - write_key "$key" -fi -{relock} -"#, - key = q(&format!("{dir_}/key")), - tp = q(&target_path), - tp_name = q(&context.name(&target_path)), - work = q(&format!("{dir_}/work")), - ) - } else { - let relock = if rewrite_lock { - format!( - "{}# Not just the path, which Nix takes for the enclosing Git\n# work tree, if any.\nnix flake lock \"$@\" \"path:$(url_path \"$work\")${{subdir:+?dir=$subdir}}\" >&2\n", - override_args(name) - ) - } else { - String::new() - }; - format!( - r#"if test -e {key} && - test "$(cat {key})" = "$key" && - result=$(readlink {result}) && - nix-store --check-validity "$result" 2>/dev/null; then - : -else - # The copy keeps the name of the delivered store path. - mkdir {work} - work=$(readlink -f {work})/$(basename "$path" | cut -c34-) - cp -R "$path" "$work" - chmod -R u+w "$work" - ( - cd "$work"/"$subdir" - {apply_patches} - ) - {relock} - result=$(nix-store --add-fixed --recursive sha256 "$work") - {remove_work} - nix-store --realise "$result" \ - --add-root {result} --indirect >/dev/null - write_key "$key" -fi -make_room_for_link {tp} {tp_name} -# .populate/derived/<name>/result roots the store path. -ln -fns "$result${{subdir:+/$subdir}}" {tp} -"#, - key = q(&format!("{dir_}/key")), - result = q(&format!("{dir_}/result")), - work = q(&format!("{dir_}/work")), - tp = q(&target_path), - tp_name = q(&context.name(&target_path)), - ) - }; format!( - r#"( - # Writes the key $1 to a new file, which then replaces the key, so - # nothing at its place gets written through, e.g. a symlink. - write_key() {{ - key_file=$(mktemp {key}.XXXXXX) - printf '%s\n' "$1" > "$key_file" - mv -f "$key_file" {key} - }} - original=$(readlink -f {original}) - path=$(printf '%s\n' "$original" | cut -d/ -f1-4) - subdir=${{original#"$path"}} - subdir=${{subdir#/}} - key=$( - {{ - printf '%s\n' "$original" {spec} - {key_patches} - {key_drawn} - }} | sha256sum | cut -c-64 - ) - {remove_work} - {body} -) -"#, - key = q(&format!("{dir_}/key")), - original = q(&format!("{dir_}/original")), - spec = q(&spec), + "{}{}{require}{snippet}", + sh::prelude(), + sh::draw::DEFINITIONS ) - }; + } - // Generates the flake of a source in .populate/flakes/<name>. - let generate = |name: &str| -> String { - let mut drawn: Vec<&str> = Vec::new(); - for f in &follows[name] { - if !drawn.contains(&f.source.as_str()) { - drawn.push(&f.source); - } + fn run(&self, snippet: &str, flakes: bool, args: &[&str]) -> Result<()> { + self.context.run(&Self::on_target(snippet, flakes), args) + } + + /// What snippet prints, as the fields ending in null bytes. + fn fields(&self, snippet: &str, flakes: bool, args: &[&str]) -> Result<Vec<String>> { + let output = self + .context + .output(&Self::on_target(snippet, flakes), args)?; + let mut fields: Vec<String> = output.split('\0').map(str::to_string).collect(); + // What follows the last null byte. + fields.pop(); + Ok(fields) + } + + /// Resolves paths on the target. + fn resolve(&self, paths: &[String], flakes: bool) -> Result<Vec<Resolved>> { + let args: Vec<&str> = paths.iter().map(String::as_str).collect(); + let fields = self.fields(sh::draw::RESOLVE, flakes, &args)?; + if fields.len() != 2 * paths.len() { + return Err(Failure::Message( + "error: can't resolve paths on the target".to_string(), + )); } - let json = |s: &str| serde_json::to_string(s).expect("serializes"); - // Nix ignores overrides below an input that follows another input, - // so an input of an input drawn from a source gets overridden at - // that source. - let override_line = |f: &Follows| -> String { - format!( - "inputs.{}.follows = {};", - drawn_path(name, &f.path) - .iter() - .map(|n| json(n)) - .collect::<Vec<_>>() - .join(".inputs."), - json(&input_name(&f.source)) - ) - }; - let mut text = format!("(\nurl=$(flake_url {})\n", q(name)); - for (i, source) in drawn.iter().enumerate() { - text += &format!("url{}={}\n", i + 1, url(source)); + Ok(fields + .chunks(2) + .map(|field| Resolved { + path: field[1].clone(), + dir: field[0].contains('d'), + git: field[0].contains('g'), + submodules: field[0].contains('s'), + flake: field[0].contains('f'), + }) + .collect()) + } + + /// The flake URL of the source named name, at resolved. + fn flake_url(name: &str, resolved: &Resolved) -> Result<String> { + if !resolved.dir { + return Err(Failure::Message(format!( + "error: source {name} is not a directory" + ))); } - let fd = q(&format!("{flakes_dir}/{name}")); - text += &format!("rm -fR {fd}\nmkdir {fd}\ncd {fd}\n{{\n echo '{{'\n"); - text += &format!( - " printf ' inputs.%s.url = \"%s\";\\n' {} \"$url\"\n", - q(&json(&input_name(name))) - ); - for (i, source) in drawn.iter().enumerate() { - text += &format!( - " printf ' inputs.%s.url = \"%s\";\\n' {} \"$url{}\"\n", - q(&json(&input_name(source))), - i + 1 - ); - if !generated.contains(source) { - text += &format!( - " if ! test -e \"$(readlink -f {})\"/flake.nix; then\n printf ' inputs.%s.flake = false;\\n' {}\n fi\n", - q(&format!("{dir}/{source}")), - q(&json(&input_name(source))) - ); + let path = url_path(&resolved.path); + Ok(if resolved.git && resolved.submodules { + format!("git+file:{path}?submodules=1") + } else if resolved.git { + format!("git+file:{path}") + } else { + format!("path:{path}") + }) + } + + /// The flake URL of the source named name, as it is now. + fn own_url(&self, name: &str) -> Result<String> { + let resolved = self.resolve(&[self.path(name)], false)?; + Self::flake_url(name, &resolved[0]) + } + + /// The sources that the source named name draws inputs from, each + /// once. + fn drawn_from(&self, name: &str) -> Vec<&'a str> { + let mut drawn: Vec<&'a str> = Vec::new(); + for f in &self.follows[name] { + let source = self + .sources + .get_key_value(&f.source) + .map(|(k, _)| k.as_str()); + if let Some(source) = source.filter(|s| !drawn.contains(s)) { + drawn.push(source); } } - for f in &follows[name] { - text += &format!(" printf ' %s\\n' {}\n", q(&override_line(f))); - } - text += &format!( - " printf ' outputs = inputs: inputs.%s;\\n' {}\n echo '}}'\n}} > flake.nix\n", - q(&json(&input_name(name))) - ); - text += "# Not just ., which Nix takes for the enclosing Git work tree, if any.\nnix flake lock \"path:$(url_path \"$(pwd -P)\")\" >&2\n)\n"; - text - }; + drawn + } - let process = |name: &str| -> String { - let mut text = String::new(); - if derived.contains(name) { - text += &derive(name); - } - if generated.contains(name) { - text += &generate(name); - } else if drawing.contains(name) && !derived.contains(name) { - text += &rewrite(name); + /// Where the flake that inputs drawn from the source named name use + /// is: its generated flake, if it has one, so overrides don't need to + /// be repeated, or else the flake in the source. + fn drawn_flake(&self, name: &str) -> String { + if self.generated.contains(name) { + self.state("flakes", name) + } else { + self.path(name) } - text - }; + } - // Removes the entries of the state directory that sources named names - // don't need anymore, and the directory once it's empty. - let remove_stale = |state_dir: &str, names: &BTreeSet<&str>| -> String { - let mut call = format!("remove_others {} ''", q(state_dir)); - for name in names { - call += &format!(" {}", q(name)); + /// The URLs of the flakes that inputs drawn from the sources use, and + /// whether they have a flake.nix. Nix refuses path: URLs with + /// symlinks on the way, which the target directory may be, so paths + /// get resolved. + fn drawn_urls(&self, sources: &[&str]) -> Result<BTreeMap<String, (String, bool)>> { + let paths: Vec<String> = sources.iter().map(|s| self.drawn_flake(s)).collect(); + let resolved = self.resolve(&paths, false)?; + let mut urls = BTreeMap::new(); + for (source, resolved) in sources.iter().zip(resolved) { + let url = if self.generated.contains(source) { + format!("path:{}", url_path(&resolved.path)) + } else { + Self::flake_url(source, &resolved)? + }; + // A generated flake is a flake. + let flake = resolved.flake || self.generated.contains(source); + urls.insert(source.to_string(), (url, flake)); } - format!( - "if test -d {d}; then\n {call}\n if test -z \"$(ls -A {d})\"; then\n rmdir {d}\n fi\nfi\n", - d = q(state_dir) - ) - }; + Ok(urls) + } - let all_names: Vec<&str> = sources.keys().map(String::as_str).collect(); - let mut text = format!("set -efu\n{}\n", use_state(&sentinel, &all_names)); - if !names.is_empty() { - if !drawing.is_empty() { - text += "require_flakes 'drawing flake inputs from sources'\n"; + /// nix flake lock's arguments for drawing the inputs of the flake in + /// the source named name from other sources. + fn override_args(&self, name: &str) -> Result<Vec<String>> { + let urls = self.drawn_urls(&self.drawn_from(name))?; + let mut args = Vec::new(); + for f in &self.follows[name] { + args.push("--override-input".to_string()); + args.push(drawn_path(name, &f.path, &self.follows, &self.generated).join("/")); + args.push(urls[&f.source].0.clone()); } - text += &format!( - r#" -# Nix rewrites lock files in place, which also changes what else is a hard -# link to the lock file, e.g. in a backup, or what it is a symlink to. So -# the lock file of the flake in the directory $1 gets replaced by a copy -# first. The copy goes to a new file, so nothing that's there already gets -# written through. -own_lock() {{ - if test -e "$1"/flake.lock; then - own_lock_copy=$(mktemp "$1"/.flake.lock.XXXXXX) - cp -p "$1"/flake.lock "$own_lock_copy" - mv -f "$own_lock_copy" "$1"/flake.lock - fi -}} - -# Prints the flake URL of the source named $1. -flake_url() {{ - path=$(readlink -f {dir}/"$1") - if ! test -d "$path"; then - >&2 printf 'error: source %s is not a directory\n' "$1" - exit 1 - fi - if test -e "$path"/.git; then - if test -e "$path"/.gitmodules; then - printf 'git+file:%s?submodules=1' "$(url_path "$path")" - else - printf 'git+file:%s' "$(url_path "$path")" - fi - else - printf 'path:%s' "$(url_path "$path")" - fi -}} + Ok(args) + } -# Inputs can only be drawn into flakes, so check all sources before -# changing anything. -missing= -"#, - dir = q(dir) - ); - for name in &drawing { - let p = if derived.contains(name) { - format!("{derived_dir}/{name}/original") + /// Fails unless each source drawing inputs contains a flake that has + /// the inputs drawn, before anything changes. Nix ignores inputs drawn + /// that a flake doesn't have, with a warning, so a misspelt input name + /// would draw nothing. + fn check(&self) -> Result<()> { + // The flake in a source as delivered. + let delivered = |name: &str| { + if self.derived.contains(name) { + format!("{}/original", self.state("derived", name)) } else { - format!("{dir}/{name}") - }; - text += &format!( - "if ! test -e {}/flake.nix; then\n missing=\"$missing \"{}\nfi\n", - q(&p), - q(name) - ); + self.path(name) + } + }; + let drawing: Vec<&str> = self.drawing.iter().copied().collect(); + let paths: Vec<String> = drawing.iter().map(|n| delivered(n)).collect(); + let resolved = self.resolve(&paths, true)?; + let missing: Vec<String> = drawing + .iter() + .zip(&resolved) + .filter(|(_, resolved)| !resolved.flake) + .map(|(name, _)| { + format!("error: source {name} draws flake inputs from other sources, but contains no flake.nix") + }) + .collect(); + if !missing.is_empty() { + return Err(Failure::Message(missing.join("\n"))); } - let derived_names: Vec<&str> = derived.iter().copied().collect(); - text += &format!( - r#"if test -n "$missing"; then - >&2 printf 'error: source %s draws flake inputs from other sources, but contains no flake.nix\n' $missing - exit 1 -fi -# Nix ignores inputs drawn that the flake doesn't have, with a warning, so a -# misspelt input name would draw nothing. Prints the flake URL of the -# flake in the source named $1 as delivered. -own_flake_url() {{ - case " "{derived_names}" " in - *" $1 "*) - printf 'path:%s' "$(url_path "$(readlink -f {derived_dir}/"$1"/original)")" - ;; - *) - flake_url "$1" - ;; - esac -}} -# The lock files of the flakes in the sources, as Nix locks them now, with -# an empty flake in place of each input drawn from a source, which may not -# be reachable, as the one the flake's lock file pins. -locks_dir=$(mktemp -d) -echo '{{ outputs = _: {{}}; }}' > "$locks_dir"/flake.nix -"#, - derived_names = q(&derived_names.join(" ")), - derived_dir = q(&derived_dir), - ); + // The lock files of the flakes in the sources, as Nix locks them + // now, with an empty flake in place of each input drawn from a + // source, which may not be reachable, as the one the flake's lock + // file pins. let mut locked: Vec<String> = Vec::new(); for name in &drawing { - for f in &follows[*name] { - for (c, _) in input_checks(name, &f.path, &follows) { + for f in &self.follows[name] { + for (c, _) in input_checks(name, &f.path, &self.follows) { if !locked.contains(&c) { locked.push(c); } } } } - for name in &locked { - text += "set --\n"; - for f in &follows[name.as_str()] { - text += &format!( - "set -- \"$@\" --override-input {} \"path:$locks_dir\"\n", - q(&f.path.join("/")) - ); - } - text += &format!( - r#"if ! nix flake metadata --json --no-write-lock-file "$@" \ - "$(own_flake_url {n})" > "$locks_dir"/{n}.json; then - >&2 printf 'error: source %s draws inputs from other sources, but its flake can'"'"'t be locked\n' \ - {n} - exit 1 -fi -"#, - n = q(name) - ); + let paths: Vec<String> = locked.iter().map(|n| delivered(n)).collect(); + let resolved = self.resolve(&paths, false)?; + let mut locks = BTreeMap::new(); + for (name, resolved) in locked.iter().zip(&resolved) { + let url = if self.derived.contains(name.as_str()) { + format!("path:{}", url_path(&resolved.path)) + } else { + Self::flake_url(name, resolved)? + }; + let overrides: Vec<String> = self.follows[name.as_str()] + .iter() + .map(|f| f.path.join("/")) + .collect(); + let mut args = vec![url.as_str(), name.as_str()]; + args.extend(overrides.iter().map(String::as_str)); + let output = self + .context + .output(&Self::on_target(sh::draw::METADATA, true), &args)?; + let metadata: Value = serde_json::from_str(&output).map_err(|e| { + Failure::Message(format!("error: can't read flake metadata of {url}: {e}")) + })?; + locks.insert(name.clone(), metadata["locks"].clone()); } - text += &format!( - r#"# Fails, naming the input, unless the flake in the source named $1 has -# an input at the path $2, a JSON list of input names. $3 is the source -# that the input $4 of the source $5 gets drawn from. -require_input() {{ - if ! test "$( - METADATA=$(cat "$locks_dir/$1.json") INPUT=$2 nix eval --impure --raw --expr {expr} - )" = 1; then - if test "$1" = "$5"; then - >&2 printf 'error: source %s draws input %s from source %s, but its flake has no input %s\n' \ - "$5" "$4" "$3" "$4" - else - >&2 printf 'error: source %s draws input %s from source %s, but the flake of source %s has no input %s\n' \ - "$5" "$4" "$3" "$1" "$(printf %s "$2" | tr -d '[]"' | tr , /)" - fi - exit 1 - fi -}} -"#, - expr = q(&format!( - "({LOCK_HAS_INPUT})\n (builtins.fromJSON (builtins.getEnv \"METADATA\")).locks\n (builtins.fromJSON (builtins.getEnv \"INPUT\"))\n" - )) - ); + for name in &drawing { - for f in &follows[*name] { - for (c, path) in input_checks(name, &f.path, &follows) { - text += &format!( - "require_input {} {} \\\n {} {} {}\n", - q(&c), - q(&serde_json::to_string(&path).expect("serializes")), - q(&f.source), - q(&f.path.join("/")), - q(name) - ); + for f in &self.follows[name] { + let input = f.path.join("/"); + for (c, path) in input_checks(name, &f.path, &self.follows) { + if lock_has_input(&locks[&c], &path) { + continue; + } + let source = &f.source; + return Err(Failure::Message(if c == *name { + format!("error: source {name} draws input {input} from source {source}, but its flake has no input {input}") + } else { + format!( + "error: source {name} draws input {input} from source {source}, but the flake of source {c} has no input {}", + path.join("/") + ) + })); } } } - text += "rm -R \"$locks_dir\"\n\n"; - if !generated.is_empty() { - text += &format!("mkdir -p {}\n", q(&flakes_dir)); + Ok(()) + } + + /// Rewrites the lock file of the flake in a file or git source in + /// place. + fn rewrite(&self, name: &str) -> Result<()> { + let args = self.override_args(name)?; + let resolved = self.resolve(&[self.path(name)], false)?; + let url = Self::flake_url(name, &resolved[0])?; + let mut all = vec![resolved[0].path.as_str(), url.as_str()]; + all.extend(args.iter().map(String::as_str)); + self.run(sh::draw::LOCK, true, &all) + } + + /// Derives what the target path of a flake or flake-input source gets + /// from what got delivered to .populate/derived/<name>/original: a + /// copy, which gets patched and gets its lock file rewritten. Unless + /// materialized, the copy gets added to the store, and the target path + /// becomes a link to it. Copies get made again only if what they + /// derive from changes: materialized ones if the delivered store path, + /// the patches, or what inputs get drawn from which sources changes, + /// and others also if the sources they draw inputs from change. The + /// key of a copy, in .populate/derived/<name>/key, is the hash of all + /// that. + fn derive(&self, name: &str) -> Result<()> { + let settings = &self.sources[name].settings; + let materialize = bool_setting(settings, "materialize", false); + let rewrite_lock = + bool_setting(settings, "rewriteLock", true) && self.drawing.contains(name); + let target_path = self.path(name); + let target_name = self.context.name(&target_path); + let state = self.state("derived", name); + let patch_dir = if patches(settings).is_empty() { + String::new() + } else { + self.state("patches", name) + }; + let args = if rewrite_lock { + self.override_args(name)? + } else { + Vec::new() + }; + // Sources outside the store can change without changing their + // path. + let drawn: Vec<String> = if rewrite_lock && !materialize { + self.drawn_from(name) + .iter() + .map(|s| self.drawn_flake(s)) + .collect() + } else { + Vec::new() + }; + + let mut state_args = vec![state.as_str(), target_path.as_str(), patch_dir.as_str()]; + state_args.extend(drawn.iter().map(String::as_str)); + let fields = self.fields(sh::draw::DERIVED_STATE, false, &state_args)?; + let malformed = + || Failure::Message(format!("error: can't read the state of source {name}")); + let (head, rest) = fields.split_at_checked(4).ok_or_else(malformed)?; + let [original, key_file, result, target_kind] = head else { + return Err(malformed()); + }; + let separator = rest + .iter() + .position(String::is_empty) + .ok_or_else(malformed)?; + let (patch_sums, hashes) = (&rest[..separator], &rest[separator + 1..]); + if hashes.len() != drawn.len() { + return Err(malformed()); + } + + // Like Nix' toJSON: attributes sorted by name. + let follows: Vec<Value> = self.follows[name] + .iter() + .map(|f| serde_json::json!({ "path": f.path, "source": f.source })) + .collect(); + let spec = serde_json::json!({ + "follows": follows, + "materialize": materialize, + "rewriteLock": rewrite_lock, + }); + let mut text = format!("{original}\n{spec}\n"); + for sum in patch_sums { + text += &format!("{sum}\n"); + } + if !drawn.is_empty() { + for arg in &args { + text += &format!("{arg}\n"); + } + for hash in hashes.iter().filter(|h| !h.is_empty()) { + text += &format!("{hash}\n"); + } + } + let key = sha256_hex(text.as_bytes()); + let unchanged = *key_file == key; + + if materialize { + if !unchanged || target_kind != "directory" { + let work = format!("{state}/work"); + let key_path = format!("{state}/key"); + self.run( + sh::draw::MATERIALIZE, + false, + &[ + original, + &work, + &target_path, + &target_name, + &patch_dir, + name, + &key_path, + &key, + ], + )?; + } + if rewrite_lock { + let url = self.own_url(name)?; + let mut all = vec![target_path.as_str(), url.as_str()]; + all.extend(args.iter().map(String::as_str)); + self.run(sh::draw::LOCK, true, &all)?; + } + return Ok(()); } - for name in &sorted { - text += &process(name); + + let (path, subdir) = split_store_path(original); + let result = if unchanged { result.as_str() } else { "" }; + let mut all = vec![ + state.as_str(), + path, + subdir, + &target_path, + &target_name, + result, + &patch_dir, + name, + &key, + ]; + all.extend(args.iter().map(String::as_str)); + self.run(sh::draw::ADD, rewrite_lock, &all) + } + + /// Generates the flake of a source in .populate/flakes/<name>, which + /// re-exports the outputs of the flake in the source, with inputs drawn + /// from other sources. + fn generate(&self, name: &str) -> Result<()> { + let json = |s: &str| serde_json::to_string(s).expect("serializes"); + let url = self.own_url(name)?; + let drawn = self.drawn_from(name); + let urls = self.drawn_urls(&drawn)?; + let mut text = format!( + "{{\n inputs.{}.url = \"{url}\";\n", + json(&input_name(name)) + ); + for source in &drawn { + let (url, flake) = &urls[*source]; + let input = json(&input_name(source)); + text += &format!(" inputs.{input}.url = \"{url}\";\n"); + if !flake { + text += &format!(" inputs.{input}.flake = false;\n"); + } + } + // Nix ignores overrides below an input that follows another input, + // so an input of an input drawn from a source gets overridden at + // that source. + for f in &self.follows[name] { + let path: Vec<String> = drawn_path(name, &f.path, &self.follows, &self.generated) + .iter() + .map(|n| json(n)) + .collect(); + text += &format!( + " inputs.{}.follows = {};\n", + path.join(".inputs."), + json(&input_name(&f.source)) + ); } + text += &format!( + " outputs = inputs: inputs.{};\n}}\n", + json(&input_name(name)) + ); + self.run( + sh::draw::GENERATE, + true, + &[&self.state("flakes", name), &text], + ) + } + + /// Removes the state that sources don't need anymore. + fn clean_up(&self) -> Result<()> { + let list = |names: &mut dyn Iterator<Item = &str>| names.collect::<Vec<_>>().join(" "); + let all = list(&mut self.sources.keys().map(String::as_str)); + let not_flakes = list( + &mut self + .sources + .iter() + .filter(|(_, s)| !matches!(s.kind.as_str(), "flake" | "flake-input")) + .map(|(n, _)| n.as_str()), + ); + let generated = list(&mut self.generated.iter().copied()); + let derived = list(&mut self.derived.iter().copied()); + let patched = list( + &mut self + .sources + .iter() + .filter(|(_, s)| !patches(&s.settings).is_empty()) + .map(|(n, _)| n.as_str()), + ); + self.run( + sh::draw::CLEAN_UP, + false, + &[ + &self.sentinel(), + &all, + ¬_flakes, + &generated, + &derived, + &patched, + ], + ) } - text += "# Remove roots of flake inputs of sources that are neither flake nor\n# flake-input sources anymore.\n"; - for (name, source) in sources { - if !matches!(source.kind.as_str(), "flake" | "flake-input") { - text += &format!("rm -fR {}\n", q(&format!("{sentinel}/flake-inputs/{name}"))); +} + +/// Draws inputs of flakes in sources from other sources, derives what the +/// target paths of flake and flake-input sources that get patched, get +/// their lock files rewritten, or are materialized get, and removes state +/// that sources don't need anymore. Sources get processed in the order +/// they draw inputs from each other. +/// +/// Inputs get drawn by rewriting the lock file of the flake in the source +/// with nix flake lock --override-input: in place for file and git sources, +/// and in a copy for flake and flake-input sources. Sources with +/// rewriteLock set to false, and symlink sources, get a generated flake in +/// .populate/flakes/<name> instead, which re-exports the outputs of the +/// flake in the source, with those inputs drawn from the other sources. +/// Lock files get rewritten or generated on every run, so they match the +/// sources. +pub fn flakes(context: &Context, dir: &str, sources: &BTreeMap<String, Source>) -> Result<()> { + let follows: BTreeMap<&str, Vec<Follows>> = sources + .iter() + .map(|(n, s)| (n.as_str(), s.follows())) + .collect(); + let names_where = |p: &dyn Fn(&Source) -> bool| -> BTreeSet<&str> { + sources + .iter() + .filter(|(_, s)| p(s)) + .map(|(n, _)| n.as_str()) + .collect() + }; + let flakes = Flakes { + context, + dir, + sources, + drawing: names_where(&|s| !s.follows().is_empty()), + generated: names_where(&has_generated_flake), + derived: names_where(&derives_flake), + follows, + }; + let names: BTreeSet<&str> = flakes.drawing.union(&flakes.derived).copied().collect(); + if !names.is_empty() { + let mut args = vec![flakes.sentinel()]; + args.extend(sources.keys().cloned()); + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + flakes.run(sh::draw::USE_STATE, false, &args)?; + if !flakes.drawing.is_empty() { + flakes.check()?; + } + let drawn_from = |name: &str| -> Vec<String> { + flakes.follows[name] + .iter() + .map(|f| f.source.clone()) + .collect() + }; + for name in toposort(&names, &drawn_from) { + if flakes.derived.contains(name) { + flakes.derive(name)?; + } + if flakes.generated.contains(name) { + flakes.generate(name)?; + } else if flakes.drawing.contains(name) && !flakes.derived.contains(name) { + flakes.rewrite(name)?; + } } } - let fi = q(&format!("{sentinel}/flake-inputs")); - text += &format!( - "if test -d {fi} &&\n test -z \"$(ls -A {fi})\"; then\n rmdir {fi}\nfi\n# Remove what sources left behind that don't need it anymore.\n" - ); - text += &remove_stale(&flakes_dir, &generated); - text += &remove_stale(&derived_dir, &derived); - text += &remove_stale(&patches_dir, &patched); - format!("{}\n{text}", sh::prelude()) + flakes.clean_up() } /// The names sorted so each comes after the ones it draws inputs from, as @@ -798,6 +827,7 @@ fn sha256_hex(data: &[u8]) -> String { #[cfg(test)] mod tests { use super::*; + use serde_json::json; #[test] fn sha256() { @@ -810,4 +840,127 @@ mod tests { "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" ); } + + #[test] + fn url_paths() { + assert_eq!(url_path("/var/src/a-b_c.d~e"), "/var/src/a-b_c.d~e"); + assert_eq!(url_path("/x y?#%"), "/x%20y%3f%23%25"); + assert_eq!(url_path("/รค"), "/%c3%a4"); + } + + #[test] + fn input_names() { + assert_eq!(input_name("nixpkgs"), "nixpkgs"); + assert_eq!( + input_name("self"), + format!("source-{}", &sha256_hex(b"self")[..16]) + ); + assert!(input_name("1x").starts_with("source-")); + assert!(input_name("a.b").starts_with("source-")); + } + + #[test] + fn store_paths() { + assert_eq!( + split_store_path("/nix/store/abc-source/sub/dir"), + ("/nix/store/abc-source", "sub/dir") + ); + assert_eq!( + split_store_path("/nix/store/abc-source"), + ("/nix/store/abc-source", "") + ); + } + + fn path(names: &[&str]) -> Vec<String> { + names.iter().map(|n| n.to_string()).collect() + } + + /// A lock file with inputs a and b, where a has an input c, and its + /// input b follows the root's b. + fn locks() -> Value { + json!({ + "root": "root", + "nodes": { + "root": { "inputs": { "a": "a", "b": "b_2" } }, + "a": { "inputs": { "b": ["b"], "c": "c", "gone": [] } }, + "b_2": { "inputs": { "d": "d" } }, + "c": {}, + "d": {} + } + }) + } + + #[test] + fn lock_inputs() { + let locks = locks(); + assert!(lock_has_input(&locks, &path(&[]))); + assert!(lock_has_input(&locks, &path(&["a"]))); + assert!(lock_has_input(&locks, &path(&["a", "c"]))); + assert!(lock_has_input(&locks, &path(&["a", "b", "d"]))); + assert!(!lock_has_input(&locks, &path(&["x"]))); + assert!(!lock_has_input(&locks, &path(&["a", "x"]))); + assert!(!lock_has_input(&locks, &path(&["a", "b", "x"]))); + // An empty follows path refers to the root. + assert!(lock_has_input(&locks, &path(&["a", "gone", "a"]))); + } + + #[test] + fn cyclic_follows() { + let mut locks = locks(); + locks["nodes"]["a"]["inputs"]["b"] = json!(["a", "b"]); + assert!(!lock_has_input(&locks, &path(&["a", "b"]))); + } + + /// A source's name, and the paths of the inputs it draws, with the + /// sources it draws them from. + type Entry<'a> = (&'a str, &'a [(&'a [&'a str], &'a str)]); + + fn follows(entries: &[Entry]) -> BTreeMap<&'static str, Vec<Follows>> { + entries + .iter() + .map(|(name, fs)| { + let name: &'static str = Box::leak(name.to_string().into_boxed_str()); + let fs = fs + .iter() + .map(|(p, s)| Follows { + path: path(p), + source: s.to_string(), + }) + .collect(); + (name, fs) + }) + .collect() + } + + #[test] + fn drawn_paths() { + // top draws dep from source dep, and dep's inner from source + // inner; dep draws nothing itself. + let follows = follows(&[ + ("top", &[(&["dep"], "dep"), (&["dep", "inner"], "inner")]), + ("dep", &[]), + ("inner", &[]), + ]); + let none = BTreeSet::new(); + assert_eq!( + drawn_path("top", &path(&["dep", "inner"]), &follows, &none), + path(&["dep", "inner"]) + ); + let generated: BTreeSet<&str> = ["top"].into(); + assert_eq!( + drawn_path("top", &path(&["dep"]), &follows, &generated), + path(&["top", "dep"]) + ); + assert_eq!( + drawn_path("top", &path(&["dep", "inner"]), &follows, &generated), + path(&["dep", "inner"]) + ); + assert_eq!( + input_checks("top", &path(&["dep", "inner"]), &follows), + vec![ + ("top".to_string(), path(&["dep"])), + ("dep".to_string(), path(&["inner"])) + ] + ); + } } |