diff options
| -rw-r--r-- | flake.nix | 3 | ||||
| -rw-r--r-- | lib/default.nix | 60 | ||||
| -rw-r--r-- | lib/types/default.nix | 11 | ||||
| -rw-r--r-- | pkgs/krops/checks.nix (renamed from pkgs/populate/checks.nix) | 0 | ||||
| -rw-r--r-- | pkgs/krops/populate.nix | 84 | ||||
| -rw-r--r-- | pkgs/overlay.nix | 5 | ||||
| -rw-r--r-- | pkgs/populate/default.nix | 106 | ||||
| -rw-r--r-- | pkgs/populate/docs/design.md | 59 | ||||
| -rw-r--r-- | pkgs/populate/lib/default.nix | 78 | ||||
| -rw-r--r-- | pkgs/populate/lib/types/populate.nix (renamed from lib/types/populate.nix) | 0 | ||||
| -rw-r--r-- | pkgs/populate/lib/types/posix.nix (renamed from lib/types/posix.nix) | 0 | ||||
| -rw-r--r-- | pkgs/populate/program.nix | 25 |
12 files changed, 231 insertions, 200 deletions
@@ -20,7 +20,8 @@ let pkgs = nixpkgs.legacyPackages.${system}; krops = pkgs.callPackage ./pkgs/krops { inherit populate; }; - populate = pkgs.callPackage ./pkgs/populate {}; + populate = pkgs.callPackage ./pkgs/krops/populate.nix { inherit populate-unwrapped; }; + populate-unwrapped = pkgs.callPackage ./pkgs/populate {}; in { inherit populate; inherit (krops) rebuild runShell withNixOutputMonitor writeCommand writeDeploy writeTest; diff --git a/lib/default.nix b/lib/default.nix index 0ffdff2..4eb6438 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -4,60 +4,8 @@ let { lib = nixpkgs.lib // builtins // { - evalSource = let - eval = source: lib.evalModules { - modules = lib.singleton { - _file = toString ./.; - imports = map (source: { inherit source; }) (lib.toList source); - options.source = lib.mkOption { - default = {}; - type = lib.types.attrsOf lib.types.source; - }; - }; - }; - sanitize = x: lib.getAttr (lib.typeOf x) { - bool = x; - list = map sanitize x; - set = lib.mapAttrs - (lib.const sanitize) - (lib.filterAttrs - (name: value: name != "_module" && value != null) x); - string = x; - }; - in - # This function's return value can be used as pkgs.populate input. - source: sanitize (eval source).config.source; - - # Inputs a source draws from other sources, as list of attribute sets like - # { path = [ "stockholm" "nixpkgs" ]; source = "nixpkgs"; }. - flakeFollows = source: let - flatten = prefix: inputs: lib.concatLists (lib.mapAttrsToList - (name: input: - lib.optional (input.follows or null != null) { - path = prefix ++ [ name ]; - source = input.follows; - } ++ - flatten (prefix ++ [ name ]) (input.inputs or {})) - inputs); - in - flatten [] (source.${source.type}.inputs or {}); - - # Tells whether a source's inputs get drawn from other sources by a - # generated flake in .populate/flakes, instead of by rewriting the lock - # file of the flake in the source. Symlink sources point at other - # sources, whose lock files stay as they are. - hasGeneratedFlake = source: - lib.flakeFollows source != [] && - (source.type == "symlink" || !(source.${source.type}.rewriteLock or true)); - - # Tells whether a flake or flake-input source gets delivered as a copy, - # which gets patched, gets its lock file rewritten, or is materialized. - derivesFlake = source: let - source' = source.${source.type}; - in - lib.elem source.type [ "flake" "flake-input" ] && - (source'.materialize or false || source'.patches or [] != [] || - (source'.rewriteLock or true && lib.flakeFollows source != [])); + # What populate's Nix library defines. + inherit (populate.lib) evalSource flakeFollows hasGeneratedFlake derivesFlake; maybeHostName = default: let # We're parsing /etc/hostname here because reading @@ -153,9 +101,11 @@ let { test = re: x: lib.isString x && lib.testString re x; testString = re: x: lib.match re x != null; - types = nixpkgs.lib.types // import ./types { lib = body; }; + types = nixpkgs.lib.types // populate.lib.types; }; nixpkgs.lib = import <nixpkgs/lib>; + populate.lib = import ../pkgs/populate/lib { lib = nixpkgs.lib; }; + } diff --git a/lib/types/default.nix b/lib/types/default.nix index c4bf517..2023a84 100644 --- a/lib/types/default.nix +++ b/lib/types/default.nix @@ -1,8 +1,3 @@ -{ lib }@args: let { - - body = lib.foldl' (res: path: res // import path args) {} [ - ./populate.nix - ./posix.nix - ]; - -} +# The types of krops' lib that aren't nixpkgs', which populate's Nix library +# defines. +{ lib }: (import ../../pkgs/populate/lib { inherit lib; }).types diff --git a/pkgs/populate/checks.nix b/pkgs/krops/checks.nix index 22b45c9..22b45c9 100644 --- a/pkgs/populate/checks.nix +++ b/pkgs/krops/checks.nix diff --git a/pkgs/krops/populate.nix b/pkgs/krops/populate.nix new file mode 100644 index 0000000..74ba04a --- /dev/null +++ b/pkgs/krops/populate.nix @@ -0,0 +1,84 @@ +# pkgs.populate: given populate's arguments, it checks them, and returns a +# script running populate-unwrapped, the program in ../populate, with them, +# the sources as JSON. +{ coreutils, findutils, git, nix, openssh, pass, passage, rsync, writers +, populate-unwrapped +, ... +}@args: + +with import ../../lib; + +let + checks = import ./checks.nix; + + # Arguments populate doesn't use, which overrides written for krops 1 may + # set. They come through ..., because callPackage would set named ones. + unused = filter (name: args ? ${name}) [ "dash" "implementation" "jq" ]; + unexpected = attrNames (removeAttrs args ([ + "coreutils" "findutils" "git" "nix" "openssh" "pass" "passage" "rsync" + "writers" "populate-unwrapped" + ] ++ unused)); + + # The parts of a target string, like mkTarget parses it, without its + # defaults, which populate fills in when it runs. + parseTarget = s: let + parts = match "(([^@]*)@)?(([^:/]+))?(:([^/]+))?(/.*)?" s; + in + if parts == null then throw "krops: invalid target ${toJSON s}" + else { host = elemAt parts 3; path = elemAt parts 6; }; + + populate = + { backup ? false, force ? false, prune ? false, source, target }: + let + # A target string goes to populate as it is, so populate decides what + # it leaves out when it runs. An attribute set, as mkTarget returns + # it, goes part by part, with whether it's local decided now, like the + # other functions of krops do. + parsed = parseTarget target; + path = if isString target then (if parsed.path != null then parsed.path else "/var/src") else target.path; + host = if isString target then (if parsed.host != null then parsed.host else "localhost") else target.host; + targetArgs = + if isString target then [ target ] + else [ + "--user" (if target.user or null == null then "" else target.user) + "--host" target.host + "--path" target.path + ] ++ optionals (target.port or null != null) [ + "--port" (toString target.port) + ] ++ optional (target.sudo or false) "--sudo" + ++ concatMap (arg: [ "--ssh-arg" arg ]) (flatten (target.extraOptions or [])) + ++ [ (if isLocalTarget target then "--local" else "--remote") ]; + in + assert checks.checkTargetPath path; + assert all checks.checkSourceName (attrNames source); + assert checks.checkInputs source; + assert isString target || checks.checkCopyingWithSudo target source; + writers.writeDash "populate.${host}" /* sh */ '' + set -efu + echo + "$0" >&2 + PATH=${makeBinPath [ + coreutils findutils git nix openssh pass passage rsync + ]}''${PATH:+:$PATH} + export PATH + # The sources on a file descriptor of their own, so the commands + # populate runs keep standard input. Store paths in them stay + # dependencies of this script, so building it builds them. The JSON is + # a single line, which can't be the delimiter. + exec ${populate-unwrapped}/bin/populate ${escapeShellArgs ( + optional backup "--backup" ++ + optional force "--force" ++ + optional prune "--prune" + )} /dev/fd/3 ${escapeShellArgs targetArgs} 3<<'EOF' + ${toJSON source} + EOF + ''; + +in + +if unexpected != [] then + throw "krops: populate called with unexpected argument ${head unexpected}" +else + foldr + (name: warn "krops: populate's argument ${name} is unused") + populate + unused diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index caffdb5..506b4ad 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -1,4 +1,7 @@ self: super: { krops = self.callPackage ./krops {}; - populate = self.callPackage ./populate {}; + # populate's Nix API, a function from its arguments to a script running + # populate-unwrapped, the program. + populate = self.callPackage ./krops/populate.nix {}; + populate-unwrapped = self.callPackage ./populate {}; } diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index bd8e772..0c98b28 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -1,86 +1,28 @@ -# Populate, which delivers sources to a target directory. Given populate's -# arguments, it checks them, and returns a script running the program in -# this directory with them, the sources as JSON. rustProgram is that program, e.g. one -# built with cargo during development. -{ coreutils, findutils, git, nix, openssh, pass, passage, rsync, writers -, rustPlatform ? null -, rustProgram ? import ./program.nix { inherit rustPlatform; } -, ... -}@args: - -with import ../../lib; +# populate, built from the crate in this directory: nix-build here, or +# callPackage. +{ rustPlatform ? (import <nixpkgs> {}).rustPlatform }: let - checks = import ./checks.nix; - - # Arguments populate doesn't use, which overrides written for krops 1 may - # set. They come through ..., because callPackage would set named ones. - unused = filter (name: args ? ${name}) [ "dash" "implementation" "jq" ]; - unexpected = attrNames (removeAttrs args ([ - "coreutils" "findutils" "git" "nix" "openssh" "pass" "passage" "rsync" - "writers" "rustPlatform" "rustProgram" - ] ++ unused)); - - # The parts of a target string, like mkTarget parses it, without its - # defaults, which populate fills in when it runs. - parseTarget = s: let - parts = match "(([^@]*)@)?(([^:/]+))?(:([^/]+))?(/.*)?" s; - in - if parts == null then throw "krops: invalid target ${toJSON s}" - else { host = elemAt parts 3; path = elemAt parts 6; }; - - populate = - { backup ? false, force ? false, prune ? false, source, target }: - let - # A target string goes to populate as it is, so populate decides what - # it leaves out when it runs. An attribute set, as mkTarget returns - # it, goes part by part, with whether it's local decided now, like the - # other functions of krops do. - parsed = parseTarget target; - path = if isString target then (if parsed.path != null then parsed.path else "/var/src") else target.path; - host = if isString target then (if parsed.host != null then parsed.host else "localhost") else target.host; - targetArgs = - if isString target then [ target ] - else [ - "--user" (if target.user or null == null then "" else target.user) - "--host" target.host - "--path" target.path - ] ++ optionals (target.port or null != null) [ - "--port" (toString target.port) - ] ++ optional (target.sudo or false) "--sudo" - ++ concatMap (arg: [ "--ssh-arg" arg ]) (flatten (target.extraOptions or [])) - ++ [ (if isLocalTarget target then "--local" else "--remote") ]; - in - assert checks.checkTargetPath path; - assert all checks.checkSourceName (attrNames source); - assert checks.checkInputs source; - assert isString target || checks.checkCopyingWithSudo target source; - writers.writeDash "populate.${host}" /* sh */ '' - set -efu - echo + "$0" >&2 - PATH=${makeBinPath [ - coreutils findutils git nix openssh pass passage rsync - ]}''${PATH:+:$PATH} - export PATH - # The sources on a file descriptor of their own, so the commands - # populate runs keep standard input. Store paths in them stay - # dependencies of this script, so building it builds them. The JSON is - # a single line, which can't be the delimiter. - exec ${rustProgram}/bin/populate ${escapeShellArgs ( - optional backup "--backup" ++ - optional force "--force" ++ - optional prune "--prune" - )} /dev/fd/3 ${escapeShellArgs targetArgs} 3<<'EOF' - ${toJSON source} - EOF - ''; - + cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); in -if unexpected != [] then - throw "krops: populate called with unexpected argument ${head unexpected}" -else - foldr - (name: warn "krops: populate's argument ${name} is unused") - populate - unused +rustPlatform.buildRustPackage { + pname = cargoToml.package.name; + inherit (cargoToml.package) version; + # Without what cargo doesn't build from, so changes to the Nix files, the + # docs, or the tests don't rebuild the program. By names, not by paths + # relative to this directory: the paths the filter gets needn't start + # with toString ./., e.g. if a symlink leads to this directory. + src = builtins.path { + name = "populate-src"; + path = ./.; + filter = path: type: let + name = baseNameOf path; + in + !(builtins.elem name [ + ".gitignore" "README.md" "docs" "lib" "target" "tests" + ] || + builtins.match ".*\\.nix" name != null); + }; + cargoLock.lockFile = ./Cargo.lock; +} diff --git a/pkgs/populate/docs/design.md b/pkgs/populate/docs/design.md index cf9d356..940ab5f 100644 --- a/pkgs/populate/docs/design.md +++ b/pkgs/populate/docs/design.md @@ -1,15 +1,16 @@ # Populate's design -Populate delivers sources to a target directory. It consists of two parts: +Populate delivers sources to a target directory. It consists of: -- Nix code, which turns populate's arguments into a command line: the - sources as `evalSource` returns them, as JSON, the target, and the - options. It checks the arguments during evaluation, so mistakes show - early. -- A program, which reads the sources and delivers them to the target its - command line names. It knows nothing about Nix expressions: whatever - produces sources as JSON can drive it. +- A program, which reads the sources as JSON and delivers them to the + target its command line names. It knows nothing about Nix expressions: + whatever produces sources as JSON can drive it. +- A Nix library, `lib`, with the types of sources, which describe how to + write them in Nix, and `evalSource`, which turns sources written so into + the JSON the program reads. +krops uses both: its `pkgs.populate` turns populate's arguments into a +command line, and checks them during evaluation, so mistakes show early. So there is one program for each version of populate, instead of a script for each set of arguments, and the program can be written in any language. This directory holds the program in Rust. Another implementation, e.g. in @@ -18,20 +19,22 @@ the same tests. ## Nix side -`default.nix` is `pkgs.populate`. Given populate's arguments, `{ backup ? -false, force ? false, prune ? false, source, target }`, it checks them with -`checks.nix`, and returns a script running the program with the sources as -JSON on file descriptor 3, so the commands populate runs keep standard -input. A target string goes to the program as it is. A target attribute -set, as `mkTarget` returns it, goes part by part, as options, with -`--local` or `--remote` as krops' lib's `isLocalTarget` decides during -evaluation, like the other functions of krops. The script puts the tools -populate runs in front of `PATH`, and depends on the store paths in the -sources, so building it builds them. `rustProgram` is the program, by -default built by `program.nix`. +`default.nix` builds the program: `nix-build` in this directory, or +`callPackage`, as krops' `pkgs.populate-unwrapped`. `lib/default.nix` is +the Nix library, taking nixpkgs' lib. -`writeDeploy` and the other functions of `pkgs.krops` use `pkgs.populate`, -so their arguments stay as they are. +krops' `pkgs.populate`, in its `pkgs/krops/populate.nix`, is populate's Nix +API there. Given populate's arguments, `{ backup ? false, force ? false, +prune ? false, source, target }`, it checks them with krops' +`pkgs/krops/checks.nix`, and returns a script running the program with the +sources as JSON on file descriptor 3, so the commands populate runs keep +standard input. A target string goes to the program as it is. A target +attribute set, as `mkTarget` returns it, goes part by part, as options, +with `--local` or `--remote` as krops' lib's `isLocalTarget` decides +during evaluation, like the other functions of krops. The script puts the +tools populate runs in front of `PATH`, and depends on the store paths in +the sources, so building it builds them. krops' `writeDeploy` and its +other functions use `pkgs.populate`, so their arguments stay as they are. ## Invocation @@ -99,13 +102,13 @@ doesn't go unnoticed. The sources carry no version: this is version 0, and a later version of populate that reads them differently will mark them. Store paths in the sources, e.g. of a `file` source that is a derivation, -must exist when the program runs. The script `default.nix` writes depends -on them. Something producing sources at run time has to build them first. +must exist when the program runs. The script krops' `pkgs.populate` +writes depends on them. Something producing sources at run time has to build them first. ## Checks -The program checks the sources and the target as `checks.nix` does, with -the same messages: +The program checks the sources and the target as krops' `checks.nix` does +during evaluation, with the same messages: - The target's host isn't empty, and neither it nor its user starts with `-`, so ssh can't take them for options. `checks.nix` doesn't check @@ -122,9 +125,9 @@ the same messages: since `evalSource` checks the types. Checks that depend on whether the target is local come on top, e.g. -refusing to copy flakes to remote targets using sudo. `checks.nix` does -that during evaluation too, where `default.nix` decides whether the target -is local. +refusing to copy flakes to remote targets using sudo. krops' `checks.nix` +does that during evaluation too, where `pkgs.populate` decides whether the +target is local. ## Output diff --git a/pkgs/populate/lib/default.nix b/pkgs/populate/lib/default.nix new file mode 100644 index 0000000..e9155c5 --- /dev/null +++ b/pkgs/populate/lib/default.nix @@ -0,0 +1,78 @@ +# populate's Nix library: the types of sources, which describe how to write +# them in Nix, and evalSource, which turns sources written so into the +# sources populate reads as JSON. lib is nixpkgs' lib. +{ lib ? import <nixpkgs/lib> }: + +let + # nixpkgs' lib, with what the types use besides. + lib' = lib // { + test = re: x: lib.isString x && lib.match re x != null; + types = lib.types // types; + }; + + types = + import ./types/posix.nix { lib = lib'; } // + import ./types/populate.nix { lib = lib'; }; + +in rec { + + inherit types; + + # The sources, a list of attribute sets of sources by name, or one, with + # shorthands expanded and defaults filled in, and without settings that + # are null, as populate reads them, e.g. with builtins.toJSON. + evalSource = let + eval = source: lib.evalModules { + modules = lib.singleton { + _file = toString ./.; + imports = map (source: { inherit source; }) (lib.toList source); + options.source = lib.mkOption { + default = {}; + type = lib.types.attrsOf types.source; + }; + }; + }; + sanitize = x: lib.getAttr (lib.typeOf x) { + bool = x; + list = map sanitize x; + set = lib.mapAttrs + (lib.const sanitize) + (lib.filterAttrs + (name: value: name != "_module" && value != null) x); + string = x; + }; + in + source: sanitize (eval source).config.source; + + # Inputs a source draws from other sources, as list of attribute sets like + # { path = [ "stockholm" "nixpkgs" ]; source = "nixpkgs"; }. + flakeFollows = source: let + flatten = prefix: inputs: lib.concatLists (lib.mapAttrsToList + (name: input: + lib.optional (input.follows or null != null) { + path = prefix ++ [ name ]; + source = input.follows; + } ++ + flatten (prefix ++ [ name ]) (input.inputs or {})) + inputs); + in + flatten [] (source.${source.type}.inputs or {}); + + # Tells whether a source's inputs get drawn from other sources by a + # generated flake in .populate/flakes, instead of by rewriting the lock + # file of the flake in the source. Symlink sources point at other + # sources, whose lock files stay as they are. + hasGeneratedFlake = source: + flakeFollows source != [] && + (source.type == "symlink" || !(source.${source.type}.rewriteLock or true)); + + # Tells whether a flake or flake-input source gets delivered as a copy, + # which gets patched, gets its lock file rewritten, or is materialized. + derivesFlake = source: let + source' = source.${source.type}; + in + lib.elem source.type [ "flake" "flake-input" ] && + (source'.materialize or false || source'.patches or [] != [] || + (source'.rewriteLock or true && flakeFollows source != [])); + +} diff --git a/lib/types/populate.nix b/pkgs/populate/lib/types/populate.nix index 56b0292..56b0292 100644 --- a/lib/types/populate.nix +++ b/pkgs/populate/lib/types/populate.nix diff --git a/lib/types/posix.nix b/pkgs/populate/lib/types/posix.nix index e8f464e..e8f464e 100644 --- a/lib/types/posix.nix +++ b/pkgs/populate/lib/types/posix.nix diff --git a/pkgs/populate/program.nix b/pkgs/populate/program.nix deleted file mode 100644 index fbb364c..0000000 --- a/pkgs/populate/program.nix +++ /dev/null @@ -1,25 +0,0 @@ -# The program delivering sources, built from the crate in this directory. -{ rustPlatform }: - -let - cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); -in - -rustPlatform.buildRustPackage { - pname = cargoToml.package.name; - inherit (cargoToml.package) version; - # Without what cargo doesn't build from, so changes to the Nix files or - # the docs don't rebuild the program. By names, not by paths relative to - # this directory: the paths the filter gets needn't start with - # toString ./., e.g. if a symlink leads to this directory. - src = builtins.path { - name = "populate-src"; - path = ./.; - filter = path: type: let - name = baseNameOf path; - in - !(builtins.elem name [ ".gitignore" "docs" "target" ] || - builtins.match ".*\\.nix" name != null); - }; - cargoLock.lockFile = ./Cargo.lock; -} |
