diff options
| -rw-r--r-- | README.md | 228 | ||||
| -rw-r--r-- | ci.nix | 4 | ||||
| -rw-r--r-- | flake.lock | 27 | ||||
| -rw-r--r-- | flake.nix | 29 | ||||
| -rw-r--r-- | lib/default.nix | 43 | ||||
| -rw-r--r-- | lib/types/populate.nix | 91 | ||||
| -rw-r--r-- | pkgs/default.nix | 8 | ||||
| -rw-r--r-- | pkgs/krops/default.nix | 160 | ||||
| -rw-r--r-- | pkgs/populate/default.nix | 165 |
9 files changed, 646 insertions, 109 deletions
@@ -1,12 +1,13 @@ -# krops (krebs ops) +# krops (krebs operations) -krops is a lightweigt toolkit to deploy NixOS systems, remotely or locally. +krops is a lightweight toolkit to deploy NixOS systems, remotely or locally. ## Some Features - store your secrets in [password store](https://www.passwordstore.org/) -- build your system remotely + or [passage](https://github.com/FiloSottile/passage) +- build your systems remotely - minimal overhead (it's basically just `nixos-rebuild switch`!) - run from custom nixpkgs branch/checkout/fork @@ -15,12 +16,12 @@ krops is a lightweigt toolkit to deploy NixOS systems, remotely or locally. Create a file named `krops.nix` (name doesn't matter) with following content: -``` +```nix let krops = (import <nixpkgs> {}).fetchgit { url = https://cgit.krebsco.de/krops/; - rev = "3022582ade8049e6ccf18f358cedb996d6716945"; - sha256 = "0k3zhv2830z4bljcdvf6ciwjihk2zzcn9y23p49c6sba5hbsd6jb"; + rev = "v1.25.0"; + sha256 = "07mg3iaqjf1w49vmwfchi7b1w55bh7rvsbgicp2m47gnj9alwdb6"; }; lib = import "${krops}/lib"; @@ -52,11 +53,175 @@ in ``` and run `$(nix-build --no-out-link krops.nix)` to deploy the target machine. +krops exports some funtions under `krops.` namely: + +## writeDeploy -Under the hood, this will make the sources available on the target machine +This will make the sources available on the target machine below `/var/src`, and execute `nixos-rebuild switch -I /var/src`. +### `target` + +The `target` attribute to `writeDeploy` can either be a string or an attribute +set, specifying where to make the sources available, as well as where to run +the deployment. + +If specified as string, the format could be described as: +``` +[[USER]@]HOST[:PORT][/SOME/PATH] +``` + +Portions in square brackets are optional. + +If the `USER` is the empty string, as in e.g. `@somehost`, then the username +will be obtained by ssh from its configuration files. + +If the `target` attribute is an attribute set, then it can specify the +attributes `extraOptions`, `host`, `path`, `port`, `sudo`, and `user`. +The `extraOptions` is a list of strings that get passed to ssh as additional +arguments. The `sudo` attribute is a boolean and if set to true, then it's +possible to to deploy to targets that disallow sshing in as root, but allow +(preferably passwordless) sudo. +Example: + +```nix +pkgs.krops.writeDeploy "deploy" { + source = /* ... */; + target = lib.mkTarget "user@host/path" // { + extraOptions = [ + "-o" "LogLevel=DEBUG" + ]; + sudo = true; + }; +} +``` +For more details about the `target` attribute, please check the `mkTarget` +function in [lib/default.nix](lib/default.nix). + +### `backup` (optional, defaults to false) + +Backup all paths specified in source before syncing new sources. + +### `buildTarget` (optional) + +If set the evaluation and build of the system will be executed on this host. +`buildTarget` takes the same arguments as target. +Sources will be synced to both `buildTarget` and `target`. +Built packages will be uploaded from the `buildTarget` to `target` directly +This requires the building machine to have ssh access to the target. +To build the system on the same machine, that runs the krops command, +set up a local ssh service and set the build host to localhost. + +### `crossDeploy` (optional, defaults to false) + +Use this option if target host architecture is not the same as the build host +architecture as set by `buildHost` i.e. deploying to aarch64 from a x86_64 +machine. Setting this option will disable building & running nix in the wrong +architecture when running `nixos-rebuild` on the deploying machine. It is +required to set `nixpkgs.localSystem.system` in the NixOS configuration to the +architecture of the target host. This option is only useful if the build host +also has remote builders that are capable of producing artifacts for the deploy +architecture. + +### `fast` (optional, defaults to false) + +Run `nixos-rebuild` immediately without building the system in a dedicated `nix +build` step. + +### `force` (optional, defaults to false) + +Create the sentinel file (`/var/src/.populate`) before syncing the new source. + +### `operation` (optional, defaults to "switch") + +Specifies which `nixos-rebuild` operation to perform. + +### `useNixOutputMonitor` (optional, defaults to `"opportunistic"`) + +Specifies when to pipe `nixos-rebuild`'s output to +[nom](https://github.com/maralorn/nix-output-monitor). + +Supported values: + +* `"opportunistic"` (default) - + Use `nom` only if it is present on the target machine. + +* `"optimistic"` - + Use `nom`, assuming it is present on the target machine. + +* `"pessimistic"` - + Use `nom` via `nix-shell` on the target machine. + +* `true` - + Use `nom`. + If it is not present on the target machine, then use it via `nix-shell`. + +* `false` - + Don't use `nom` + +## writeTest + +Very similiar to writeDeploy, but just builds the system on the target without +activating it. + +This basically makes the sources available on the target machine +below `/var/src`, and executes `NIX_PATH=/var/src nix-build -A system '<nixpkgs/nixos>'`. + +### `target` + +[see `writeDeploy`](#writeDeploy) + +### `backup` (optional, defaults to false) + +[see `writeDeploy`](#writeDeploy) + +### `force` (optional, defaults to false) + +[see `writeDeploy`](#writeDeploy) + +### `trace` (optional, defaults to false) + +run nix-build with `--show-trace` + +## writeCommand + +This can be used to run other commands than `nixos-rebuild` or pre/post build hooks. + +### `command` + +A function which takes the targetPath as an attribute. +Example to activate/deactivate a swapfile before/after build: + +```nix +pkgs.krops.writeCommand "deploy-with-swap" { + source = source; + target = "root@YOUR_IP_ADDRESS_OR_HOST_NAME_HERE"; + command = targetPath: '' + swapon /var/swapfile + nixos-rebuild -I ${targetPath} switch + swapoff /var/swapfile + ''; +} +``` + +### `target` + +[see `writeDeploy`](#writeDeploy) + +### `backup` (optional, defaults to false) + +[see `writeDeploy`](#writeDeploy) + +### `force` (optional, defaults to false) + +[see `writeDeploy`](#writeDeploy) + +### `allocateTTY` (optional, defaults to false) + +whether the ssh session should do a pseudo-terminal allocation. +sets `-t` on the ssh command. + ## Source Types ### `derivation` @@ -77,13 +242,28 @@ using [`rsync`](https://rsync.samba.org/). Supported attributes: * `path` - - absolute path to files that should by transfered + absolute path to files that should by transferred. * `useChecksum` (optional) - boolean that controls whether file contents should be checked to decide whether a file has changed. This is useful when `path` points at files with mangled timestamps, e.g. the Nix store. + The default value is `true` if `path` is a derivation, and `false` otherwise. + +* `filters` (optional) + List of filters that should be passed to [`rsync`](https://rsync.samba.org/). + Filters are specified as attribute sets with the attributes `type` and + `pattern`. Supported filter types are `include` and `exclude`. + Checkout the filter rules section in the + [rsync manual](https://download.samba.org/pub/rsync/rsync.html) + for further information. + +* `deleteExcluded` (optional) + boolean that controls whether the excluded directories should be deleted + if they exist on the target. This is passed to the `--delete-excluded` option + of rsync. Defaults to `true`. + ### `git` @@ -100,6 +280,10 @@ Supported attributes: * `clean.exclude` - List of patterns that should be excluded from Git cleaning. +* `shallow` (optional) + boolean that controls whether only the requested commit ref. should be fetched + instead of the whole history, to save disk space and bandwith. Defaults to `false`. + ### `pass` @@ -115,6 +299,29 @@ Supported attributes: sub-directory in the password store. +### `passage` + +The passage source type decrypts files from a local +[passage store](https://github.com/FiloSottile/passage) +and transfers them to the target using +[`rsync`](https://rsync.samba.org/). + +Supported attributes: + +* `dir` - + Path to the passage store. + For a partial transfer, this may point to a subdirectory. + Example: `~/.passage/store/hosts/MYHOSTNAME` + +* `identities_file` (optional) - + Path to the identities file. + Defaults to `~/.passage/identities`. + +* `age` (optional) - + Path of the age binary. + Defaults to `age` (absolute path gets resolved using `passage`'s search path.) + + ### `pipe` Executes a local command, capture its stdout, and send that as a file to the @@ -143,8 +350,9 @@ Supported attributes: ## Communication -Comments, questions, pull-requests, etc. are very welcome, and can be directed +Comments, questions, pull-requests and patches, etc. are very welcome, and can be directed at: -- IRC: #krebs at freenode +- IRC: #krebs at hackint - Mail: [spam@krebsco.de](mailto:spam@krebsco.de) +- Github: https://github.com/krebs/krops/ @@ -5,7 +5,7 @@ let pkgs = import "${krops}/pkgs" {}; source = lib.evalSource [{ - nixos-config.file = toString (pkgs.writeText "nixos-config" '' + nixos-config.file = pkgs.writeText "nixos-config" '' { pkgs, ... }: { fileSystems."/" = { device = "/dev/sda1"; }; @@ -13,7 +13,7 @@ let services.openssh.enable = true; environment.systemPackages = [ pkgs.git ]; } - ''); + ''; nixpkgs.symlink = toString <nixpkgs>; }]; in { diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..51e2759 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1689940971, + "narHash": "sha256-397xShPnFqPC59Bmpo3lS+/Aw0yoDRMACGo1+h2VJMo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9ca785644d067445a4aa749902b29ccef61f7476", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..ec13152 --- /dev/null +++ b/flake.nix @@ -0,0 +1,29 @@ +{ + description = "krops - krebs operations"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + }; + + outputs = { self, nixpkgs, ... }: + let + supportedSystems = [ + "x86_64-linux" + "i686-linux" + "aarch64-linux" + "riscv64-linux" + ]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + in + { + lib = forAllSystems (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + krops = pkgs.callPackage ./pkgs/krops {}; + populate = pkgs.callPackage ./pkgs/populate {}; + in { + inherit populate; + inherit (krops) rebuild runShell withNixOutputMonitor writeCommand writeDeploy writeTest; + }); + }; +} diff --git a/lib/default.nix b/lib/default.nix index 48e005e..3bbd754 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -28,13 +28,23 @@ let { # This function's return value can be used as pkgs.populate input. source: sanitize (eval source).config.source; - getHostName = let + maybeHostName = default: let # We're parsing /etc/hostname here because reading # /proc/sys/kernel/hostname yields "" - y = lib.filter lib.types.label.check (lib.splitString "\n" (lib.readFile /etc/hostname)); + path = "/etc/hostname"; + lines = lib.splitString "\n" (lib.readFile path); + hostNames = lib.filter lib.types.label.check lines; in - if lib.length y != 1 then throw "malformed /etc/hostname" else - lib.elemAt y 0; + if lib.pathExists path then + if lib.length hostNames == 1 then + lib.head hostNames + else + lib.trace "malformed ${path}" default + else + default; + + firstWord = s: + lib.head (lib.match "^([^[:space:]]*).*" s); isLocalTarget = let origin = lib.mkTarget ""; @@ -43,15 +53,26 @@ let { lib.elem target.host [origin.host "localhost"]; mkTarget = s: let - default = defVal: val: if val != null then val else defVal; - parse = lib.match "(([^@]+)@)?(([^:/]+))?(:([^/]+))?(/.*)?" s; + parse = lib.match "(([^@]*)@)?(([^:/]+))?(:([^/]+))?(/.*)?" s; elemAt' = xs: i: if lib.length xs > i then lib.elemAt xs i else null; + filterNull = lib.filterAttrs (n: v: v != null); in { - user = default (lib.getEnv "LOGNAME") (elemAt' parse 1); - host = default (lib.maybeEnv "HOSTNAME" lib.getHostName) (elemAt' parse 3); - port = default "22" /* "ssh"? */ (elemAt' parse 5); - path = default "/var/src" /* no default? */ (elemAt' parse 6); - }; + user = lib.maybeEnv "LOGNAME" null; + host = lib.maybeEnv "HOSTNAME" (lib.maybeHostName "localhost"); + port = null; + path = "/var/src"; + sudo = false; + extraOptions = []; + } // (if lib.isString s then filterNull { + user = elemAt' parse 1; + host = elemAt' parse 3; + port = elemAt' parse 5; + path = elemAt' parse 6; + } else s); + + mkUserPortSSHOpts = target: + (lib.optionals (target.user != null) ["-l" target.user]) ++ + (lib.optionals (target.port != null) ["-p" target.port]); shell = let isSafeChar = lib.testString "[-+./0-9:=A-Z_a-z]"; diff --git a/lib/types/populate.nix b/lib/types/populate.nix index 49996f6..6264f99 100644 --- a/lib/types/populate.nix +++ b/lib/types/populate.nix @@ -21,11 +21,15 @@ }; file = lib.mkOption { apply = x: - if lib.types.absolute-pathname.check x + if lib.types.absolute-pathname.check x || lib.types.package.check x then { path = x; } else x; default = null; - type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.file); + type = lib.types.nullOr (lib.types.oneOf [ + lib.types.absolute-pathname + lib.types.package + source-types.file + ]); }; git = lib.mkOption { default = null; @@ -35,6 +39,17 @@ default = null; type = lib.types.nullOr source-types.pass; }; + passage = lib.mkOption { + apply = x: + if lib.types.pathname.check x + then { dir = x; } + else x; + default = null; + type = lib.types.nullOr (lib.types.oneOf [ + lib.types.pathname + source-types.passage + ]); + }; pipe = lib.mkOption { apply = x: if lib.types.absolute-pathname.check x @@ -54,6 +69,18 @@ }; }); + filter = lib.types.submodule { + options = { + type = lib.mkOption { + type = lib.types.enum ["include" "exclude"]; + default = "exclude"; + }; + pattern = lib.mkOption { + type = lib.types.str; + }; + }; + }; + source-types = { derivation = lib.types.submodule { options = { @@ -71,6 +98,43 @@ default = false; type = lib.types.bool; }; + exclude = lib.mkOption { + apply = x: + if x != [] then + lib.warn + "file.*.exclude is deprecated in favor of file.*.filters" + x + else + x; + description = '' + DEPRECATED, use `filters`. + ''; + type = lib.types.listOf lib.types.str; + default = []; + example = [".git"]; + }; + filters = lib.mkOption { + type = lib.types.listOf filter; + default = []; + example = [ + { + type = "include"; + pattern = "*.nix"; + } + { + type = "include"; + pattern = "*/"; + } + { + type = "exclude"; + pattern = "*"; + } + ]; + }; + deleteExcluded = lib.mkOption { + default = true; + type = lib.types.bool; + }; }; }; git = lib.types.submodule { @@ -81,12 +145,20 @@ type = lib.types.listOf lib.types.str; }; }; + fetchAlways = lib.mkOption { + type = lib.types.bool; + default = false; + }; ref = lib.mkOption { type = lib.types.str; # TODO lib.types.git.ref }; url = lib.mkOption { type = lib.types.str; # TODO lib.types.git.url }; + shallow = lib.mkOption { + default = false; + type = lib.types.bool; + }; }; }; pass = lib.types.submodule { @@ -99,6 +171,21 @@ }; }; }; + passage = lib.types.submodule { + options = { + age = lib.mkOption { + default = "age"; + type = lib.types.pathname; + }; + dir = lib.mkOption { + type = lib.types.pathname; + }; + identities_file = lib.mkOption { + default = toString ~/.passage/identities; + type = lib.types.pathname; + }; + }; + }; pipe = lib.types.submodule { options = { command = lib.mkOption { diff --git a/pkgs/default.nix b/pkgs/default.nix index 682b6b4..76c7f11 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -1,15 +1,7 @@ { overlays ? [], ... }@args: -let - nix-writers = builtins.fetchGit { - url = https://cgit.krebsco.de/nix-writers/; - rev = "c528cf970e292790b414b4c1c8c8e9d7e73b2a71"; - }; -in - import <nixpkgs> (args // { overlays = [ (import ./overlay.nix) - (import "${nix-writers}/pkgs") ] ++ overlays; }) diff --git a/pkgs/krops/default.nix b/pkgs/krops/default.nix index a9a9e34..04c38cf 100644 --- a/pkgs/krops/default.nix +++ b/pkgs/krops/default.nix @@ -2,56 +2,137 @@ let lib = import ../../lib; in -{ exec, nix, openssh, populate, writeDash }: rec { +{ nix, openssh, populate, writers }: rec { - build = target: - exec "build.${target.host}" rec { - filename = "${openssh}/bin/ssh"; - argv = [ - filename - "-l" target.user - "-p" target.port - "-t" - target.host - (lib.concatStringsSep " " [ - "nix build" - "-I ${lib.escapeShellArg target.path}" - "--no-link -f '<nixpkgs/nixos>'" - "config.system.build.toplevel" - ]) - ]; - }; + rebuild = { + useNixOutputMonitor + }: + args: target: + runShell target {} + (withNixOutputMonitor target useNixOutputMonitor /* sh */ '' + NIX_PATH=${lib.escapeShellArg target.path} \ + nixos-rebuild ${lib.escapeShellArgs args} + ''); - rebuild = args: target: - exec "rebuild.${target.host}" rec { - filename = "${openssh}/bin/ssh"; - argv = [ - filename - "-l" target.user - "-p" target.port - target.host - "nixos-rebuild -I ${lib.escapeShellArg target.path} ${ - lib.concatMapStringsSep " " lib.escapeShellArg args - }" - ]; - }; + runShell = target: { + allocateTTY ? false + }: command: + let + command' = /* sh */ '' + ${lib.optionalString target.sudo "sudo"} \ + /bin/sh -c ${lib.escapeShellArg command} + ''; + in + if lib.isLocalTarget target + then command' + else + writers.writeDash "krops.${target.host}.${lib.firstWord command}" '' + exec ${openssh}/bin/ssh ${lib.escapeShellArgs (lib.flatten [ + (lib.mkUserPortSSHOpts target) + (if allocateTTY then "-t" else "-T") + target.extraOptions + target.host + command' + ])} + ''; - writeDeploy = name: { backup ? false, force ? false, source, target }: let + withNixOutputMonitor = target: mode_: command: let + mode = + lib.getAttr (lib.typeOf mode_) { + bool = lib.toJSON mode_; + string = mode_; + }; + in /* sh */ '' + printf '# use nix-output-monitor: %s\n' ${lib.escapeShellArg mode} >&2 + ${lib.getAttr mode rec { + opportunistic = /* sh */ '' + if command -v nom >/dev/null; then + ${optimistic} + else + ${false} + fi + ''; + optimistic = /* sh */ '' + (${command}) 2>&1 | nom + ''; + pessimistic = /* sh */ '' + NIX_PATH=${lib.escapeShellArg target.path} \ + nix-shell -p nix-output-monitor --run ${lib.escapeShellArg optimistic} + ''; + true = /* sh */ '' + if command -v nom >/dev/null; then + ${optimistic} + else + ${pessimistic} + fi + ''; + false = command; + }} + ''; + + writeCommand = name: { + command ? (targetPath: "echo ${targetPath}"), + backup ? false, + force ? false, + allocateTTY ? false, + source, + target + }: let target' = lib.mkTarget target; in - writeDash name '' + writers.writeDash name '' set -efu ${populate { inherit backup force source; target = target'; }} - ${rebuild ["dry-build"] target'} - ${build target'} - ${rebuild ["switch"] target'} + ${runShell target' { inherit allocateTTY; } (command target'.path)} ''; - writeTest = name: { backup ? false, force ? false, source, target }: let + writeDeploy = name: { + backup ? false, + buildTarget ? null, + crossDeploy ? false, + fast ? null, + force ? false, + operation ? "switch", + source, + target, + useNixOutputMonitor ? "opportunistic" + }: let + buildTarget' = + if buildTarget == null + then target' + else lib.mkTarget buildTarget; + target' = lib.mkTarget target; + in + lib.traceIf (fast != null) "writeDeploy: it's now always fast, setting the `fast` attribute is deprecated and will be removed in future" ( + writers.writeDash name '' + set -efu + ${lib.optionalString (buildTarget' != target') + (populate { inherit backup force source; target = buildTarget'; })} + ${populate { inherit backup force source; target = target'; }} + ${rebuild { inherit useNixOutputMonitor; } ([ + operation + ] ++ lib.optionals crossDeploy [ + "--no-build-nix" + ] ++ lib.optionals (buildTarget' != target') [ + "--build-host" "${buildTarget'.user}@${buildTarget'.host}" + "--target-host" "${target'.user}@${target'.host}" + ] ++ lib.optionals target'.sudo [ + "--use-remote-sudo" + ]) buildTarget'} + '' + ); + + writeTest = name: { + backup ? false, + force ? false, + source, + target, + trace ? false + }: let target' = lib.mkTarget target; in assert lib.isLocalTarget target'; - writeDash name '' + writers.writeDash name '' set -efu ${populate { inherit backup force source; target = target'; }} >&2 NIX_PATH=${lib.escapeShellArg target'.path} \ @@ -59,8 +140,7 @@ in -A system \ --keep-going \ --no-out-link \ - --show-trace \ + ${lib.optionalString trace "--show-trace"} \ '<nixpkgs/nixos>' ''; - } diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix index ad70d2e..7129b90 100644 --- a/pkgs/populate/default.nix +++ b/pkgs/populate/default.nix @@ -1,12 +1,12 @@ with import ../../lib; with shell; -{ coreutils, dash, findutils, git, jq, openssh, pass, rsync, writeDash }: +{ coreutils, dash, findutils, git, jq, openssh, pass, passage, rsync, writers }: let check = { force, target }: let sentinelFile = "${target.path}/.populate"; - in shell' target /* sh */ '' + in runShell target /* sh */ '' ${optionalString force /* sh */ '' mkdir -vp ${quote (dirOf sentinelFile)} >&2 touch ${quote sentinelFile} @@ -23,7 +23,7 @@ let do-backup = { target }: let sentinelFile = "${target.path}/.populate"; in - shell' target /* sh */ '' + runShell target /* sh */ '' if ! test -d ${quote sentinelFile}; then >&2 printf 'error" sentinel file is not a directory: %s\n' ${quote ( optionalString (!isLocalTarget target) "${target.host}:" + @@ -40,20 +40,40 @@ let ${target.path}/.populate/backup/ ''; - pop.derivation = target: source: shell' target /* sh */ '' + pop.derivation = target: source: runShell target /* sh */ '' nix-build -E ${quote source.text} -o ${quote target.path} >&2 ''; pop.file = target: source: let - configAttrs = ["useChecksum"]; - config = filterAttrs (name: _: elem name configAttrs) source; + config = rsyncDefaultConfig // derivedConfig // sourceConfig; + derivedConfig = { + useChecksum = + if isStorePath source.path + then true + else rsyncDefaultConfig.useChecksum; + }; + sourceConfig = + filterAttrs (name: _: elem name (attrNames rsyncDefaultConfig)) source; + sourcePath = + if isStorePath source.path + then quote (toString source.path) + else quote source.path; in - rsync' target config (quote source.path); + rsync' target config sourcePath; - pop.git = target: source: shell' target /* sh */ '' + pop.git = target: source: runShell target /* sh */ '' set -efu + # Remove target path if it doesn't look like a git worktree. + # This can happen e.g. when it had a different type earlier. + if ! test -e ${quote target.path}/.git; then + rm -fR ${quote target.path} + fi if ! test -e ${quote target.path}; then - git clone --recurse-submodules ${quote source.url} ${quote target.path} + ${if source.shallow then /* sh */ '' + git init ${quote target.path} + '' else /* sh */ '' + git clone --recurse-submodules ${quote source.url} ${quote target.path} + ''} fi cd ${quote target.path} if ! url=$(git config remote.origin.url); then @@ -66,9 +86,21 @@ let hash=${quote source.ref} if ! test "$(git log --format=%H -1)" = "$hash"; then - if ! git log -1 "$hash" >/dev/null 2>&1; then - git fetch origin - fi + ${if source.fetchAlways then /* sh */ '' + ${if source.shallow then /* sh */ '' + git fetch --depth=1 origin "$hash" + '' else /* sh */ '' + git fetch origin + ''} + '' else /* sh */ '' + if ! git log -1 "$hash" >/dev/null 2>&1; then + ${if source.shallow then /* sh */ '' + git fetch --depth=1 origin "$hash" + '' else /* sh */ '' + git fetch origin + ''} + fi + ''} git reset --hard "$hash" >&2 git submodule update --init --recursive fi @@ -87,8 +119,16 @@ let umask 0077 if test -e ${quote source.dir}/.git; then - local_pass_info=${quote source.name}\ $(${git}/bin/git -C ${quote source.dir} log -1 --format=%H ${quote source.name}) - remote_pass_info=$(${shell' target /* sh */ '' + local_pass_info=${quote source.name}\ $( + ${git}/bin/git -C ${quote source.dir} log -1 --format=%H ${quote source.name} + # we append a hash for every symlink, otherwise we would miss updates on + # files where the symlink points to + ${findutils}/bin/find ${quote source.dir}/${quote source.name} -type l \ + -exec ${coreutils}/bin/realpath {} + | + ${coreutils}/bin/sort | + ${findutils}/bin/xargs -r -n 1 ${git}/bin/git -C ${quote source.dir} log -1 --format=%H + ) + remote_pass_info=$(${runShell target /* sh */ '' cat ${quote target.path}/.pass_info || : ''}) @@ -103,45 +143,80 @@ let rm -fR "$tmp_dir" } - ${findutils}/bin/find ${quote passPrefix} -type f | + ${findutils}/bin/find ${quote passPrefix} -type f -follow ! -name .gpg-id | while read -r gpg_path; do rel_name=''${gpg_path#${quote passPrefix}} rel_name=''${rel_name%.gpg} pass_date=$( - ${git}/bin/git -C ${quote source.dir} log -1 --format=%aI "$gpg_path" + if test -e ${quote source.dir}/.git; then + ${git}/bin/git -C ${quote source.dir} log -1 --format=%aI "$gpg_path" + fi ) pass_name=${quote source.name}/$rel_name tmp_path=$tmp_dir/$rel_name ${coreutils}/bin/mkdir -p "$(${coreutils}/bin/dirname "$tmp_path")" PASSWORD_STORE_DIR=${quote source.dir} ${pass}/bin/pass show "$pass_name" > "$tmp_path" - ${coreutils}/bin/touch -d "$pass_date" "$tmp_path" + if [ -n "$pass_date" ]; then + ${coreutils}/bin/touch -d "$pass_date" "$tmp_path" + fi done if test -n "''${local_pass_info-}"; then echo "$local_pass_info" > "$tmp_dir"/.pass_info fi - ${rsync' target {} /* sh */ "$tmp_dir"} + ${rsync' target rsyncDefaultConfig /* sh */ "$tmp_dir"} + ''; + + pop.passage = target: source: /* sh */ '' + set -efu + + export PASSAGE_AGE=${quote source.age} + export PASSAGE_DIR=${quote source.dir} + export PASSAGE_IDENTITIES_FILE=${quote source.identities_file} + + umask 0077 + + tmp_dir=$(${coreutils}/bin/mktemp -dt populate-passage.XXXXXXXX) + trap cleanup EXIT + cleanup() { + rm -fR "$tmp_dir" + } + + ${findutils}/bin/find "$PASSAGE_DIR" -type f -name \*.age -follow | + while read -r age_path; do + + rel_name=''${age_path#$PASSAGE_DIR} + rel_name=''${rel_name%.age} + + tmp_path=$tmp_dir/$rel_name + + ${coreutils}/bin/mkdir -p "$(${coreutils}/bin/dirname "$tmp_path")" + ${passage}/bin/passage show "$rel_name" > "$tmp_path" + ${coreutils}/bin/touch -r "$age_path" "$tmp_path" + done + + ${rsync' target rsyncDefaultConfig /* sh */ "$tmp_dir"} ''; pop.pipe = target: source: /* sh */ '' ${quote source.command} | { - ${shell' target /* sh */ "cat > ${quote target.path}"} + ${runShell target /* sh */ "cat > ${quote target.path}"} } ''; # TODO rm -fR instead of ln -f? - pop.symlink = target: source: shell' target /* sh */ '' + pop.symlink = target: source: runShell target /* sh */ '' ln -fnsT ${quote source.target} ${quote target.path} ''; populate = target: name: source: let source' = source.${source.type}; target' = target // { path = "${target.path}/${name}"; }; - in writeDash "populate.${target'.host}.${name}" '' + in writers.writeDash "populate.${target'.host}.${name}" '' set -efu ${pop.${source.type} target' source'} ''; @@ -152,38 +227,56 @@ let source_path=$source_path/ fi ${rsync}/bin/rsync \ - ${optionalString (config.useChecksum or false) /* sh */ "--checksum"} \ + ${optionalString config.useChecksum /* sh */ "--checksum"} \ + ${optionalString target.sudo /* sh */ "--rsync-path=\"sudo rsync\""} \ + ${concatMapStringsSep " " + (pattern: /* sh */ "--exclude ${quote pattern}") + config.exclude} \ + ${concatMapStringsSep " " + (filter: /* sh */ "--${filter.type} ${quote filter.pattern}") + config.filters} \ -e ${quote (ssh' target)} \ -vFrlptD \ - --delete-excluded \ + ${optionalString config.deleteExcluded /* sh */ "--delete-excluded"} \ "$source_path" \ ${quote ( - optionalString (!isLocalTarget target) - "${target.user}@${target.host}:" + + optionalString (!isLocalTarget target) ( + (optionalString (target.user != "") "${target.user}@") + + "${target.host}:" + ) + target.path )} \ >&2 ''; - shell' = target: script: + rsyncDefaultConfig = { + useChecksum = false; + exclude = []; + filters = []; + deleteExcluded = true; + }; + + runShell = target: command: if isLocalTarget target - then script - else /* sh */ '' - ${ssh' target} ${quote target.host} ${quote script} - ''; + then command + else + if target.sudo then /* sh */ '' + ${ssh' target} ${quote target.host} ${quote "sudo bash -c ${quote command}"} + '' else '' + ${ssh' target} ${quote target.host} ${quote command} + ''; - ssh' = target: concatMapStringsSep " " quote [ + ssh' = target: concatMapStringsSep " " quote (flatten [ "${openssh}/bin/ssh" - "-l" target.user - "-o" "ControlPersist=no" - "-p" target.port |