aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--README.md77
-rw-r--r--ci.nix4
-rw-r--r--flake.lock27
-rw-r--r--flake.nix29
-rw-r--r--lib/default.nix8
-rw-r--r--lib/types/populate.nix34
-rw-r--r--pkgs/krops/default.nix124
-rw-r--r--pkgs/populate/default.nix87
8 files changed, 320 insertions, 70 deletions
diff --git a/README.md b/README.md
index 6648fa0..d8268e5 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,8 @@ krops is a lightweight toolkit to deploy NixOS systems, remotely or locally.
## Some Features
- store your secrets in [password store](https://www.passwordstore.org/)
-- build your system remotely
+ or [passage](https://github.com/FiloSottile/passage)
+- build your systems remotely
- minimal overhead (it's basically just `nixos-rebuild switch`!)
- run from custom nixpkgs branch/checkout/fork
@@ -19,8 +20,8 @@ Create a file named `krops.nix` (name doesn't matter) with following content:
let
krops = (import <nixpkgs> {}).fetchgit {
url = https://cgit.krebsco.de/krops/;
- rev = "v1.17.0";
- sha256 = "150jlz0hlb3ngf9a1c9xgcwzz1zz8v2lfgnzw08l3ajlaaai8smd";
+ rev = "v1.25.0";
+ sha256 = "07mg3iaqjf1w49vmwfchi7b1w55bh7rvsbgicp2m47gnj9alwdb6";
};
lib = import "${krops}/lib";
@@ -96,7 +97,7 @@ pkgs.krops.writeDeploy "deploy" {
}
```
For more details about the `target` attribute, please check the `mkTarget`
-function in [lib/default.nix](lib/defaults.nix).
+function in [lib/default.nix](lib/default.nix).
### `backup` (optional, defaults to false)
@@ -125,13 +126,40 @@ architecture.
### `fast` (optional, defaults to false)
-Run `nixos-rebuild switch` immediately without building the system
-in a dedicated `nix build` step.
+Run `nixos-rebuild` immediately without building the system in a dedicated `nix
+build` step.
### `force` (optional, defaults to false)
Create the sentinel file (`/var/src/.populate`) before syncing the new source.
+### `operation` (optional, defaults to "switch")
+
+Specifies which `nixos-rebuild` operation to perform.
+
+### `useNixOutputMonitor` (optional, defaults to `"opportunistic"`)
+
+Specifies when to pipe `nixos-rebuild`'s output to
+[nom](https://github.com/maralorn/nix-output-monitor).
+
+Supported values:
+
+* `"opportunistic"` (default) -
+ Use `nom` only if it is present on the target machine.
+
+* `"optimistic"` -
+ Use `nom`, assuming it is present on the target machine.
+
+* `"pessimistic"` -
+ Use `nom` via `nix-shell` on the target machine.
+
+* `true` -
+ Use `nom`.
+ If it is not present on the target machine, then use it via `nix-shell`.
+
+* `false` -
+ Don't use `nom`
+
## writeTest
Very similiar to writeDeploy, but just builds the system on the target without
@@ -152,6 +180,10 @@ below `/var/src`, and executes `NIX_PATH=/var/src nix-build -A system '<nixpkgs/
[see `writeDeploy`](#writeDeploy)
+### `trace` (optional, defaults to false)
+
+run nix-build with `--show-trace`
+
## writeCommand
This can be used to run other commands than `nixos-rebuild` or pre/post build hooks.
@@ -185,6 +217,10 @@ pkgs.krops.writeCommand "deploy-with-swap" {
[see `writeDeploy`](#writeDeploy)
+### `allocateTTY` (optional, defaults to false)
+
+whether the ssh session should do a pseudo-terminal allocation.
+sets `-t` on the ssh command.
## Source Types
@@ -206,13 +242,15 @@ using [`rsync`](https://rsync.samba.org/).
Supported attributes:
* `path` -
- absolute path to files that should by transfered
+ absolute path to files that should by transferred.
* `useChecksum` (optional) -
boolean that controls whether file contents should be checked to decide
whether a file has changed. This is useful when `path` points at files
with mangled timestamps, e.g. the Nix store.
+ The default value is `true` if `path` is a derivation, and `false` otherwise.
+
* `filters` (optional)
List of filters that should be passed to [`rsync`](https://rsync.samba.org/).
Filters are specified as attribute sets with the attributes `type` and
@@ -261,6 +299,29 @@ Supported attributes:
sub-directory in the password store.
+### `passage`
+
+The passage source type decrypts files from a local
+[passage store](https://github.com/FiloSottile/passage)
+and transfers them to the target using
+[`rsync`](https://rsync.samba.org/).
+
+Supported attributes:
+
+* `dir` -
+ Path to the passage store.
+ For a partial transfer, this may point to a subdirectory.
+ Example: `~/.passage/store/hosts/MYHOSTNAME`
+
+* `identities_file` (optional) -
+ Path to the identities file.
+ Defaults to `~/.passage/identities`.
+
+* `age` (optional) -
+ Path of the age binary.
+ Defaults to `age` (absolute path gets resolved using `passage`'s search path.)
+
+
### `pipe`
Executes a local command, capture its stdout, and send that as a file to the
@@ -292,6 +353,6 @@ Supported attributes:
Comments, questions, pull-requests and patches, etc. are very welcome, and can be directed
at:
-- IRC: #krebs at freenode
+- IRC: #krebs at hackint
- Mail: [spam@krebsco.de](mailto:spam@krebsco.de)
- Github: https://github.com/krebs/krops/
diff --git a/ci.nix b/ci.nix
index 258a4e6..c57f4d7 100644
--- a/ci.nix
+++ b/ci.nix
@@ -5,7 +5,7 @@ let
pkgs = import "${krops}/pkgs" {};
source = lib.evalSource [{
- nixos-config.file = toString (pkgs.writeText "nixos-config" ''
+ nixos-config.file = pkgs.writeText "nixos-config" ''
{ pkgs, ... }: {
fileSystems."/" = { device = "/dev/sda1"; };
@@ -13,7 +13,7 @@ let
services.openssh.enable = true;
environment.systemPackages = [ pkgs.git ];
}
- '');
+ '';
nixpkgs.symlink = toString <nixpkgs>;
}];
in {
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 0000000..51e2759
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,27 @@
+{
+ "nodes": {
+ "nixpkgs": {
+ "locked": {
+ "lastModified": 1689940971,
+ "narHash": "sha256-397xShPnFqPC59Bmpo3lS+/Aw0yoDRMACGo1+h2VJMo=",
+ "owner": "NixOS",
+ "repo": "nixpkgs",
+ "rev": "9ca785644d067445a4aa749902b29ccef61f7476",
+ "type": "github"
+ },
+ "original": {
+ "owner": "NixOS",
+ "ref": "nixos-unstable",
+ "repo": "nixpkgs",
+ "type": "github"
+ }
+ },
+ "root": {
+ "inputs": {
+ "nixpkgs": "nixpkgs"
+ }
+ }
+ },
+ "root": "root",
+ "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000..ec13152
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,29 @@
+{
+ description = "krops - krebs operations";
+
+ inputs = {
+ nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+ };
+
+ outputs = { self, nixpkgs, ... }:
+ let
+ supportedSystems = [
+ "x86_64-linux"
+ "i686-linux"
+ "aarch64-linux"
+ "riscv64-linux"
+ ];
+ forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
+ in
+ {
+ lib = forAllSystems (system:
+ let
+ pkgs = nixpkgs.legacyPackages.${system};
+ krops = pkgs.callPackage ./pkgs/krops {};
+ populate = pkgs.callPackage ./pkgs/populate {};
+ in {
+ inherit populate;
+ inherit (krops) rebuild runShell withNixOutputMonitor writeCommand writeDeploy writeTest;
+ });
+ };
+}
diff --git a/lib/default.nix b/lib/default.nix
index 357f5b5..3bbd754 100644
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -57,9 +57,9 @@ let {
elemAt' = xs: i: if lib.length xs > i then lib.elemAt xs i else null;
filterNull = lib.filterAttrs (n: v: v != null);
in {
- user = lib.getEnv "LOGNAME";
+ user = lib.maybeEnv "LOGNAME" null;
host = lib.maybeEnv "HOSTNAME" (lib.maybeHostName "localhost");
- port = "22";
+ port = null;
path = "/var/src";
sudo = false;
extraOptions = [];
@@ -70,6 +70,10 @@ let {
path = elemAt' parse 6;
} else s);
+ mkUserPortSSHOpts = target:
+ (lib.optionals (target.user != null) ["-l" target.user]) ++
+ (lib.optionals (target.port != null) ["-p" target.port]);
+
shell = let
isSafeChar = lib.testString "[-+./0-9:=A-Z_a-z]";
quoteChar = c:
diff --git a/lib/types/populate.nix b/lib/types/populate.nix
index 894b8cc..6264f99 100644
--- a/lib/types/populate.nix
+++ b/lib/types/populate.nix
@@ -21,11 +21,15 @@
};
file = lib.mkOption {
apply = x:
- if lib.types.absolute-pathname.check x
+ if lib.types.absolute-pathname.check x || lib.types.package.check x
then { path = x; }
else x;
default = null;
- type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.file);
+ type = lib.types.nullOr (lib.types.oneOf [
+ lib.types.absolute-pathname
+ lib.types.package
+ source-types.file
+ ]);
};
git = lib.mkOption {
default = null;
@@ -35,6 +39,17 @@
default = null;
type = lib.types.nullOr source-types.pass;
};
+ passage = lib.mkOption {
+ apply = x:
+ if lib.types.pathname.check x
+ then { dir = x; }
+ else x;
+ default = null;
+ type = lib.types.nullOr (lib.types.oneOf [
+ lib.types.pathname
+ source-types.passage
+ ]);
+ };
pipe = lib.mkOption {
apply = x:
if lib.types.absolute-pathname.check x
@@ -156,6 +171,21 @@
};
};
};
+ passage = lib.types.submodule {
+ options = {
+ age = lib.mkOption {
+ default = "age";
+ type = lib.types.pathname;
+ };
+ dir = lib.mkOption {
+ type = lib.types.pathname;
+ };
+ identities_file = lib.mkOption {
+ default = toString ~/.passage/identities;
+ type = lib.types.pathname;
+ };
+ };
+ };
pipe = lib.types.submodule {
options = {
command = lib.mkOption {
diff --git a/pkgs/krops/default.nix b/pkgs/krops/default.nix
index 8336b51..04c38cf 100644
--- a/pkgs/krops/default.nix
+++ b/pkgs/krops/default.nix
@@ -4,40 +4,77 @@ in
{ nix, openssh, populate, writers }: rec {
- build = target:
- runShell target (lib.concatStringsSep " " [
- "nix build"
- "-I ${lib.escapeShellArg target.path}"
- "--no-link -f '<nixpkgs/nixos>'"
- "config.system.build.toplevel"
- ]);
+ rebuild = {
+ useNixOutputMonitor
+ }:
+ args: target:
+ runShell target {}
+ (withNixOutputMonitor target useNixOutputMonitor /* sh */ ''
+ NIX_PATH=${lib.escapeShellArg target.path} \
+ nixos-rebuild ${lib.escapeShellArgs args}
+ '');
- rebuild = args: target:
- runShell target "nixos-rebuild -I ${lib.escapeShellArg target.path} ${
- lib.concatMapStringsSep " " lib.escapeShellArg args
- }";
-
- runShell = target: command:
+ runShell = target: {
+ allocateTTY ? false
+ }: command:
let
- command' = if target.sudo then "sudo ${command}" else command;
+ command' = /* sh */ ''
+ ${lib.optionalString target.sudo "sudo"} \
+ /bin/sh -c ${lib.escapeShellArg command}
+ '';
in
if lib.isLocalTarget target
then command'
else
writers.writeDash "krops.${target.host}.${lib.firstWord command}" ''
exec ${openssh}/bin/ssh ${lib.escapeShellArgs (lib.flatten [
- (lib.optionals (target.user != "") ["-l" target.user])
- "-p" target.port
- "-T"
+ (lib.mkUserPortSSHOpts target)
+ (if allocateTTY then "-t" else "-T")
target.extraOptions
target.host
- command'])}
+ command'
+ ])}
'';
+ withNixOutputMonitor = target: mode_: command: let
+ mode =
+ lib.getAttr (lib.typeOf mode_) {
+ bool = lib.toJSON mode_;
+ string = mode_;
+ };
+ in /* sh */ ''
+ printf '# use nix-output-monitor: %s\n' ${lib.escapeShellArg mode} >&2
+ ${lib.getAttr mode rec {
+ opportunistic = /* sh */ ''
+ if command -v nom >/dev/null; then
+ ${optimistic}
+ else
+ ${false}
+ fi
+ '';
+ optimistic = /* sh */ ''
+ (${command}) 2>&1 | nom
+ '';
+ pessimistic = /* sh */ ''
+ NIX_PATH=${lib.escapeShellArg target.path} \
+ nix-shell -p nix-output-monitor --run ${lib.escapeShellArg optimistic}
+ '';
+ true = /* sh */ ''
+ if command -v nom >/dev/null; then
+ ${optimistic}
+ else
+ ${pessimistic}
+ fi
+ '';
+ false = command;
+ }}
+ '';
+
writeCommand = name: {
command ? (targetPath: "echo ${targetPath}"),
backup ? false,
force ? false,
+ allocateTTY ? false,
source,
target
}: let
@@ -46,17 +83,19 @@ in
writers.writeDash name ''
set -efu
${populate { inherit backup force source; target = target'; }}
- ${runShell target' (command target'.path)}
+ ${runShell target' { inherit allocateTTY; } (command target'.path)}
'';
writeDeploy = name: {
backup ? false,
buildTarget ? null,
crossDeploy ? false,
- fast ? false,
+ fast ? null,
force ? false,
+ operation ? "switch",
source,
- target
+ target,
+ useNixOutputMonitor ? "opportunistic"
}: let
buildTarget' =
if buildTarget == null
@@ -64,32 +103,31 @@ in
else lib.mkTarget buildTarget;
target' = lib.mkTarget target;
in
- writers.writeDash name ''
- set -efu
- ${lib.optionalString (buildTarget' != target')
- (populate { inherit backup force source; target = buildTarget'; })}
- ${populate { inherit backup force source; target = target'; }}
- ${lib.optionalString (! fast) ''
- ${rebuild ["dry-build"] buildTarget'}
- ${build buildTarget'}
- ''}
- ${rebuild ([
- "switch"
- ] ++ lib.optionals crossDeploy [
- "--no-build-nix"
- ] ++ lib.optionals (buildTarget' != target') [
- "--build-host" "${buildTarget'.user}@${buildTarget'.host}"
- "--target-host" "${target'.user}@${target'.host}"
- ] ++ lib.optionals target'.sudo [
- "--use-remote-sudo"
- ]) buildTarget'}
- '';
+ lib.traceIf (fast != null) "writeDeploy: it's now always fast, setting the `fast` attribute is deprecated and will be removed in future" (
+ writers.writeDash name ''
+ set -efu
+ ${lib.optionalString (buildTarget' != target')
+ (populate { inherit backup force source; target = buildTarget'; })}
+ ${populate { inherit backup force source; target = target'; }}
+ ${rebuild { inherit useNixOutputMonitor; } ([
+ operation
+ ] ++ lib.optionals crossDeploy [
+ "--no-build-nix"
+ ] ++ lib.optionals (buildTarget' != target') [
+ "--build-host" "${buildTarget'.user}@${buildTarget'.host}"
+ "--target-host" "${target'.user}@${target'.host}"
+ ] ++ lib.optionals target'.sudo [
+ "--use-remote-sudo"
+ ]) buildTarget'}
+ ''
+ );
writeTest = name: {
backup ? false,
force ? false,
source,
- target
+ target,
+ trace ? false
}: let
target' = lib.mkTarget target;
in
@@ -102,7 +140,7 @@ in
-A system \
--keep-going \
--no-out-link \
- --show-trace \
+ ${lib.optionalString trace "--show-trace"} \
'<nixpkgs/nixos>'
'';
}
diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix
index 517b12e..7129b90 100644
--- a/pkgs/populate/default.nix
+++ b/pkgs/populate/default.nix
@@ -1,7 +1,7 @@
with import ../../lib;
with shell;
-{ coreutils, dash, findutils, git, jq, openssh, pass, rsync, writers }:
+{ coreutils, dash, findutils, git, jq, openssh, pass, passage, rsync, writers }:
let
check = { force, target }: let
@@ -45,18 +45,34 @@ let
'';
pop.file = target: source: let
- configAttrs = ["useChecksum" "exclude" "filters" "deleteExcluded"];
- config = filterAttrs (name: _: elem name configAttrs) source;
+ config = rsyncDefaultConfig // derivedConfig // sourceConfig;
+ derivedConfig = {
+ useChecksum =
+ if isStorePath source.path
+ then true
+ else rsyncDefaultConfig.useChecksum;
+ };
+ sourceConfig =
+ filterAttrs (name: _: elem name (attrNames rsyncDefaultConfig)) source;
+ sourcePath =
+ if isStorePath source.path
+ then quote (toString source.path)
+ else quote source.path;
in
- rsync' target config (quote source.path);
+ rsync' target config sourcePath;
pop.git = target: source: runShell target /* sh */ ''
set -efu
+ # Remove target path if it doesn't look like a git worktree.
+ # This can happen e.g. when it had a different type earlier.
+ if ! test -e ${quote target.path}/.git; then
+ rm -fR ${quote target.path}
+ fi
if ! test -e ${quote target.path}; then
${if source.shallow then /* sh */ ''
git init ${quote target.path}
'' else /* sh */ ''
- git clone --recurse-submodules ${quote source.url} ${quote target.path}
+ git clone --recurse-submodules ${quote source.url} ${quote target.path}
''}
fi
cd ${quote target.path}
@@ -103,7 +119,15 @@ let
umask 0077
if test -e ${quote source.dir}/.git; then
- local_pass_info=${quote source.name}\ $(${git}/bin/git -C ${quote source.dir} log -1 --format=%H ${quote source.name})
+ local_pass_info=${quote source.name}\ $(
+ ${git}/bin/git -C ${quote source.dir} log -1 --format=%H ${quote source.name}
+ # we append a hash for every symlink, otherwise we would miss updates on
+ # files where the symlink points to
+ ${findutils}/bin/find ${quote source.dir}/${quote source.name} -type l \
+ -exec ${coreutils}/bin/realpath {} + |
+ ${coreutils}/bin/sort |
+ ${findutils}/bin/xargs -r -n 1 ${git}/bin/git -C ${quote source.dir} log -1 --format=%H
+ )
remote_pass_info=$(${runShell target /* sh */ ''
cat ${quote target.path}/.pass_info || :
''})
@@ -144,7 +168,38 @@ let
echo "$local_pass_info" > "$tmp_dir"/.pass_info
fi
- ${rsync' target {} /* sh */ "$tmp_dir"}
+ ${rsync' target rsyncDefaultConfig /* sh */ "$tmp_dir"}
+ '';
+
+ pop.passage = target: source: /* sh */ ''
+ set -efu
+
+ export PASSAGE_AGE=${quote source.age}
+ export PASSAGE_DIR=${quote source.dir}
+ export PASSAGE_IDENTITIES_FILE=${quote source.identities_file}
+
+ umask 0077
+
+ tmp_dir=$(${coreutils}/bin/mktemp -dt populate-passage.XXXXXXXX)
+ trap cleanup EXIT
+ cleanup() {
+ rm -fR "$tmp_dir"
+ }
+
+ ${findutils}/bin/find "$PASSAGE_DIR" -type f -name \*.age -follow |
+ while read -r age_path; do
+
+ rel_name=''${age_path#$PASSAGE_DIR}
+ rel_name=''${rel_name%.age}
+
+ tmp_path=$tmp_dir/$rel_name
+
+ ${coreutils}/bin/mkdir -p "$(${coreutils}/bin/dirname "$tmp_path")"
+ ${passage}/bin/passage show "$rel_name" > "$tmp_path"
+ ${coreutils}/bin/touch -r "$age_path" "$tmp_path"
+ done
+
+ ${rsync' target rsyncDefaultConfig /* sh */ "$tmp_dir"}
'';
pop.pipe = target: source: /* sh */ ''
@@ -172,17 +227,17 @@ let
source_path=$source_path/
fi
${rsync}/bin/rsync \
- ${optionalString (config.useChecksum or false) /* sh */ "--checksum"} \
+ ${optionalString config.useChecksum /* sh */ "--checksum"} \
${optionalString target.sudo /* sh */ "--rsync-path=\"sudo rsync\""} \
${concatMapStringsSep " "
(pattern: /* sh */ "--exclude ${quote pattern}")
- (config.exclude or [])} \
+ config.exclude} \
${concatMapStringsSep " "
(filter: /* sh */ "--${filter.type} ${quote filter.pattern}")
- (config.filters or [])} \
+ config.filters} \
-e ${quote (ssh' target)} \
-vFrlptD \
- ${optionalString (config.deleteExcluded or true) /* sh */ "--delete-excluded"} \
+ ${optionalString config.deleteExcluded /* sh */ "--delete-excluded"} \
"$source_path" \
${quote (
optionalString (!isLocalTarget target) (
@@ -194,6 +249,13 @@ let
>&2
'';
+ rsyncDefaultConfig = {
+ useChecksum = false;
+ exclude = [];
+ filters = [];
+ deleteExcluded = true;
+ };
+
runShell = target: command:
if isLocalTarget target
then command
@@ -206,8 +268,7 @@ let
ssh' = target: concatMapStringsSep " " quote (flatten [
"${openssh}/bin/ssh"
- (optionals (target.user != "") ["-l" target.user])
- "-p" target.port
+ (mkUserPortSSHOpts target)
"-T"
target.extraOptions
]);