aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authortv <tv@krebsco.de>2018-02-28 21:02:21 +0100
committertv <tv@krebsco.de>2018-02-28 21:02:21 +0100
commite89cf20d4310070a877c2e24a287659546b561c9 (patch)
tree18e19a41b65d03782a955235b2a67407157eaa9f
import stockholm's deployment tools
https://cgit.krebsco.de/stockholm 877b4104370c1ea9698a449e376e2842d7c372fd
-rw-r--r--lib/default.nix56
-rw-r--r--lib/types/default.nix8
-rw-r--r--lib/types/populate.nix93
-rw-r--r--lib/types/posix.nix54
-rw-r--r--pkgs/default.nix7
-rw-r--r--pkgs/kops/default.nix48
-rw-r--r--pkgs/overlay.nix18
-rw-r--r--pkgs/populate/default.nix20
-rwxr-xr-xpkgs/populate/populate.sh276
9 files changed, 580 insertions, 0 deletions
diff --git a/lib/default.nix b/lib/default.nix
new file mode 100644
index 0000000..7197fe9
--- /dev/null
+++ b/lib/default.nix
@@ -0,0 +1,56 @@
+let {
+
+ body = lib;
+
+ lib = nixpkgs.lib // builtins // {
+
+ evalSource = let
+ eval = source: lib.evalModules {
+ modules = lib.singleton {
+ _file = toString ./.;
+ imports = map (source: { inherit source; }) (lib.toList source);
+ options.source = lib.mkOption {
+ default = {};
+ type = lib.types.attrsOf lib.types.source;
+ };
+ };
+ };
+ sanitize = x: lib.getAttr (lib.typeOf x) {
+ set = lib.mapAttrs
+ (lib.const sanitize)
+ (lib.filterAttrs
+ (name: value: name != "_module" && value != null) x);
+ string = x;
+ };
+ in
+ # This function's return value can be used as pkgs.populate input.
+ source: sanitize (eval source).config.source;
+
+ getHostName = let
+ # We're parsing /etc/hostname here because reading
+ # /proc/sys/kernel/hostname yields ""
+ y = lib.filter lib.types.label.check (lib.splitString "\n" (lib.readFile /etc/hostname));
+ in
+ if lib.length y != 1 then throw "malformed /etc/hostname" else
+ lib.elemAt y 0;
+
+ mkTarget = s: let
+ default = defVal: val: if val != null then val else defVal;
+ parse = lib.match "(([^@]+)@)?(([^:/]+))?(:([^/]+))?(/.*)?" s;
+ elemAt' = xs: i: if lib.length xs > i then lib.elemAt xs i else null;
+ in {
+ user = default (lib.getEnv "LOGNAME") (elemAt' parse 1);
+ host = default (lib.maybeEnv "HOSTNAME" lib.getHostName) (elemAt' parse 3);
+ port = default "22" /* "ssh"? */ (elemAt' parse 5);
+ path = default "/var/src" /* no default? */ (elemAt' parse 6);
+ };
+
+ test = re: x: lib.isString x && lib.testString re x;
+ testString = re: x: lib.match re x != null;
+
+ types = nixpkgs.lib.types // import ./types { lib = body; };
+ };
+
+ nixpkgs.lib = import <nixpkgs/lib>;
+
+}
diff --git a/lib/types/default.nix b/lib/types/default.nix
new file mode 100644
index 0000000..c4bf517
--- /dev/null
+++ b/lib/types/default.nix
@@ -0,0 +1,8 @@
+{ lib }@args: let {
+
+ body = lib.foldl' (res: path: res // import path args) {} [
+ ./populate.nix
+ ./posix.nix
+ ];
+
+}
diff --git a/lib/types/populate.nix b/lib/types/populate.nix
new file mode 100644
index 0000000..3b13df0
--- /dev/null
+++ b/lib/types/populate.nix
@@ -0,0 +1,93 @@
+{ lib }: rec {
+
+ source = lib.types.submodule ({ config, ... }: {
+ options = {
+ type = let
+ known-types = lib.attrNames source-types;
+ type-candidates = lib.filter (k: config.${k} != null) known-types;
+ in lib.mkOption {
+ default = if lib.length type-candidates == 1
+ then lib.head type-candidates
+ else throw "cannot determine type";
+ type = lib.types.enum known-types;
+ };
+ file = lib.mkOption {
+ apply = x:
+ if lib.types.absolute-pathname.check x
+ then { path = x; }
+ else x;
+ default = null;
+ type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.file);
+ };
+ git = lib.mkOption {
+ default = null;
+ type = lib.types.nullOr source-types.git;
+ };
+ pass = lib.mkOption {
+ default = null;
+ type = lib.types.nullOr source-types.pass;
+ };
+ pipe = lib.mkOption {
+ apply = x:
+ if lib.types.absolute-pathname.check x
+ then { command = x; }
+ else x;
+ default = null;
+ type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.pipe);
+ };
+ symlink = lib.mkOption {
+ apply = x:
+ if lib.types.pathname.check x
+ then { target = x; }
+ else x;
+ default = null;
+ type = lib.types.nullOr (lib.types.either lib.types.pathname source-types.symlink);
+ };
+ };
+ });
+
+ source-types = {
+ file = lib.types.submodule {
+ options = {
+ path = lib.mkOption {
+ type = lib.types.absolute-pathname;
+ };
+ };
+ };
+ git = lib.types.submodule {
+ options = {
+ ref = lib.mkOption {
+ type = lib.types.str; # TODO lib.types.git.ref
+ };
+ url = lib.mkOption {
+ type = lib.types.str; # TODO lib.types.git.url
+ };
+ };
+ };
+ pass = lib.types.submodule {
+ options = {
+ dir = lib.mkOption {
+ type = lib.types.absolute-pathname;
+ };
+ name = lib.mkOption {
+ type = lib.types.pathname; # TODO relative-pathname
+ };
+ };
+ };
+ pipe = lib.types.submodule {
+ options = {
+ command = lib.mkOption {
+ type = lib.types.absolute-pathname;
+ };
+ };
+ };
+ symlink = lib.types.submodule {
+ options = {
+ target = lib.mkOption {
+ type = lib.types.pathname; # TODO relative-pathname
+ };
+ };
+ };
+ };
+
+}
diff --git a/lib/types/posix.nix b/lib/types/posix.nix
new file mode 100644
index 0000000..e8f464e
--- /dev/null
+++ b/lib/types/posix.nix
@@ -0,0 +1,54 @@
+{ lib }: rec {
+
+ # RFC952, B. Lexical grammar, <hname>
+ hostname = lib.mkOptionType {
+ name = "hostname";
+ check = x: lib.isString x && lib.all label.check (lib.splitString "." x);
+ merge = lib.mergeOneOption;
+ };
+
+ # RFC952, B. Lexical grammar, <name>
+ # RFC1123, 2.1 Host Names and Numbers
+ label = lib.mkOptionType {
+ name = "label";
+ # TODO case-insensitive labels
+ check = lib.test "[0-9A-Za-z]([0-9A-Za-z-]*[0-9A-Za-z])?";
+ merge = lib.mergeOneOption;
+ };
+
+ # POSIX.1‐2013, 3.278 Portable Filename Character Set
+ filename = lib.mkOptionType {
+ name = "POSIX filename";
+ check = lib.test "([0-9A-Za-z._])[0-9A-Za-z._-]*";
+ merge = lib.mergeOneOption;
+ };
+
+ # POSIX.1‐2013, 3.2 Absolute Pathname
+ absolute-pathname = lib.mkOptionType {
+ name = "POSIX absolute pathname";
+ check = x: lib.isString x && lib.substring 0 1 x == "/" && pathname.check x;
+ merge = lib.mergeOneOption;
+ };
+
+ # POSIX.1‐2013, 3.267 Pathname
+ pathname = lib.mkOptionType {
+ name = "POSIX pathname";
+ check = x:
+ let
+ # The filter is used to normalize paths, i.e. to remove duplicated and
+ # trailing slashes. It also removes leading slashes, thus we have to
+ # check for "/" explicitly below.
+ xs = lib.filter (s: lib.stringLength s > 0) (lib.splitString "/" x);
+ in
+ lib.isString x && (x == "/" || (lib.length xs > 0 && lib.all filename.check xs));
+ merge = lib.mergeOneOption;
+ };
+
+ # POSIX.1-2013, 3.431 User Name
+ username = lib.mkOptionType {
+ name = "POSIX username";
+ check = filename.check;
+ merge = lib.mergeOneOption;
+ };
+
+}
diff --git a/pkgs/default.nix b/pkgs/default.nix
new file mode 100644
index 0000000..639ed13
--- /dev/null
+++ b/pkgs/default.nix
@@ -0,0 +1,7 @@
+{ overlays ? [], ... }@args:
+
+import <nixpkgs> (args // {
+ overlays = overlays ++ [
+ (import ./overlay.nix)
+ ];
+})
diff --git a/pkgs/kops/default.nix b/pkgs/kops/default.nix
new file mode 100644
index 0000000..fc52327
--- /dev/null
+++ b/pkgs/kops/default.nix
@@ -0,0 +1,48 @@
+let
+ lib = import ../../lib // {
+ isLocalTarget = let
+ origin = lib.mkTarget "";
+ in target:
+ target.host == origin.host &&
+ target.user == origin.user;
+ };
+in
+
+{ nix, openssh, populate, writeDash, writeJSON }: {
+
+ writeDeploy = name: { source, target }: let
+ target' = lib.mkTarget target;
+ in
+ writeDash name ''
+ set -efu
+
+ ${populate}/bin/populate \
+ ${target'.user}@${target'.host}:${target'.port}${target'.path} \
+ < ${writeJSON "${name}-source.json" source}
+
+ ${openssh}/bin/ssh \
+ ${target'.user}@${target'.host} -p ${target'.port} \
+ nixos-rebuild switch -I ${target'.path}
+ '';
+
+ writeTest = name: { source, target }: let
+ target' = lib.mkTarget target;
+ in
+ assert lib.isLocalTarget target';
+ writeDash name ''
+ set -efu
+
+ ${populate}/bin/populate --force \
+ ${target'.path} \
+ < ${writeJSON "${name}-source.json" source}
+
+ ${nix}/bin/nix-build \
+ -A config.system.build.toplevel \
+ -I ${target'.path} \
+ --arg modules '[<nixos-config>]' \
+ --no-out-link \
+ --show-trace \
+ '<nixpkgs/nixos/lib/eval-config.nix>'
+ '';
+
+}
diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix
new file mode 100644
index 0000000..89024bd
--- /dev/null
+++ b/pkgs/overlay.nix
@@ -0,0 +1,18 @@
+let
+ lib = import ../lib;
+in
+
+self: super: {
+ kops = self.callPackage ./kops {};
+ populate = self.callPackage ./populate {};
+ writeDash = name: text: self.writeScript name ''
+ #! ${self.dash}/bin/dash
+ ${text}
+ '';
+ writeJSON = name: value: self.runCommand name {
+ json = lib.toJSON value;
+ passAsFile = [ "json" ];
+ } /* sh */ ''
+ ${self.jq}/bin/jq . "$jsonPath" > "$out"
+ '';
+}
diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix
new file mode 100644
index 0000000..acb5a5f
--- /dev/null
+++ b/pkgs/populate/default.nix
@@ -0,0 +1,20 @@
+{ coreutils, findutils, git, gnused, jq, openssh, pass, rsync, runCommand, stdenv }:
+
+let
+ PATH = stdenv.lib.makeBinPath [
+ coreutils
+ findutils
+ git
+ gnused
+ jq
+ openssh
+ pass
+ rsync
+ ];
+in
+
+runCommand "populate-2.2.0" {} ''
+ mkdir -p $out/bin
+ cp ${./populate.sh} $out/bin/populate
+ sed -i '1s,.*,&\nPATH=${PATH},' $out/bin/populate
+''
diff --git a/pkgs/populate/populate.sh b/pkgs/populate/populate.sh
new file mode 100755
index 0000000..670d25b
--- /dev/null
+++ b/pkgs/populate/populate.sh
@@ -0,0 +1,276 @@
+#! /bin/sh
+set -efu
+
+main() {(
+ self=$(readlink -f "$0")
+ basename=${0##*/}
+
+ debug=false
+ force=false
+ origin_host=${HOSTNAME-cat /proc/sys/kernel/hostname}
+ origin_user=$LOGNAME
+ target_spec=
+
+
+ abort=false
+
+ error() {
+ echo "$basename: error: $1" >&2
+ abort=true
+ }
+
+ for arg; do
+ case $arg in
+ --force)
+ force=true
+ ;;
+ -*)
+ error "bad argument: $arg"
+ ;;
+ *)
+ if test -n "$target_spec"; then
+ error "bad argument: $arg"
+ else
+ target_spec=$arg
+ fi
+ ;;
+ esac
+ done
+
+ if test -z "$target_spec"; then
+ error 'no target specified'
+ fi
+
+ if test "$abort" = true; then
+ exit 11
+ fi
+
+ target=$(
+ export origin_host
+ export origin_user
+ echo "$target_spec" | jq -R '
+ def default(value; f): if . == null then value else f end;
+ def default(value): default(value; .);
+
+ match("^(?:([^@]+)@)?(?:([^:/]+))?(?::([^/]+))?(/.*)?")
+ | {
+ user: .captures[0].string | default(env.origin_user),
+ host: .captures[1].string | default(env.origin_host),
+ port: .captures[2].string | default(22;
+ if test("^[0-9]+$") then fromjson else
+ error(@json "bad target port: \(.)")
+ end),
+ path: .captures[3].string | default("/var/src"),
+ }
+ '
+ )
+
+ echo $target | jq . >&2
+
+ target_host=$(echo $target | jq -r .host)
+ target_path=$(echo $target | jq -r .path)
+ target_port=$(echo $target | jq -r .port)
+ target_user=$(echo $target | jq -r .user)
+
+ if test "$force" = true; then
+ force_target
+ else
+ check_target
+ fi
+
+ jq -c 'to_entries | group_by(.value.type) | flatten[]' |
+ while read -r source; do
+ key=$(echo "$source" | jq -r .key)
+ type=$(echo "$source" | jq -r .value.type)
+ conf=$(echo "$source" | jq -r .value.${type})
+
+ printf '\e[1;33m%s\e[m\n' "populate_$type $key $conf" >&2
+
+ populate_"$type" "$key" "$conf"
+ done
+)}
+
+# Safeguard to prevent clobbering of misspelled targets.
+# This function has to be called first.
+check_target() {
+ {
+ echo target_host=$(quote "$target_host")
+ echo target_path=$(quote "$target_path")
+ echo 'sentinel_file=$target_path/.populate'
+ echo 'if ! test -f "$sentinel_file"; then'
+ echo ' echo "error: missing sentinel file: $target_host:$sentinel_file" >&2'
+ echo ' exit 1'
+ echo 'fi'
+ } \
+ |
+ target_shell
+}
+
+force_target() {
+ {
+ echo target_path=$(quote "$target_path")
+ echo 'sentinel_file=$target_path/.populate'
+ echo 'mkdir -vp "$target_path"'
+ echo 'touch "$sentinel_file"'
+ } \
+ |
+ target_shell
+}
+
+is_local_target() {
+ test "$target_host" = "$origin_host" &&
+ test "$target_user" = "$origin_user"
+}
+
+populate_file() {(
+ file_name=$1
+ file_path=$(echo "$2" | jq -r .path)
+
+ if is_local_target; then
+ file_target=$target_path/$file_name
+ else
+ file_target=$target_user@$target_host:$target_path/$file_name
+ fi
+
+ rsync \
+ -vFrlptD \
+ --delete-excluded \
+ "$file_path"/ \
+ -e "ssh -o ControlPersist=no -p $target_port" \
+ "$file_target"
+)}
+
+populate_git() {(
+ git_name=$1
+ git_url=$(echo "$2" | jq -r .url)
+ git_ref=$(echo "$2" | jq -r .ref)
+
+ git_work_tree=$target_path/$git_name
+
+ {
+ echo set -efu
+
+ echo git_url=$(quote "$git_url")
+ echo git_ref=$(quote "$git_ref")
+
+ echo git_work_tree=$(quote "$git_work_tree")
+
+ echo 'if ! test -e "$git_work_tree"; then'
+ echo ' git clone "$git_url" "$git_work_tree"'
+ echo 'fi'
+
+ echo 'cd $git_work_tree'
+
+ echo 'if ! url=$(git config remote.origin.url); then'
+ echo ' git remote add origin "$git_url"'
+ echo 'elif test "$url" != "$git_url"; then'
+ echo ' git remote set-url origin "$git_url"'
+ echo 'fi'
+
+ # TODO resolve git_ref to commit hash
+ echo 'hash=$git_ref'
+
+ echo 'if ! test "$(git log --format=%H -1)" = "$hash"; then'
+ echo ' if ! git log -1 "$hash" >/dev/null 2>&1; then'
+ echo ' git fetch origin'
+ echo ' fi'
+ echo ' git checkout "$hash" -- "$git_work_tree"'
+ echo ' git -c advice.detachedHead=false checkout -f "$hash"'
+ echo 'fi'
+
+ echo 'git clean -dfx'
+
+ } \
+ |
+ target_shell
+)}
+
+populate_pass() {(
+ pass_target_name=$1
+ pass_dir=$(echo "$2" | jq -r .dir)
+ pass_name_root=$(echo "$2" | jq -r .name)
+
+ if is_local_target; then
+ pass_target=$target_path/$pass_target_name
+ else
+ pass_target=$target_user@$target_host:$target_path/$pass_target_name
+ fi
+
+ umask 0077
+
+ tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX)
+ trap cleanup EXIT
+ cleanup() {
+ rm -fR "$tmp_dir"
+ }
+
+ pass_prefix=$pass_dir/$pass_name_root/
+
+ find "$pass_prefix" -type f |
+ while read -r pass_gpg_file_path; do
+
+ rel_name=${pass_gpg_file_path:${#pass_prefix}}
+ rel_name=${rel_name%.gpg}
+
+ pass_name=$pass_name_root/$rel_name
+ tmp_path=$tmp_dir/$rel_name
+
+ mkdir -p "$(dirname "$tmp_path")"
+ PASSWORD_STORE_DIR=$pass_dir pass show "$pass_name" > "$tmp_path"
+ done
+
+ rsync \
+ --checksum \
+ -vFrlptD \
+ --delete-excluded \
+ "$tmp_dir"/ \
+ -e "ssh -o ControlPersist=no -p $target_port" \
+ "$pass_target"
+)}
+
+populate_pipe() {(
+ pipe_target_name=$1
+ pipe_command=$(echo "$2" | jq -r .command)
+
+ result_path=$target_path/$pipe_target_name
+
+ "$pipe_command" | target_shell -c "cat > $(quote "$result_path")"
+)}
+
+populate_symlink() {(
+ symlink_name=$1
+ symlink_target=$(echo "$2" | jq -r .target)
+ link_name=$target_path/$symlink_name
+
+ {
+ # TODO rm -fR instead of ln -f?
+ echo ln -fns $(quote "$symlink_target" "$link_name")
+ } \
+ |
+ target_shell
+)}
+
+quote() {
+ printf %s "$1" | sed 's/./\\&/g'
+ while test $# -gt 1; do
+ printf ' '
+ shift
+ printf %s "$1" | sed 's/./\\&/g'
+ done
+ echo
+}
+
+target_shell() {
+ if is_local_target; then
+ /bin/sh "$@"
+ else
+ ssh "$target_host" \
+ -l "$target_user" \
+ -o ControlPersist=no \
+ -p "$target_port" \
+ -T \
+ /bin/sh "$@"
+ fi
+}
+
+main "$@"