deploy: merge next

30 files changed, 51 insertions, 338 deletions

diff --git a/.gitignore b/.gitignore
index a37850ab7..bad1d00ee 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /.graveyard
 /hosts
 /secrets
+/tmp
diff --git a/bin/backtrace b/bin/backtrace
deleted file mode 100755
index 15d60a9b4..000000000
--- a/bin/backtrace
+++ /dev/null
@@ -1,32 +0,0 @@
-#! /bin/sh
-set -euf
-
-exec >&2
-
-pid=$$
-while :; do
-  cd /proc/$pid
-  cmdline=$(cat cmdline | tr '\0' ' ' | sed 's/ $//')
-  ppid=$(grep PPid status | awk '{print$2}')
-  printf '%5d %s\n' $pid "$cmdline"
-  if [ $pid = 1 ]; then
-    break
-  else
-    pid=$ppid
-  fi
-done \
-  | sed \
-      "$(printf '
-          s:\(/bin/sh \)%s/\([^ ]\+\):\e[32m\\2\e[m:g
-          s:%s/\([^ ]\+\):\e[32;1m\\1\e[m:g
-        ' \
-          "$(dirname "$0" | bre-escape | sed 's/:/\\:/g')" \
-          "$(dirname "$0" | bre-escape | sed 's/:/\\:/g')" \
-      )" \
-  | tac
-
-ps='Press ^C to abort: '
-while :; do
-  printf '\e[K\e[31;1m%s\e[m' "$ps"
-  read -r _input
-done
diff --git a/bin/bre-escape b/bin/bre-escape
deleted file mode 100755
index ae961b0e6..000000000
--- a/bin/bre-escape
+++ /dev/null
@@ -1,5 +0,0 @@
-#! /bin/sh
-# bre-escape : lines string |> lines bre-escaped-string
-set -euf
-
-sed 's:[\.\[\\\*\^\$]:\\&:g'
diff --git a/bin/bre-invert-word b/bin/bre-invert-word
deleted file mode 100755
index 677ba2e97..000000000
--- a/bin/bre-invert-word
+++ /dev/null
@@ -1,15 +0,0 @@
-#! /bin/sh
-# bre-invert-word : string -> BRE
-set -euf
-
-# TODO escape chars in the resulting BRE.
-awk -v input="$1" '
-  BEGIN {
-    split(input,s,"")
-    for (i in s) {
-      c=s[i]
-      printf "\\|%s[^%s]", y, c
-      y = y c
-    }
-  }
-'
diff --git a/bin/cac-ssh b/bin/cac-ssh
index e01f76b10..a0ec5dcf3 100755
--- a/bin/cac-ssh
+++ b/bin/cac-ssh
@@ -1,7 +1,7 @@
 #! /bin/sh
 set -euf
 
-server=$(json-assert-type object "$1")
+server=$1
 shift
 
 address=$(echo $server | jq -r .ip)
diff --git a/next/bin/copy-secrets b/bin/copy-secrets
index 7398d4fb1..36854eaf1 100755
--- a/next/bin/copy-secrets
+++ b/bin/copy-secrets
@@ -13,11 +13,11 @@ if ! test -e "$secrets_rsync"; then
   exit # nothing to do
 fi
 
-retiolum_secret=$(query $host services.retiolum.privateKeyFile)
-retiolum_uid=$(query $host users.extraUsers.retiolum-tinc.uid)
+retiolum_secret=$(nixos-query $host services.retiolum.privateKeyFile)
+retiolum_uid=$(nixos-query $host users.extraUsers.retiolum-tinc.uid)
 
 ejabberd_secret=/etc/ejabberd/ejabberd.pem
-ejabberd_uid=$(query $host users.extraUsers.ejabberd.uid)
+ejabberd_uid=$(nixos-query $host users.extraUsers.ejabberd.uid)
 
 rsync -cz --chown=0:0 -vr "$secrets_rsync/" "$target:/"
 
diff --git a/bin/filter-secrets b/bin/filter-secrets
deleted file mode 100755
index 6fcce73c1..000000000
--- a/bin/filter-secrets
+++ /dev/null
@@ -1,6 +0,0 @@
-#! /bin/sh
-# filter_secrets : lines string |> lines secrets-file-candidate
-set -euf
-
-# Notice how false positives are possible.
-sed -n 's:^\(.*/\)\?\(secrets/.*\):'"${PWD//:/\\:}"'/\2:p'
diff --git a/bin/import-statements b/bin/import-statements
deleted file mode 100755
index 12c887970..000000000
--- a/bin/import-statements
+++ /dev/null
@@ -1,10 +0,0 @@
-#! /bin/sh
-# import-statements : lines (path ":" string) |> lines (path ":" relpath)
-set -euf
-sed -n '
-      s@^\([^:]\+:\)\('"$(bre-invert-word import)"'\)*\<import\s\+@\1@
-      t1;d
-  :1; s@^\([^:]\+:\)\(\.*/\S*\)@\1\2\n@
-      t2;d
-  :2; P;D
-'
diff --git a/bin/json-assert-type b/bin/json-assert-type
deleted file mode 100755
index 29cadad65..000000000
--- a/bin/json-assert-type
+++ /dev/null
@@ -1,18 +0,0 @@
-#! /bin/sh
-set -euf
-
-formal_type=$1
-
-actual_value=$2
-actual_type=$(echo $actual_value | jq -r type)
-
-if [ "$actual_type" != "$formal_type" ]; then
-  backtrace
-  printf 'error: expected %s, got %s\n' \
-    "$formal_type" \
-    "$actual_type" \
-    >&2
-  exit 1
-fi
-
-echo "$actual_value"
diff --git a/bin/list-hosts b/bin/list-hosts
deleted file mode 100755
index e25a8ac4f..000000000
--- a/bin/list-hosts
+++ /dev/null
@@ -1,7 +0,0 @@
-#! /bin/sh
-# list-hosts : lines tinc-host-file
-set -euf
-
-# Precondition: $PWD/hosts is the correct repository :)
-git -C hosts ls-tree --name-only HEAD \
-  | awk '{print ENVIRON["PWD"]"/hosts/"$$0}'
diff --git a/bin/list-module-imports b/bin/list-module-imports
deleted file mode 100755
index 39d11bf34..000000000
--- a/bin/list-module-imports
+++ /dev/null
@@ -1,20 +0,0 @@
-#! /bin/sh
-# list-module-imports : nix-file -> lines nix-file
-set -euf
-
-if echo "$1" | grep -q ^/; then
-  :
-else
-  set -- "./$1"
-fi
-
-imports=$(nix-instantiate \
-    -I secrets=secrets \
-    --strict \
-    --json \
-    --eval \
-    -E \
-    "with builtins; with import ./lib/modules.nix; map toString (list-imports $1)")
-
-echo "$imports" \
-  | jq -r .[]
diff --git a/bin/ls-bre b/bin/ls-bre
deleted file mode 100755
index ae978895c..000000000
--- a/bin/ls-bre
+++ /dev/null
@@ -1,12 +0,0 @@
-#! /bin/sh
-# ls-bre : directory -> BRE
-# Create a BRE from the files in a directory.
-set -euf
-
-ls "$1" \
-  | tr \\n / \
-  | sed '
-      s:[\.\[\\\*\^\$]:\\&:g
-      s:/$::
-      s:/:\\|:g
-    '
diff --git a/bin/make-parent-dirs b/bin/make-parent-dirs
deleted file mode 100755
index f4717b249..000000000
--- a/bin/make-parent-dirs
+++ /dev/null
@@ -1,10 +0,0 @@
-#! /bin/sh
-# make-parent-dirs : lines path |> lines directory
-# List all parent directories of a path.
-set -euf
-
-set -- "$(sed -n 's|/[^/]*$||p' | grep . | sort | uniq)"
-if echo "$1" | grep -q .; then
-  echo "$1"
-  echo "$1" | make-parent-dirs
-fi
diff --git a/bin/make-relative-to b/bin/make-relative-to
deleted file mode 100755
index 9d947e175..000000000
--- a/bin/make-relative-to
+++ /dev/null
@@ -1,6 +0,0 @@
-#! /bin/sh
-# make-relative-to : lines path |> directory -> lines path
-# Non-matching paths won't get altered.
-set -euf
-
-sed "s:^$(echo "$1/" | bre-escape | sed 's/:/\\:/g')::"
diff --git a/bin/make-rsync-filter b/bin/make-rsync-filter
deleted file mode 100755
index 26e070adb..000000000
--- a/bin/make-rsync-filter
+++ /dev/null
@@ -1,33 +0,0 @@
-#! /bin/sh
-# make-rsync-filter : nixos-config -> rsync-filter
-set -euf
-
-main=$1
-
-hosts=$(list-hosts)
-module_imports=$(list-module-imports "$main")
-other_imports=$(
-  echo "$module_imports" \
-    | xargs grep -H . \
-    | import-statements \
-    | slash-path-relpath \
-    | undot-paths \
-    | sort \
-    | uniq \
-    | sed '/\.nix$/!s:$:/default.nix:' \
-    )
-secrets=$(echo "$module_imports" | xargs cat | quoted-strings | filter-secrets)
-
-# TODO collect all other paths from *_imports
-
-abs_deps=$(
-  echo "$hosts"
-  echo "$module_imports"
-  echo "$other_imports"
-  echo "$secrets"
-)
-
-rel_deps=$(echo "$abs_deps" | make-relative-to "$PWD")
-filter=$(echo "$rel_deps" | make-rsync-whitelist)
-
-echo "$filter"
diff --git a/bin/make-rsync-whitelist b/bin/make-rsync-whitelist
deleted file mode 100755
index a1b09c801..000000000
--- a/bin/make-rsync-whitelist
+++ /dev/null
@@ -1,15 +0,0 @@
-#! /bin/sh
-# make-rsync-whitelist : lines relpath |> liens rsync-filter
-set -euf
-
-set -- "$(cat)"
-
-# include all files in stdin and their directories
-{
-  echo "$1"
-  echo "$1" | make-parent-dirs | sort | uniq
-} \
-  | sed 's|^|+ /|'
-
-# exclude everything else
-echo '- *'
diff --git a/next/bin/build b/bin/nixos-build
index 40bbec344..a0c9551fa 100755
--- a/next/bin/build
+++ b/bin/nixos-build
@@ -8,10 +8,13 @@ host=$1
 
 #target=root@$host
 
+nixpkgs=$nixpkgs_root/$host
 nixos_config=$config_root/modules/$host
 secrets_nix=$secrets_root/$host/nix
 secrets_rsync=$secrets_root/$host/rsync
 
+nixos-fetch-git "$host"
+
 nix-build \
     -I "$nixpkgs" \
     -I nixos-config="$nixos_config" \
diff --git a/next/bin/deploy b/bin/nixos-deploy
index 1c1d977da..6b8418696 100755
--- a/next/bin/deploy
+++ b/bin/nixos-deploy
@@ -5,7 +5,7 @@
 set -euf
 
 host=$1
-system=${2-$(build "$host")}
+system=${2-$(nixos-build "$host")}
 
 target=root@$host
 
diff --git a/bin/nixos-fetch-git b/bin/nixos-fetch-git
new file mode 100755
index 000000000..7002208b5
--- /dev/null
+++ b/bin/nixos-fetch-git
@@ -0,0 +1,32 @@
+#! /bin/sh
+#
+# nixos-fetch-git : nixos-config -> ...
+#
+set -euf
+
+host=$1
+
+target=root@$host
+
+git_rev=$(nixos-query "$host" nixpkgs.rev)
+git_url=$(nixos-query "$host" nixpkgs.url)
+
+worktree=$nixpkgs_root/$host
+
+if [ ! -d "$worktree" ]; then
+  mkdir -p "$worktree"
+fi
+
+cd "$worktree"
+
+git init -q
+
+if ! current_url=$(git config remote.src.url); then
+  git remote add src "$git_url"
+elif [ "$current_url" != "$git_url" ]; then
+  git remote set-url src "$git_url"
+fi
+
+git fetch src
+
+git checkout "$git_rev"
diff --git a/next/bin/query b/bin/nixos-query
index 0e55a6cec..65b5c9672 100755
--- a/next/bin/query
+++ b/bin/nixos-query
@@ -4,6 +4,7 @@ set -euf
 host=$1
 attr=$2
 
+nixpkgs=$nixpkgs_root/$host
 nixos_config=$config_root/modules/$host
 secrets_nix=$secrets_root/$host/nix
 secrets_rsync=$secrets_root/$host/rsync
diff --git a/bin/nixpkgs-rev b/bin/nixpkgs-rev
deleted file mode 100755
index 1acde1e4e..000000000
--- a/bin/nixpkgs-rev
+++ /dev/null
@@ -1,13 +0,0 @@
-#! /bin/sh
-# nixpkgs-rev : nixos-config -> git_rev
-set -euf
-nix-instantiate \
-    -I nixos-config="$1" \
-    --eval \
-    --json \
-    -E \
-    '
-      (import <nixos-config> {config={}; pkgs={};}).nixpkgs.rev
-    ' \
-    2> /dev/null \
-  | jq -r . 2> /dev/null
diff --git a/bin/nixpkgs-url b/bin/nixpkgs-url
deleted file mode 100755
index 9549f0c77..000000000
--- a/bin/nixpkgs-url
+++ /dev/null
@@ -1,13 +0,0 @@
-#! /bin/sh
-# nixpkgs-url : nixos-config -> git_url
-set -euf
-nix-instantiate \
-    -I nixos-config="$1" \
-    --eval \
-    --json \
-    -E \
-    '
-      (import <nixos-config> {config={}; pkgs={};}).nixpkgs.url
-    ' \
-    2> /dev/null \
-  | jq -r . 2> /dev/null
diff --git a/bin/quoted-strings b/bin/quoted-strings
deleted file mode 100755
index e64039101..000000000
--- a/bin/quoted-strings
+++ /dev/null
@@ -1,15 +0,0 @@
-#! /bin/sh
-# quoted_strings : lines string |> lines string
-# Extract all (double-) quoted strings from stdin.
-#
-# 0. find begin of string or skip line
-# 1. find end of string or skip line
-# 2. print string and continue after string
-set -euf
-
-sed '
-      s:[^"]*"::                  ;t1;d
-  :1; s:\(\([^"]\|\\"\)*\)":\1\n: ;t2;d
-  :2; P;D
-' \
-  | sed 's:\\":":g'
diff --git a/bin/slash-path-relpath b/bin/slash-path-relpath
deleted file mode 100755
index 40230a70c..000000000
--- a/bin/slash-path-relpath
+++ /dev/null
@@ -1,8 +0,0 @@
-#! /bin/sh
-# slash_path_relpath : lines (path ":" relpath) |> lines path
-#
-# Example: "/foo/bar: baz" => "/foo/baz"
-#
-set -euf
-
-sed -n 's@/[^/]\+:@/@p'
diff --git a/bin/ssh-deploy b/bin/ssh-deploy
deleted file mode 100755
index fe50677df..000000000
--- a/bin/ssh-deploy
+++ /dev/null
@@ -1,26 +0,0 @@
-#! /bin/sh
-# ssh-deploy : nixos-config x [user@]hostname -> ()
-set -xeuf
-
-main=$1
-target=$2
-nixpkgs_dir=/var/nixpkgs # TODO make configurable
-
-git_url=$(nixpkgs-url $main)
-git_rev=$(nixpkgs-rev $main)
-
-if [ "$git_url" = '' ] || [ "$git_rev" = '' ]; then
-  echo "specify nixpkgs.url and nixpkgs.rev in $main !"
-  exit 23
-fi
-
-filter=$(make-rsync-filter "$main")
-
-echo "$filter" \
-  | rsync -f '. -' -zvrlptD --delete-excluded ./ "$target":/etc/nixos/
-
-ssh-fetch-git "$target" "$nixpkgs_dir" "$git_url" "$git_rev"
-ssh "$target" nixos-rebuild switch \
-  -I nixos-config=/etc/nixos/"$main" \
-  -I nixpkgs="$nixpkgs_dir" \
-  -I secrets=/etc/nixos/secrets \
diff --git a/bin/ssh-fetch-git b/bin/ssh-fetch-git
deleted file mode 100755
index 7de58ab73..000000000
--- a/bin/ssh-fetch-git
+++ /dev/null
@@ -1,35 +0,0 @@
-#! /bin/sh
-# ssh-fetch-git : [user@]hostname x remote_dir x git_url x git_rev -> ()
-set -euf
-
-target=$1
-remote_dir=$2
-git_url=$3
-git_rev=$4
-
-echo '
-  set -euf
-
-  if [ ! -d "$remote_dir" ]; then
-    mkdir -p "$remote_dir"
-  fi
-
-  cd "$remote_dir"
-
-  git init -q
-
-  if ! current_url=$(git config remote.src.url); then
-    git remote add src "$git_url"
-  elif [ $current_url != $git_url ]; then
-    git remote set-url src "$git_url"
-  fi
-
-  git fetch src
-
-  git checkout "$git_rev"
-' \
-  | ssh "$target" env \
-        remote_dir="$remote_dir" \
-        git_rev="$git_rev" \
-        git_url="$git_url" \
-      /bin/sh
diff --git a/bin/undot-paths b/bin/undot-paths
deleted file mode 100755
index 2ed86bdec..000000000
--- a/bin/undot-paths
+++ /dev/null
@@ -1,14 +0,0 @@
-#! /bin/sh
-# undot_paths : lines path |> lines path
-# Remove all dots (. and ..) from input paths.
-set -euf
-
-sed '
-  :0
-  s://\+:/:g
-  s:/\.\(/\|$\):\1:g
-  s:/[^/]\+/\.\.\(/\|$\):\1:g
-  s:^/\(\.\./\)\+:/:
-  t0
-  s:^$:/:
-'
diff --git a/deploy b/deploy
index 43021d545..6034196e1 100755
--- a/deploy
+++ b/deploy
@@ -1,16 +1,16 @@
 #! /bin/sh
 #
-# usage: ./deploy HOST [[USER@]HOST]
+# usage: ./deploy HOST
 #
 set -euf
 
-PATH="$PWD/bin${PATH+:$PATH}"
-export PATH
-
-user=root
 host=$1
 
-config=./modules/$host/default.nix
-target=${2-$user@$host}
+export PATH="$PWD/bin:$PATH"
+#export nixpkgs=/var/nixpkgs
+export nixpkgs_root=$PWD/tmp/nixpkgs
+export config_root=$PWD
+export retiolum_hosts=$PWD/hosts
+export secrets_root=$PWD/secrets
 
-exec ssh-deploy "$config" "$target"
+exec nixos-deploy "$host"
diff --git a/modules/wu/default.nix b/modules/wu/default.nix
index 3258779ee..84a8361af 100644
--- a/modules/wu/default.nix
+++ b/modules/wu/default.nix
@@ -322,7 +322,7 @@ in
 
   services.retiolum = {
     enable = true;
-    hosts = /etc/nixos/hosts;
+    hosts = <retiolum-hosts>;
     connectTo = [
       "gum"
       "pigstarter"
diff --git a/next/profile b/next/profile
deleted file mode 100755
index 138c271f7..000000000
--- a/next/profile
+++ /dev/null
@@ -1,11 +0,0 @@
-#! /bin/sh
-export PATH=$HOME/src/config/next/bin:$PATH
-export nixpkgs=/var/nixpkgs
-export config_root=$HOME/src/config
-export retiolum_hosts=$HOME/src/config/hosts
-export secrets_root=$HOME/src/config/secrets
-
-export PS1='\[\e[1;35m\]config>\[\e[m\] '
-
-cd /var/empty
-exec /bin/sh