sh: functions -> bin/

48 files changed, 639 insertions, 620 deletions

diff --git a/bin/_cac_curl_api_v1 b/bin/_cac_curl_api_v1
new file mode 100755
index 000000000..65acebd9a
--- /dev/null
+++ b/bin/_cac_curl_api_v1
@@ -0,0 +1,10 @@
+#! /bin/sh
+set -euf
+
+exec _cac_exec curl -fsS "$1" "https://panel.cloudatcost.com/api/v1/$2.php" $(
+  shift 2
+  set -- "$@" login="$cac_login" key="$cac_key"
+  for arg; do
+    echo -d $(printf '%s' "$arg" | urlencode)
+  done
+)
diff --git a/bin/_cac_exec b/bin/_cac_exec
new file mode 100755
index 000000000..c932454e2
--- /dev/null
+++ b/bin/_cac_exec
@@ -0,0 +1,8 @@
+#! /bin/sh
+set -euf
+
+if test -z "${cac_via-}"; then
+  exec "$@"
+else
+  exec ssh -q "$cac_via" -t "$@"
+fi
diff --git a/bin/_cac_get_api_v1 b/bin/_cac_get_api_v1
new file mode 100755
index 000000000..67aac8560
--- /dev/null
+++ b/bin/_cac_get_api_v1
@@ -0,0 +1,3 @@
+#! /bin/sh
+set -euf
+exec _cac_curl_api_v1 -G "$@"
diff --git a/bin/_cac_post_api_v1 b/bin/_cac_post_api_v1
new file mode 100755
index 000000000..b946ed9fa
--- /dev/null
+++ b/bin/_cac_post_api_v1
@@ -0,0 +1,3 @@
+#! /bin/sh
+set -euf
+exec _cac_curl_api_v1 -XPOST "$@"
diff --git a/bin/bre-escape b/bin/bre-escape
new file mode 100755
index 000000000..ae961b0e6
--- /dev/null
+++ b/bin/bre-escape
@@ -0,0 +1,5 @@
+#! /bin/sh
+# bre-escape : lines string |> lines bre-escaped-string
+set -euf
+
+sed 's:[\.\[\\\*\^\$]:\\&:g'
diff --git a/bin/bre-invert-word b/bin/bre-invert-word
new file mode 100755
index 000000000..677ba2e97
--- /dev/null
+++ b/bin/bre-invert-word
@@ -0,0 +1,15 @@
+#! /bin/sh
+# bre-invert-word : string -> BRE
+set -euf
+
+# TODO escape chars in the resulting BRE.
+awk -v input="$1" '
+  BEGIN {
+    split(input,s,"")
+    for (i in s) {
+      c=s[i]
+      printf "\\|%s[^%s]", y, c
+      y = y c
+    }
+  }
+'
diff --git a/bin/cac-cloudpro-build b/bin/cac-cloudpro-build
new file mode 100755
index 000000000..782fa0d72
--- /dev/null
+++ b/bin/cac-cloudpro-build
@@ -0,0 +1,5 @@
+#! /bin/sh
+set -euf
+
+# default os=26 is CentOS-7-64bit
+exec _cac_post_api_v1 cloudpro/build cpu="$1" ram="$2" storage="$3" os="${4-26}"
diff --git a/bin/cac-cloudpro-delete b/bin/cac-cloudpro-delete
new file mode 100755
index 000000000..ee1dbbc7e
--- /dev/null
+++ b/bin/cac-cloudpro-delete
@@ -0,0 +1,3 @@
+#! /bin/sh
+set -euf
+exec _cac_post_api_v1 cloudpro/delete sid="$1"
diff --git a/bin/cac-cloudpro-resources b/bin/cac-cloudpro-resources
new file mode 100755
index 000000000..9ec5872e7
--- /dev/null
+++ b/bin/cac-cloudpro-resources
@@ -0,0 +1,3 @@
+#! /bin/sh
+set -euf
+exec _cac_get_api_v1 cloudpro/resources
diff --git a/bin/cac-console b/bin/cac-console
new file mode 100755
index 000000000..ed1cbd5ff
--- /dev/null
+++ b/bin/cac-console
@@ -0,0 +1,3 @@
+#! /bin/sh
+set -euf
+exec _cac_post_api_v1 console sid="$1"
diff --git a/bin/cac-get-server-by b/bin/cac-get-server-by
new file mode 100755
index 000000000..b46062518
--- /dev/null
+++ b/bin/cac-get-server-by
@@ -0,0 +1,15 @@
+#! /bin/sh
+set -euf
+
+cac-listservers \
+  | jq \
+    --arg k "$1" \
+    --arg v "$2" \
+    '
+      map(select(.[$k]==$v)) |
+      if (. | length) == 1 then
+        .[0]
+      else
+        .
+      end
+    '
diff --git a/bin/cac-listservers b/bin/cac-listservers
new file mode 100755
index 000000000..1e815d2af
--- /dev/null
+++ b/bin/cac-listservers
@@ -0,0 +1,12 @@
+#! /bin/sh
+set -euf
+
+listservers=$(_cac_get_api_v1 listservers)
+status=$(echo "$listservers" | jq -r .status)
+
+if [ "$status" = ok ]; then
+  echo "$listservers" | jq -r .data
+else
+  echo "$0: bad listservers status: $status" >&2
+  exit 1
+fi
diff --git a/bin/cac-listtasks b/bin/cac-listtasks
new file mode 100755
index 000000000..14be3948a
--- /dev/null
+++ b/bin/cac-listtasks
@@ -0,0 +1,3 @@
+#! /bin/sh
+set -euf
+exec _cac_get_api_v1 listtasks
diff --git a/bin/cac-listtemplates b/bin/cac-listtemplates
new file mode 100755
index 000000000..c4414e019
--- /dev/null
+++ b/bin/cac-listtemplates
@@ -0,0 +1,4 @@
+#! /bin/sh
+set -euf
+
+exec _cac_get_api_v1 listtemplates
diff --git a/bin/cac-powerop b/bin/cac-powerop
new file mode 100755
index 000000000..c897835f0
--- /dev/null
+++ b/bin/cac-powerop
@@ -0,0 +1,3 @@
+#! /bin/sh
+set -euf
+exec _cac_post_api_v1 powerop sid="$1" action="$2"
diff --git a/bin/cac-rdns b/bin/cac-rdns
new file mode 100755
index 000000000..c2d9ecdab
--- /dev/null
+++ b/bin/cac-rdns
@@ -0,0 +1,3 @@
+#! /bin/sh
+set -euf
+exec _cac_post_api_v1 rdns sid="$1" hostname="$2"
diff --git a/bin/cac-renameserver b/bin/cac-renameserver
new file mode 100755
index 000000000..f0eff9b3d
--- /dev/null
+++ b/bin/cac-renameserver
@@ -0,0 +1,3 @@
+#! /bin/sh
+set -euf
+exec _cac_post_api_v1 renameserver sid="$1" name="$2"
diff --git a/bin/cac-runmode b/bin/cac-runmode
new file mode 100755
index 000000000..200b9fb79
--- /dev/null
+++ b/bin/cac-runmode
@@ -0,0 +1,3 @@
+#! /bin/sh
+set -euf
+exec _cac_post_api_v1 rdns sid="$1" mode="$2"
diff --git a/bin/cac-ssh b/bin/cac-ssh
new file mode 100755
index 000000000..a0ec5dcf3
--- /dev/null
+++ b/bin/cac-ssh
@@ -0,0 +1,17 @@
+#! /bin/sh
+set -euf
+
+server=$1
+shift
+
+address=$(echo $server | jq -r .ip)
+target=root@$address
+
+SSHPASS=$(echo $server | jq -r .rootpass)
+export SSHPASS
+
+exec sshpass -e ssh \
+  -o StrictHostKeyChecking=no \
+  -o UserKnownHostsFile=/dev/null \
+  "$target" \
+  "$@"
diff --git a/bin/cacnixos-networking b/bin/cacnixos-networking
new file mode 100755
index 000000000..4b246ebf1
--- /dev/null
+++ b/bin/cacnixos-networking
@@ -0,0 +1,28 @@
+#! /bin/sh
+# cacnixos-networking : cac-server x hostname -> nixos-module
+# TODO use label for hostname
+set -euf
+
+server=$1
+hostname=$2
+
+address=$(echo $server | jq -r .ip)
+gateway=$(echo $server | jq -r .gateway)
+nameserver=8.8.8.8
+netmask=$(echo $server | jq -r .netmask)
+prefix=$(netmask-to-prefix $netmask)
+
+printf '{...}:\n'
+printf '{\n'
+printf '  networking.hostName = "%s";\n' $hostname
+printf '  networking.interfaces.enp2s1.ip4 = [\n'
+printf '    {\n'
+printf '      address = "%s";\n' $address
+printf '      prefixLength = %d;\n' $prefix
+printf '    }\n'
+printf '  ];\n'
+printf '  networking.defaultGateway = "%s";\n' $gateway
+printf '  networking.nameservers = [\n'
+printf '    "%s"\n' $nameserver
+printf '  ];\n'
+printf '}\n'
diff --git a/bin/filter-secrets b/bin/filter-secrets
new file mode 100755
index 000000000..6fcce73c1
--- /dev/null
+++ b/bin/filter-secrets
@@ -0,0 +1,6 @@
+#! /bin/sh
+# filter_secrets : lines string |> lines secrets-file-candidate
+set -euf
+
+# Notice how false positives are possible.
+sed -n 's:^\(.*/\)\?\(secrets/.*\):'"${PWD//:/\\:}"'/\2:p'
diff --git a/bin/import-statements b/bin/import-statements
new file mode 100755
index 000000000..12c887970
--- /dev/null
+++ b/bin/import-statements
@@ -0,0 +1,10 @@
+#! /bin/sh
+# import-statements : lines (path ":" string) |> lines (path ":" relpath)
+set -euf
+sed -n '
+      s@^\([^:]\+:\)\('"$(bre-invert-word import)"'\)*\<import\s\+@\1@
+      t1;d
+  :1; s@^\([^:]\+:\)\(\.*/\S*\)@\1\2\n@
+      t2;d
+  :2; P;D
+'
diff --git a/bin/infest-CentOS-7-64bit b/bin/infest-CentOS-7-64bit
new file mode 100755
index 000000000..a8afea14b
--- /dev/null
+++ b/bin/infest-CentOS-7-64bit
@@ -0,0 +1,150 @@
+#! /bin/sh
+set -euf
+
+server=$1
+hostname=$2
+
+address=$(echo $server | jq -r .ip)
+RSYNC_RSH='sshpass -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
+SSHPASS=$(echo $server | jq -r .rootpass)
+export SSHPASS
+export RSYNC_RSH
+
+main="modules/$hostname/default.nix"
+target="root@$address"
+
+cacnixos-networking "$server" $hostname \
+  > modules/$hostname/networking.nix
+
+echo '(
+  set -xeuf
+  type bzip2 || yum install -y bzip2
+  type rsync || yum install -y rsync
+)' \
+  | sshpass -e ssh \
+      -o StrictHostKeyChecking=no \
+      -o UserKnownHostsFile=/dev/null \
+      "root@$address" \
+      /bin/sh
+
+make-rsync-filter "$main" \
+  | rsync -f '. -' -zvrlptD --delete-excluded ./ "$target":/etc/nixos/
+
+#
+#
+#
+echo '(
+  set -xeuf
+  groupadd -g 30000 nixbld || :
+  for i in `seq 1 10`; do
+    useradd -c "foolsgarden Nix build user $i" \
+            -d /var/empty \
+            -s /sbin/nologin \
+            -g 30000 \
+            -G 30000 \
+            -l -u $(expr 30000 + $i) \
+            nixbld$i || :
+    rm -f /var/spool/mail/nixbld$i
+  done
+
+  #curl https://nixos.org/nix/install | sh
+  nix_tar=$nix_basename.tar.bz2
+  if ! echo $nix_sha256 $nix_tar | sha256sum -c; then
+    curl -O -C - $nix_url || :
+    if ! echo $nix_sha256 $nix_tar | sha256sum -c; then
+      curl -O $nix_url || :
+      if ! echo $nix_sha256 $nix_tar | sha256sum -c; then
+        echo $0: cannot download $nix_url >&2
+        exit 5
+      fi
+    fi
+  fi
+
+  if ! test -d $nix_basename; then
+    tar jxf $nix_basename.tar.bz2
+  fi
+
+  nix_find=$nix_basename.find.txt
+  if ! echo $nix_find_sha1sum $nix_find | sha1sum -c; then
+    find $nix_basename | sort > $nix_find
+    if ! echo $nix_find_sha1sum $nix_find | sha1sum -c; then
+      echo $0: cannot unpack $nix_basename.tar.bz2 >&2
+      # TODO we could retry
+      exit 6
+    fi
+  fi
+
+  mkdir -p bin
+  PATH=$HOME/bin:$PATH
+  export PATH
+
+  # generate fake sudo because
+  # sudo: sorry, you must have a tty to run sudo
+  {
+    echo "#! /bin/sh"
+    echo "exec env \"\$@\""
+  } > bin/sudo
+  chmod +x bin/sudo
+
+  ./$nix_basename/install
+
+  . /root/.nix-profile/etc/profile.d/nix.sh
+
+  nixpkgs_expr="import <nixpkgs> { system = builtins.currentSystem; }"
+  nixpkgs_path=$(
+    find /nix/store -mindepth 1 -maxdepth 1 -name *-nixpkgs-* -type d
+  )
+
+  for i in nixos-generate-config nixos-install; do
+    nix-env \
+      --arg config "{ nix.package = ($nixpkgs_expr).nix; }" \
+      --arg pkgs "$nixpkgs_expr" \
+      --arg modulesPath "throw \"no modulesPath\"" \
+      -f $nixpkgs_path/nixpkgs/nixos/modules/installer/tools/tools.nix \
+      -iA config.system.build.$i
+  done
+
+  # TODO following fail when aborted in-between
+  if ! test -d /int; then
+    mkdir -p /int
+    mount --bind /int /mnt
+  fi
+  if ! test -d /mnt/boot; then
+    mkdir -p /mnt/boot
+    mount /dev/sda1 /mnt/boot
+  fi
+
+  mkdir -p /mnt/etc/nixos
+  rsync -zvrlptD --delete-excluded /etc/nixos/ /mnt/etc/nixos/
+
+  mkdir -m 0444 -p /mnt/var/empty
+
+  ln -s $main /mnt/etc/nixos/configuration.nix
+  nixos-install \
+    -I secrets=/etc/nixos/secrets
+
+  find / \
+    1> /root/pre-rsync-find.out \
+    2> /root/pre-rsync-find.err
+
+  rsync -va --force /int/ /
+
+  # find / -type f -mtime +1 -exec rm -v {} \; 2>&1 > rm.log
+  #   ^ too aggressive, kills journal which is bad
+  # shutdown -r now
+  # nix-channel --add https://nixos.org/channels/nixos-unstable nixos
+  # nix-channel --remove nixpkgs
+  # nix-channel --update
+
+)' \
+  | sshpass -e ssh \
+      -o StrictHostKeyChecking=no \
+      -o UserKnownHostsFile=/dev/null \
+      "root@$address" \
+      -T /usr/bin/env \
+        nix_url="$nix_url" \
+        nix_basename="$(basename $nix_url .tar.bz2)" \
+        nix_sha256="$nix_sha256" \
+        nix_find_sha1sum="$nix_find_sha1sum" \
+        main="$main" \
+        /bin/sh
diff --git a/bin/infest-cac b/bin/infest-cac
new file mode 100755
index 000000000..d7d7bb96c
--- /dev/null
+++ b/bin/infest-cac
@@ -0,0 +1,21 @@
+#! /bin/sh
+set -euf
+
+server=$(cac-get-server-by servername "$1")
+hostname=$2
+
+serverstatus=$(echo $server | jq -r .status)
+case $serverstatus in
+  'Powered On') : ;;
+  *)
+    echo $0: bad server status: $serverstatus >&2
+    exit 2
+esac
+
+template=$(echo $server | jq -r .template)
+case $template in
+  'CentOS-7-64bit') infest-"$template" "$server" "$hostname";;
+  *)
+    echo $0: bad template: $template >&2
+    exit 3
+esac
diff --git a/bin/json-assert-type b/bin/json-assert-type
new file mode 100755
index 000000000..29cadad65
--- /dev/null
+++ b/bin/json-assert-type
@@ -0,0 +1,18 @@
+#! /bin/sh
+set -euf
+
+formal_type=$1
+
+actual_value=$2
+actual_type=$(echo $actual_value | jq -r type)
+
+if [ "$actual_type" != "$formal_type" ]; then
+  backtrace
+  printf 'error: expected %s, got %s\n' \
+    "$formal_type" \
+    "$actual_type" \
+    >&2
+  exit 1
+fi
+
+echo "$actual_value"
diff --git a/bin/list-hosts b/bin/list-hosts
new file mode 100755
index 000000000..e25a8ac4f
--- /dev/null
+++ b/bin/list-hosts
@@ -0,0 +1,7 @@
+#! /bin/sh
+# list-hosts : lines tinc-host-file
+set -euf
+
+# Precondition: $PWD/hosts is the correct repository :)
+git -C hosts ls-tree --name-only HEAD \
+  | awk '{print ENVIRON["PWD"]"/hosts/"$$0}'
diff --git a/bin/list-module-imports b/bin/list-module-imports
new file mode 100755
index 000000000..39d11bf34
--- /dev/null
+++ b/bin/list-module-imports
@@ -0,0 +1,20 @@
+#! /bin/sh
+# list-module-imports : nix-file -> lines nix-file
+set -euf
+
+if echo "$1" | grep -q ^/; then
+  :
+else
+  set -- "./$1"
+fi
+
+imports=$(nix-instantiate \
+    -I secrets=secrets \
+    --strict \
+    --json \
+    --eval \
+    -E \
+    "with builtins; with import ./lib/modules.nix; map toString (list-imports $1)")
+
+echo "$imports" \
+  | jq -r .[]
diff --git a/bin/ls-bre b/bin/ls-bre
new file mode 100755
index 000000000..ae978895c
--- /dev/null
+++ b/bin/ls-bre
@@ -0,0 +1,12 @@
+#! /bin/sh
+# ls-bre : directory -> BRE
+# Create a BRE from the files in a directory.
+set -euf
+
+ls "$1" \
+  | tr \\n / \
+  | sed '
+      s:[\.\[\\\*\^\$]:\\&:g
+      s:/$::
+      s:/:\\|:g
+    '
diff --git a/bin/make-parent-dirs b/bin/make-parent-dirs
new file mode 100755
index 000000000..f4717b249
--- /dev/null
+++ b/bin/make-parent-dirs
@@ -0,0 +1,10 @@
+#! /bin/sh
+# make-parent-dirs : lines path |> lines directory
+# List all parent directories of a path.
+set -euf
+
+set -- "$(sed -n 's|/[^/]*$||p' | grep . | sort | uniq)"
+if echo "$1" | grep -q .; then
+  echo "$1"
+  echo "$1" | make-parent-dirs
+fi
diff --git a/bin/make-relative-to b/bin/make-relative-to
new file mode 100755
index 000000000..9d947e175
--- /dev/null
+++ b/bin/make-relative-to
@@ -0,0 +1,6 @@
+#! /bin/sh
+# make-relative-to : lines path |> directory -> lines path
+# Non-matching paths won't get altered.
+set -euf
+
+sed "s:^$(echo "$1/" | bre-escape | sed 's/:/\\:/g')::"
diff --git a/bin/make-rsync-filter b/bin/make-rsync-filter
new file mode 100755
index 000000000..26e070adb
--- /dev/null
+++ b/bin/make-rsync-filter
@@ -0,0 +1,33 @@
+#! /bin/sh
+# make-rsync-filter : nixos-config -> rsync-filter
+set -euf
+
+main=$1
+
+hosts=$(list-hosts)
+module_imports=$(list-module-imports "$main")
+other_imports=$(
+  echo "$module_imports" \
+    | xargs grep -H . \
+    | import-statements \
+    | slash-path-relpath \
+    | undot-paths \
+    | sort \
+    | uniq \
+    | sed '/\.nix$/!s:$:/default.nix:' \
+    )
+secrets=$(echo "$module_imports" | xargs cat | quoted-strings | filter-secrets)
+
+# TODO collect all other paths from *_imports
+
+abs_deps=$(
+  echo "$hosts"
+  echo "$module_imports"
+  echo "$other_imports"
+  echo "$secrets"
+)
+
+rel_deps=$(echo "$abs_deps" | make-relative-to "$PWD")
+filter=$(echo "$rel_deps" | make-rsync-whitelist)
+
+echo "$filter"
diff --git a/bin/make-rsync-whitelist b/bin/make-rsync-whitelist
new file mode 100755
index 000000000..a1b09c801
--- /dev/null
+++ b/bin/make-rsync-whitelist
@@ -0,0 +1,15 @@
+#! /bin/sh
+# make-rsync-whitelist : lines relpath |> liens rsync-filter
+set -euf
+
+set -- "$(cat)"
+
+# include all files in stdin and their directories
+{
+  echo "$1"
+  echo "$1" | make-parent-dirs | sort | uniq
+} \
+  | sed 's|^|+ /|'
+
+# exclude everything else
+echo '- *'
diff --git a/bin/netmask-to-prefix b/bin/netmask-to-prefix
new file mode 100755
index 000000000..1c4dbeb28
--- /dev/null
+++ b/bin/netmask-to-prefix
@@ -0,0 +1,12 @@
+#! /bin/sh
+set -euf
+
+netmask=$1
+
+binaryNetmask=$(echo $1 | sed 's/^/obase=2;/;s/\./;/g' | bc | tr -d \\n)
+binaryPrefix=$(echo $binaryNetmask | sed -n 's/^\(1*\)0*$/\1/p')
+if ! echo $binaryPrefix | grep -q .; then
+  echo $0: bad netmask: $netmask >&2
+  exit 4
+fi
+printf %s $binaryPrefix | tr -d 0 | wc -c
diff --git a/bin/nixpkgs-rev b/bin/nixpkgs-rev
new file mode 100755
index 000000000..1acde1e4e
--- /dev/null
+++ b/bin/nixpkgs-rev
@@ -0,0 +1,13 @@
+#! /bin/sh
+# nixpkgs-rev : nixos-config -> git_rev
+set -euf
+nix-instantiate \
+    -I nixos-config="$1" \
+    --eval \
+    --json \
+    -E \
+    '
+      (import <nixos-config> {config={}; pkgs={};}).nixpkgs.rev
+    ' \
+    2> /dev/null \
+  | jq -r . 2> /dev/null
diff --git a/bin/nixpkgs-url b/bin/nixpkgs-url
new file mode 100755
index 000000000..9549f0c77
--- /dev/null
+++ b/bin/nixpkgs-url
@@ -0,0 +1,13 @@
+#! /bin/sh
+# nixpkgs-url : nixos-config -> git_url
+set -euf
+nix-instantiate \
+    -I nixos-config="$1" \
+    --eval \
+    --json \
+    -E \
+    '
+      (import <nixos-config> {config={}; pkgs={};}).nixpkgs.url
+    ' \
+    2> /dev/null \
+  | jq -r . 2> /dev/null
diff --git a/bin/quoted-strings b/bin/quoted-strings
new file mode 100755
index 000000000..e64039101
--- /dev/null
+++ b/bin/quoted-strings
@@ -0,0 +1,15 @@
+#! /bin/sh
+# quoted_strings : lines string |> lines string
+# Extract all (double-) quoted strings from stdin.
+#
+# 0. find begin of string or skip line
+# 1. find end of string or skip line
+# 2. print string and continue after string
+set -euf
+
+sed '
+      s:[^"]*"::                  ;t1;d
+  :1; s:\(\([^"]\|\\"\)*\)":\1\n: ;t2;d
+  :2; P;D
+' \
+  | sed 's:\\":":g'
diff --git a/bin/slash-path-relpath b/bin/slash-path-relpath
new file mode 100755
index 000000000..40230a70c
--- /dev/null
+++ b/bin/slash-path-relpath
@@ -0,0 +1,8 @@
+#! /bin/sh
+# slash_path_relpath : lines (path ":" relpath) |> lines path
+#
+# Example: "/foo/bar: baz" => "/foo/baz"
+#
+set -euf
+
+sed -n 's@/[^/]\+:@/@p'
diff --git a/bin/ssh-deploy b/bin/ssh-deploy
new file mode 100755
index 000000000..fe50677df
--- /dev/null
+++ b/bin/ssh-deploy
@@ -0,0 +1,26 @@
+#! /bin/sh
+# ssh-deploy : nixos-config x [user@]