diff options
48 files changed, 639 insertions, 620 deletions
diff --git a/bin/_cac_curl_api_v1 b/bin/_cac_curl_api_v1 new file mode 100755 index 000000000..65acebd9a --- /dev/null +++ b/bin/_cac_curl_api_v1 @@ -0,0 +1,10 @@ +#! /bin/sh +set -euf + +exec _cac_exec curl -fsS "$1" "https://panel.cloudatcost.com/api/v1/$2.php" $( + shift 2 + set -- "$@" login="$cac_login" key="$cac_key" + for arg; do + echo -d $(printf '%s' "$arg" | urlencode) + done +) diff --git a/bin/_cac_exec b/bin/_cac_exec new file mode 100755 index 000000000..c932454e2 --- /dev/null +++ b/bin/_cac_exec @@ -0,0 +1,8 @@ +#! /bin/sh +set -euf + +if test -z "${cac_via-}"; then + exec "$@" +else + exec ssh -q "$cac_via" -t "$@" +fi diff --git a/bin/_cac_get_api_v1 b/bin/_cac_get_api_v1 new file mode 100755 index 000000000..67aac8560 --- /dev/null +++ b/bin/_cac_get_api_v1 @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_curl_api_v1 -G "$@" diff --git a/bin/_cac_post_api_v1 b/bin/_cac_post_api_v1 new file mode 100755 index 000000000..b946ed9fa --- /dev/null +++ b/bin/_cac_post_api_v1 @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_curl_api_v1 -XPOST "$@" diff --git a/bin/bre-escape b/bin/bre-escape new file mode 100755 index 000000000..ae961b0e6 --- /dev/null +++ b/bin/bre-escape @@ -0,0 +1,5 @@ +#! /bin/sh +# bre-escape : lines string |> lines bre-escaped-string +set -euf + +sed 's:[\.\[\\\*\^\$]:\\&:g' diff --git a/bin/bre-invert-word b/bin/bre-invert-word new file mode 100755 index 000000000..677ba2e97 --- /dev/null +++ b/bin/bre-invert-word @@ -0,0 +1,15 @@ +#! /bin/sh +# bre-invert-word : string -> BRE +set -euf + +# TODO escape chars in the resulting BRE. +awk -v input="$1" ' + BEGIN { + split(input,s,"") + for (i in s) { + c=s[i] + printf "\\|%s[^%s]", y, c + y = y c + } + } +' diff --git a/bin/cac-cloudpro-build b/bin/cac-cloudpro-build new file mode 100755 index 000000000..782fa0d72 --- /dev/null +++ b/bin/cac-cloudpro-build @@ -0,0 +1,5 @@ +#! /bin/sh +set -euf + +# default os=26 is CentOS-7-64bit +exec _cac_post_api_v1 cloudpro/build cpu="$1" ram="$2" storage="$3" os="${4-26}" diff --git a/bin/cac-cloudpro-delete b/bin/cac-cloudpro-delete new file mode 100755 index 000000000..ee1dbbc7e --- /dev/null +++ b/bin/cac-cloudpro-delete @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 cloudpro/delete sid="$1" diff --git a/bin/cac-cloudpro-resources b/bin/cac-cloudpro-resources new file mode 100755 index 000000000..9ec5872e7 --- /dev/null +++ b/bin/cac-cloudpro-resources @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_get_api_v1 cloudpro/resources diff --git a/bin/cac-console b/bin/cac-console new file mode 100755 index 000000000..ed1cbd5ff --- /dev/null +++ b/bin/cac-console @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 console sid="$1" diff --git a/bin/cac-get-server-by b/bin/cac-get-server-by new file mode 100755 index 000000000..b46062518 --- /dev/null +++ b/bin/cac-get-server-by @@ -0,0 +1,15 @@ +#! /bin/sh +set -euf + +cac-listservers \ + | jq \ + --arg k "$1" \ + --arg v "$2" \ + ' + map(select(.[$k]==$v)) | + if (. | length) == 1 then + .[0] + else + . + end + ' diff --git a/bin/cac-listservers b/bin/cac-listservers new file mode 100755 index 000000000..1e815d2af --- /dev/null +++ b/bin/cac-listservers @@ -0,0 +1,12 @@ +#! /bin/sh +set -euf + +listservers=$(_cac_get_api_v1 listservers) +status=$(echo "$listservers" | jq -r .status) + +if [ "$status" = ok ]; then + echo "$listservers" | jq -r .data +else + echo "$0: bad listservers status: $status" >&2 + exit 1 +fi diff --git a/bin/cac-listtasks b/bin/cac-listtasks new file mode 100755 index 000000000..14be3948a --- /dev/null +++ b/bin/cac-listtasks @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_get_api_v1 listtasks diff --git a/bin/cac-listtemplates b/bin/cac-listtemplates new file mode 100755 index 000000000..c4414e019 --- /dev/null +++ b/bin/cac-listtemplates @@ -0,0 +1,4 @@ +#! /bin/sh +set -euf + +exec _cac_get_api_v1 listtemplates diff --git a/bin/cac-powerop b/bin/cac-powerop new file mode 100755 index 000000000..c897835f0 --- /dev/null +++ b/bin/cac-powerop @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 powerop sid="$1" action="$2" diff --git a/bin/cac-rdns b/bin/cac-rdns new file mode 100755 index 000000000..c2d9ecdab --- /dev/null +++ b/bin/cac-rdns @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 rdns sid="$1" hostname="$2" diff --git a/bin/cac-renameserver b/bin/cac-renameserver new file mode 100755 index 000000000..f0eff9b3d --- /dev/null +++ b/bin/cac-renameserver @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 renameserver sid="$1" name="$2" diff --git a/bin/cac-runmode b/bin/cac-runmode new file mode 100755 index 000000000..200b9fb79 --- /dev/null +++ b/bin/cac-runmode @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 rdns sid="$1" mode="$2" diff --git a/bin/cac-ssh b/bin/cac-ssh new file mode 100755 index 000000000..a0ec5dcf3 --- /dev/null +++ b/bin/cac-ssh @@ -0,0 +1,17 @@ +#! /bin/sh +set -euf + +server=$1 +shift + +address=$(echo $server | jq -r .ip) +target=root@$address + +SSHPASS=$(echo $server | jq -r .rootpass) +export SSHPASS + +exec sshpass -e ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "$target" \ + "$@" diff --git a/bin/cacnixos-networking b/bin/cacnixos-networking new file mode 100755 index 000000000..4b246ebf1 --- /dev/null +++ b/bin/cacnixos-networking @@ -0,0 +1,28 @@ +#! /bin/sh +# cacnixos-networking : cac-server x hostname -> nixos-module +# TODO use label for hostname +set -euf + +server=$1 +hostname=$2 + +address=$(echo $server | jq -r .ip) +gateway=$(echo $server | jq -r .gateway) +nameserver=8.8.8.8 +netmask=$(echo $server | jq -r .netmask) +prefix=$(netmask-to-prefix $netmask) + +printf '{...}:\n' +printf '{\n' +printf ' networking.hostName = "%s";\n' $hostname +printf ' networking.interfaces.enp2s1.ip4 = [\n' +printf ' {\n' +printf ' address = "%s";\n' $address +printf ' prefixLength = %d;\n' $prefix +printf ' }\n' +printf ' ];\n' +printf ' networking.defaultGateway = "%s";\n' $gateway +printf ' networking.nameservers = [\n' +printf ' "%s"\n' $nameserver +printf ' ];\n' +printf '}\n' diff --git a/bin/filter-secrets b/bin/filter-secrets new file mode 100755 index 000000000..6fcce73c1 --- /dev/null +++ b/bin/filter-secrets @@ -0,0 +1,6 @@ +#! /bin/sh +# filter_secrets : lines string |> lines secrets-file-candidate +set -euf + +# Notice how false positives are possible. +sed -n 's:^\(.*/\)\?\(secrets/.*\):'"${PWD//:/\\:}"'/\2:p' diff --git a/bin/import-statements b/bin/import-statements new file mode 100755 index 000000000..12c887970 --- /dev/null +++ b/bin/import-statements @@ -0,0 +1,10 @@ +#! /bin/sh +# import-statements : lines (path ":" string) |> lines (path ":" relpath) +set -euf +sed -n ' + s@^\([^:]\+:\)\('"$(bre-invert-word import)"'\)*\<import\s\+@\1@ + t1;d + :1; s@^\([^:]\+:\)\(\.*/\S*\)@\1\2\n@ + t2;d + :2; P;D +' diff --git a/bin/infest-CentOS-7-64bit b/bin/infest-CentOS-7-64bit new file mode 100755 index 000000000..a8afea14b --- /dev/null +++ b/bin/infest-CentOS-7-64bit @@ -0,0 +1,150 @@ +#! /bin/sh +set -euf + +server=$1 +hostname=$2 + +address=$(echo $server | jq -r .ip) +RSYNC_RSH='sshpass -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' +SSHPASS=$(echo $server | jq -r .rootpass) +export SSHPASS +export RSYNC_RSH + +main="modules/$hostname/default.nix" +target="root@$address" + +cacnixos-networking "$server" $hostname \ + > modules/$hostname/networking.nix + +echo '( + set -xeuf + type bzip2 || yum install -y bzip2 + type rsync || yum install -y rsync +)' \ + | sshpass -e ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "root@$address" \ + /bin/sh + +make-rsync-filter "$main" \ + | rsync -f '. -' -zvrlptD --delete-excluded ./ "$target":/etc/nixos/ + +# +# +# +echo '( + set -xeuf + groupadd -g 30000 nixbld || : + for i in `seq 1 10`; do + useradd -c "foolsgarden Nix build user $i" \ + -d /var/empty \ + -s /sbin/nologin \ + -g 30000 \ + -G 30000 \ + -l -u $(expr 30000 + $i) \ + nixbld$i || : + rm -f /var/spool/mail/nixbld$i + done + + #curl https://nixos.org/nix/install | sh + nix_tar=$nix_basename.tar.bz2 + if ! echo $nix_sha256 $nix_tar | sha256sum -c; then + curl -O -C - $nix_url || : + if ! echo $nix_sha256 $nix_tar | sha256sum -c; then + curl -O $nix_url || : + if ! echo $nix_sha256 $nix_tar | sha256sum -c; then + echo $0: cannot download $nix_url >&2 + exit 5 + fi + fi + fi + + if ! test -d $nix_basename; then + tar jxf $nix_basename.tar.bz2 + fi + + nix_find=$nix_basename.find.txt + if ! echo $nix_find_sha1sum $nix_find | sha1sum -c; then + find $nix_basename | sort > $nix_find + if ! echo $nix_find_sha1sum $nix_find | sha1sum -c; then + echo $0: cannot unpack $nix_basename.tar.bz2 >&2 + # TODO we could retry + exit 6 + fi + fi + + mkdir -p bin + PATH=$HOME/bin:$PATH + export PATH + + # generate fake sudo because + # sudo: sorry, you must have a tty to run sudo + { + echo "#! /bin/sh" + echo "exec env \"\$@\"" + } > bin/sudo + chmod +x bin/sudo + + ./$nix_basename/install + + . /root/.nix-profile/etc/profile.d/nix.sh + + nixpkgs_expr="import <nixpkgs> { system = builtins.currentSystem; }" + nixpkgs_path=$( + find /nix/store -mindepth 1 -maxdepth 1 -name *-nixpkgs-* -type d + ) + + for i in nixos-generate-config nixos-install; do + nix-env \ + --arg config "{ nix.package = ($nixpkgs_expr).nix; }" \ + --arg pkgs "$nixpkgs_expr" \ + --arg modulesPath "throw \"no modulesPath\"" \ + -f $nixpkgs_path/nixpkgs/nixos/modules/installer/tools/tools.nix \ + -iA config.system.build.$i + done + + # TODO following fail when aborted in-between + if ! test -d /int; then + mkdir -p /int + mount --bind /int /mnt + fi + if ! test -d /mnt/boot; then + mkdir -p /mnt/boot + mount /dev/sda1 /mnt/boot + fi + + mkdir -p /mnt/etc/nixos + rsync -zvrlptD --delete-excluded /etc/nixos/ /mnt/etc/nixos/ + + mkdir -m 0444 -p /mnt/var/empty + + ln -s $main /mnt/etc/nixos/configuration.nix + nixos-install \ + -I secrets=/etc/nixos/secrets + + find / \ + 1> /root/pre-rsync-find.out \ + 2> /root/pre-rsync-find.err + + rsync -va --force /int/ / + + # find / -type f -mtime +1 -exec rm -v {} \; 2>&1 > rm.log + # ^ too aggressive, kills journal which is bad + # shutdown -r now + # nix-channel --add https://nixos.org/channels/nixos-unstable nixos + # nix-channel --remove nixpkgs + # nix-channel --update + +)' \ + | sshpass -e ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "root@$address" \ + -T /usr/bin/env \ + nix_url="$nix_url" \ + nix_basename="$(basename $nix_url .tar.bz2)" \ + nix_sha256="$nix_sha256" \ + nix_find_sha1sum="$nix_find_sha1sum" \ + main="$main" \ + /bin/sh diff --git a/bin/infest-cac b/bin/infest-cac new file mode 100755 index 000000000..d7d7bb96c --- /dev/null +++ b/bin/infest-cac @@ -0,0 +1,21 @@ +#! /bin/sh +set -euf + +server=$(cac-get-server-by servername "$1") +hostname=$2 + +serverstatus=$(echo $server | jq -r .status) +case $serverstatus in + 'Powered On') : ;; + *) + echo $0: bad server status: $serverstatus >&2 + exit 2 +esac + +template=$(echo $server | jq -r .template) +case $template in + 'CentOS-7-64bit') infest-"$template" "$server" "$hostname";; + *) + echo $0: bad template: $template >&2 + exit 3 +esac diff --git a/bin/json-assert-type b/bin/json-assert-type new file mode 100755 index 000000000..29cadad65 --- /dev/null +++ b/bin/json-assert-type @@ -0,0 +1,18 @@ +#! /bin/sh +set -euf + +formal_type=$1 + +actual_value=$2 +actual_type=$(echo $actual_value | jq -r type) + +if [ "$actual_type" != "$formal_type" ]; then + backtrace + printf 'error: expected %s, got %s\n' \ + "$formal_type" \ + "$actual_type" \ + >&2 + exit 1 +fi + +echo "$actual_value" diff --git a/bin/list-hosts b/bin/list-hosts new file mode 100755 index 000000000..e25a8ac4f --- /dev/null +++ b/bin/list-hosts @@ -0,0 +1,7 @@ +#! /bin/sh +# list-hosts : lines tinc-host-file +set -euf + +# Precondition: $PWD/hosts is the correct repository :) +git -C hosts ls-tree --name-only HEAD \ + | awk '{print ENVIRON["PWD"]"/hosts/"$$0}' diff --git a/bin/list-module-imports b/bin/list-module-imports new file mode 100755 index 000000000..39d11bf34 --- /dev/null +++ b/bin/list-module-imports @@ -0,0 +1,20 @@ +#! /bin/sh +# list-module-imports : nix-file -> lines nix-file +set -euf + +if echo "$1" | grep -q ^/; then + : +else + set -- "./$1" +fi + +imports=$(nix-instantiate \ + -I secrets=secrets \ + --strict \ + --json \ + --eval \ + -E \ + "with builtins; with import ./lib/modules.nix; map toString (list-imports $1)") + +echo "$imports" \ + | jq -r .[] diff --git a/bin/ls-bre b/bin/ls-bre new file mode 100755 index 000000000..ae978895c --- /dev/null +++ b/bin/ls-bre @@ -0,0 +1,12 @@ +#! /bin/sh +# ls-bre : directory -> BRE +# Create a BRE from the files in a directory. +set -euf + +ls "$1" \ + | tr \\n / \ + | sed ' + s:[\.\[\\\*\^\$]:\\&:g + s:/$:: + s:/:\\|:g + ' diff --git a/bin/make-parent-dirs b/bin/make-parent-dirs new file mode 100755 index 000000000..f4717b249 --- /dev/null +++ b/bin/make-parent-dirs @@ -0,0 +1,10 @@ +#! /bin/sh +# make-parent-dirs : lines path |> lines directory +# List all parent directories of a path. +set -euf + +set -- "$(sed -n 's|/[^/]*$||p' | grep . | sort | uniq)" +if echo "$1" | grep -q .; then + echo "$1" + echo "$1" | make-parent-dirs +fi diff --git a/bin/make-relative-to b/bin/make-relative-to new file mode 100755 index 000000000..9d947e175 --- /dev/null +++ b/bin/make-relative-to @@ -0,0 +1,6 @@ +#! /bin/sh +# make-relative-to : lines path |> directory -> lines path +# Non-matching paths won't get altered. +set -euf + +sed "s:^$(echo "$1/" | bre-escape | sed 's/:/\\:/g')::" diff --git a/bin/make-rsync-filter b/bin/make-rsync-filter new file mode 100755 index 000000000..26e070adb --- /dev/null +++ b/bin/make-rsync-filter @@ -0,0 +1,33 @@ +#! /bin/sh +# make-rsync-filter : nixos-config -> rsync-filter +set -euf + +main=$1 + +hosts=$(list-hosts) +module_imports=$(list-module-imports "$main") +other_imports=$( + echo "$module_imports" \ + | xargs grep -H . \ + | import-statements \ + | slash-path-relpath \ + | undot-paths \ + | sort \ + | uniq \ + | sed '/\.nix$/!s:$:/default.nix:' \ + ) +secrets=$(echo "$module_imports" | xargs cat | quoted-strings | filter-secrets) + +# TODO collect all other paths from *_imports + +abs_deps=$( + echo "$hosts" + echo "$module_imports" + echo "$other_imports" + echo "$secrets" +) + +rel_deps=$(echo "$abs_deps" | make-relative-to "$PWD") +filter=$(echo "$rel_deps" | make-rsync-whitelist) + +echo "$filter" diff --git a/bin/make-rsync-whitelist b/bin/make-rsync-whitelist new file mode 100755 index 000000000..a1b09c801 --- /dev/null +++ b/bin/make-rsync-whitelist @@ -0,0 +1,15 @@ +#! /bin/sh +# make-rsync-whitelist : lines relpath |> liens rsync-filter +set -euf + +set -- "$(cat)" + +# include all files in stdin and their directories +{ + echo "$1" + echo "$1" | make-parent-dirs | sort | uniq +} \ + | sed 's|^|+ /|' + +# exclude everything else +echo '- *' diff --git a/bin/netmask-to-prefix b/bin/netmask-to-prefix new file mode 100755 index 000000000..1c4dbeb28 --- /dev/null +++ b/bin/netmask-to-prefix @@ -0,0 +1,12 @@ +#! /bin/sh +set -euf + +netmask=$1 + +binaryNetmask=$(echo $1 | sed 's/^/obase=2;/;s/\./;/g' | bc | tr -d \\n) +binaryPrefix=$(echo $binaryNetmask | sed -n 's/^\(1*\)0*$/\1/p') +if ! echo $binaryPrefix | grep -q .; then + echo $0: bad netmask: $netmask >&2 + exit 4 +fi +printf %s $binaryPrefix | tr -d 0 | wc -c diff --git a/bin/nixpkgs-rev b/bin/nixpkgs-rev new file mode 100755 index 000000000..1acde1e4e --- /dev/null +++ b/bin/nixpkgs-rev @@ -0,0 +1,13 @@ +#! /bin/sh +# nixpkgs-rev : nixos-config -> git_rev +set -euf +nix-instantiate \ + -I nixos-config="$1" \ + --eval \ + --json \ + -E \ + ' + (import <nixos-config> {config={}; pkgs={};}).nixpkgs.rev + ' \ + 2> /dev/null \ + | jq -r . 2> /dev/null diff --git a/bin/nixpkgs-url b/bin/nixpkgs-url new file mode 100755 index 000000000..9549f0c77 --- /dev/null +++ b/bin/nixpkgs-url @@ -0,0 +1,13 @@ +#! /bin/sh +# nixpkgs-url : nixos-config -> git_url +set -euf +nix-instantiate \ + -I nixos-config="$1" \ + --eval \ + --json \ + -E \ + ' + (import <nixos-config> {config={}; pkgs={};}).nixpkgs.url + ' \ + 2> /dev/null \ + | jq -r . 2> /dev/null diff --git a/bin/quoted-strings b/bin/quoted-strings new file mode 100755 index 000000000..e64039101 --- /dev/null +++ b/bin/quoted-strings @@ -0,0 +1,15 @@ +#! /bin/sh +# quoted_strings : lines string |> lines string +# Extract all (double-) quoted strings from stdin. +# +# 0. find begin of string or skip line +# 1. find end of string or skip line +# 2. print string and continue after string +set -euf + +sed ' + s:[^"]*":: ;t1;d + :1; s:\(\([^"]\|\\"\)*\)":\1\n: ;t2;d + :2; P;D +' \ + | sed 's:\\":":g' diff --git a/bin/slash-path-relpath b/bin/slash-path-relpath new file mode 100755 index 000000000..40230a70c --- /dev/null +++ b/bin/slash-path-relpath @@ -0,0 +1,8 @@ +#! /bin/sh +# slash_path_relpath : lines (path ":" relpath) |> lines path +# +# Example: "/foo/bar: baz" => "/foo/baz" +# +set -euf + +sed -n 's@/[^/]\+:@/@p' diff --git a/bin/ssh-deploy b/bin/ssh-deploy new file mode 100755 index 000000000..fe50677df --- /dev/null +++ b/bin/ssh-deploy @@ -0,0 +1,26 @@ +#! /bin/sh +# ssh-deploy : nixos-config x [user@]hostname -> () +set -xeuf + +main=$1 +target=$2 +nixpkgs_dir=/var/nixpkgs # TODO make configurable + +git_url=$(nixpkgs-url $main) +git_rev=$(nixpkgs-rev $main) + +if [ "$git_url" = '' ] || [ "$git_rev" = '' ]; then + echo "specify nixpkgs.url and nixpkgs.rev in $main !" + exit 23 +fi + +filter=$(make-rsync-filter "$main") + +echo "$filter" \ + | rsync -f '. -' -zvrlptD --delete-excluded ./ "$target":/etc/nixos/ + +ssh-fetch-git "$target" "$nixpkgs_dir" "$git_url" "$git_rev" +ssh "$target" nixos-rebuild switch \ + -I nixos-config=/etc/nixos/"$main" \ + -I nixpkgs="$nixpkgs_dir" \ + -I secrets=/etc/nixos/secrets \ diff --git a/bin/ssh-fetch-git b/bin/ssh-fetch-git new file mode 100755 index 000000000..7de58ab73 --- /dev/null +++ b/bin/ssh-fetch-git @@ -0,0 +1,35 @@ +#! /bin/sh +# ssh-fetch-git : [user@]hostname x remote_dir x git_url x git_rev -> () +set -euf + +target=$1 +remote_dir=$2 +git_url=$3 +git_rev=$4 + +echo ' + set -euf + + if [ ! -d "$remote_dir" ]; then + mkdir -p "$remote_dir" + fi + + cd "$remote_dir" + + git init -q + + if ! current_url=$(git config remote.src.url); then + git remote add src "$git_url" + elif [ $current_url != $git_url ]; the |