diff options
49 files changed, 860 insertions, 912 deletions
diff --git a/.gitignore b/.gitignore index a37850ab7..bad1d00ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /.graveyard /hosts /secrets +/tmp diff --git a/bin/_cac_curl_api_v1 b/bin/_cac_curl_api_v1 new file mode 100755 index 000000000..65acebd9a --- /dev/null +++ b/bin/_cac_curl_api_v1 @@ -0,0 +1,10 @@ +#! /bin/sh +set -euf + +exec _cac_exec curl -fsS "$1" "https://panel.cloudatcost.com/api/v1/$2.php" $( + shift 2 + set -- "$@" login="$cac_login" key="$cac_key" + for arg; do + echo -d $(printf '%s' "$arg" | urlencode) + done +) diff --git a/bin/_cac_exec b/bin/_cac_exec new file mode 100755 index 000000000..c932454e2 --- /dev/null +++ b/bin/_cac_exec @@ -0,0 +1,8 @@ +#! /bin/sh +set -euf + +if test -z "${cac_via-}"; then + exec "$@" +else + exec ssh -q "$cac_via" -t "$@" +fi diff --git a/bin/_cac_get_api_v1 b/bin/_cac_get_api_v1 new file mode 100755 index 000000000..67aac8560 --- /dev/null +++ b/bin/_cac_get_api_v1 @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_curl_api_v1 -G "$@" diff --git a/bin/_cac_post_api_v1 b/bin/_cac_post_api_v1 new file mode 100755 index 000000000..b946ed9fa --- /dev/null +++ b/bin/_cac_post_api_v1 @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_curl_api_v1 -XPOST "$@" diff --git a/bin/cac-cloudpro-build b/bin/cac-cloudpro-build new file mode 100755 index 000000000..782fa0d72 --- /dev/null +++ b/bin/cac-cloudpro-build @@ -0,0 +1,5 @@ +#! /bin/sh +set -euf + +# default os=26 is CentOS-7-64bit +exec _cac_post_api_v1 cloudpro/build cpu="$1" ram="$2" storage="$3" os="${4-26}" diff --git a/bin/cac-cloudpro-delete b/bin/cac-cloudpro-delete new file mode 100755 index 000000000..ee1dbbc7e --- /dev/null +++ b/bin/cac-cloudpro-delete @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 cloudpro/delete sid="$1" diff --git a/bin/cac-cloudpro-resources b/bin/cac-cloudpro-resources new file mode 100755 index 000000000..9ec5872e7 --- /dev/null +++ b/bin/cac-cloudpro-resources @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_get_api_v1 cloudpro/resources diff --git a/bin/cac-console b/bin/cac-console new file mode 100755 index 000000000..ed1cbd5ff --- /dev/null +++ b/bin/cac-console @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 console sid="$1" diff --git a/bin/cac-get-server-by b/bin/cac-get-server-by new file mode 100755 index 000000000..b6d6b4ba3 --- /dev/null +++ b/bin/cac-get-server-by @@ -0,0 +1,17 @@ +#! /bin/sh +set -euf + +cac-listservers \ + | jq \ + --arg k "$1" \ + --arg v "$2" \ + ' + map(select(.[$k]==$v)) | + if (. | length) == 0 then + null + elif (. | length) == 1 then + .[0] + else + . + end + ' diff --git a/bin/cac-listservers b/bin/cac-listservers new file mode 100755 index 000000000..1e815d2af --- /dev/null +++ b/bin/cac-listservers @@ -0,0 +1,12 @@ +#! /bin/sh +set -euf + +listservers=$(_cac_get_api_v1 listservers) +status=$(echo "$listservers" | jq -r .status) + +if [ "$status" = ok ]; then + echo "$listservers" | jq -r .data +else + echo "$0: bad listservers status: $status" >&2 + exit 1 +fi diff --git a/bin/cac-listtasks b/bin/cac-listtasks new file mode 100755 index 000000000..14be3948a --- /dev/null +++ b/bin/cac-listtasks @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_get_api_v1 listtasks diff --git a/bin/cac-listtemplates b/bin/cac-listtemplates new file mode 100755 index 000000000..c4414e019 --- /dev/null +++ b/bin/cac-listtemplates @@ -0,0 +1,4 @@ +#! /bin/sh +set -euf + +exec _cac_get_api_v1 listtemplates diff --git a/bin/cac-powerop b/bin/cac-powerop new file mode 100755 index 000000000..c897835f0 --- /dev/null +++ b/bin/cac-powerop @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 powerop sid="$1" action="$2" diff --git a/bin/cac-rdns b/bin/cac-rdns new file mode 100755 index 000000000..c2d9ecdab --- /dev/null +++ b/bin/cac-rdns @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 rdns sid="$1" hostname="$2" diff --git a/bin/cac-renameserver b/bin/cac-renameserver new file mode 100755 index 000000000..f0eff9b3d --- /dev/null +++ b/bin/cac-renameserver @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 renameserver sid="$1" name="$2" diff --git a/bin/cac-runmode b/bin/cac-runmode new file mode 100755 index 000000000..200b9fb79 --- /dev/null +++ b/bin/cac-runmode @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 rdns sid="$1" mode="$2" diff --git a/bin/cac-ssh b/bin/cac-ssh new file mode 100755 index 000000000..a0ec5dcf3 --- /dev/null +++ b/bin/cac-ssh @@ -0,0 +1,17 @@ +#! /bin/sh +set -euf + +server=$1 +shift + +address=$(echo $server | jq -r .ip) +target=root@$address + +SSHPASS=$(echo $server | jq -r .rootpass) +export SSHPASS + +exec sshpass -e ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "$target" \ + "$@" diff --git a/bin/cacnixos-networking b/bin/cacnixos-networking new file mode 100755 index 000000000..4b246ebf1 --- /dev/null +++ b/bin/cacnixos-networking @@ -0,0 +1,28 @@ +#! /bin/sh +# cacnixos-networking : cac-server x hostname -> nixos-module +# TODO use label for hostname +set -euf + +server=$1 +hostname=$2 + +address=$(echo $server | jq -r .ip) +gateway=$(echo $server | jq -r .gateway) +nameserver=8.8.8.8 +netmask=$(echo $server | jq -r .netmask) +prefix=$(netmask-to-prefix $netmask) + +printf '{...}:\n' +printf '{\n' +printf ' networking.hostName = "%s";\n' $hostname +printf ' networking.interfaces.enp2s1.ip4 = [\n' +printf ' {\n' +printf ' address = "%s";\n' $address +printf ' prefixLength = %d;\n' $prefix +printf ' }\n' +printf ' ];\n' +printf ' networking.defaultGateway = "%s";\n' $gateway +printf ' networking.nameservers = [\n' +printf ' "%s"\n' $nameserver +printf ' ];\n' +printf '}\n' diff --git a/bin/copy-secrets b/bin/copy-secrets new file mode 100755 index 000000000..36854eaf1 --- /dev/null +++ b/bin/copy-secrets @@ -0,0 +1,28 @@ +#! /bin/sh +set -euf + +host=$1 + +target=root@$host + +nixos_config=$config_root/modules/$host +secrets_nix=$secrets_root/$host/nix +secrets_rsync=$secrets_root/$host/rsync + +if ! test -e "$secrets_rsync"; then + exit # nothing to do +fi + +retiolum_secret=$(nixos-query $host services.retiolum.privateKeyFile) +retiolum_uid=$(nixos-query $host users.extraUsers.retiolum-tinc.uid) + +ejabberd_secret=/etc/ejabberd/ejabberd.pem +ejabberd_uid=$(nixos-query $host users.extraUsers.ejabberd.uid) + +rsync -cz --chown=0:0 -vr "$secrets_rsync/" "$target:/" + +ssh "$target" -T <<EOF +set -euf +! test -f $retiolum_secret || chown -v $retiolum_uid:0 $retiolum_secret +! test -f $ejabberd_secret || chown -v $ejabberd_uid:0 $ejabberd_secret +EOF diff --git a/bin/infest-CentOS-7-64bit b/bin/infest-CentOS-7-64bit new file mode 100755 index 000000000..a8afea14b --- /dev/null +++ b/bin/infest-CentOS-7-64bit @@ -0,0 +1,150 @@ +#! /bin/sh +set -euf + +server=$1 +hostname=$2 + +address=$(echo $server | jq -r .ip) +RSYNC_RSH='sshpass -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' +SSHPASS=$(echo $server | jq -r .rootpass) +export SSHPASS +export RSYNC_RSH + +main="modules/$hostname/default.nix" +target="root@$address" + +cacnixos-networking "$server" $hostname \ + > modules/$hostname/networking.nix + +echo '( + set -xeuf + type bzip2 || yum install -y bzip2 + type rsync || yum install -y rsync +)' \ + | sshpass -e ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "root@$address" \ + /bin/sh + +make-rsync-filter "$main" \ + | rsync -f '. -' -zvrlptD --delete-excluded ./ "$target":/etc/nixos/ + +# +# +# +echo '( + set -xeuf + groupadd -g 30000 nixbld || : + for i in `seq 1 10`; do + useradd -c "foolsgarden Nix build user $i" \ + -d /var/empty \ + -s /sbin/nologin \ + -g 30000 \ + -G 30000 \ + -l -u $(expr 30000 + $i) \ + nixbld$i || : + rm -f /var/spool/mail/nixbld$i + done + + #curl https://nixos.org/nix/install | sh + nix_tar=$nix_basename.tar.bz2 + if ! echo $nix_sha256 $nix_tar | sha256sum -c; then + curl -O -C - $nix_url || : + if ! echo $nix_sha256 $nix_tar | sha256sum -c; then + curl -O $nix_url || : + if ! echo $nix_sha256 $nix_tar | sha256sum -c; then + echo $0: cannot download $nix_url >&2 + exit 5 + fi + fi + fi + + if ! test -d $nix_basename; then + tar jxf $nix_basename.tar.bz2 + fi + + nix_find=$nix_basename.find.txt + if ! echo $nix_find_sha1sum $nix_find | sha1sum -c; then + find $nix_basename | sort > $nix_find + if ! echo $nix_find_sha1sum $nix_find | sha1sum -c; then + echo $0: cannot unpack $nix_basename.tar.bz2 >&2 + # TODO we could retry + exit 6 + fi + fi + + mkdir -p bin + PATH=$HOME/bin:$PATH + export PATH + + # generate fake sudo because + # sudo: sorry, you must have a tty to run sudo + { + echo "#! /bin/sh" + echo "exec env \"\$@\"" + } > bin/sudo + chmod +x bin/sudo + + ./$nix_basename/install + + . /root/.nix-profile/etc/profile.d/nix.sh + + nixpkgs_expr="import <nixpkgs> { system = builtins.currentSystem; }" + nixpkgs_path=$( + find /nix/store -mindepth 1 -maxdepth 1 -name *-nixpkgs-* -type d + ) + + for i in nixos-generate-config nixos-install; do + nix-env \ + --arg config "{ nix.package = ($nixpkgs_expr).nix; }" \ + --arg pkgs "$nixpkgs_expr" \ + --arg modulesPath "throw \"no modulesPath\"" \ + -f $nixpkgs_path/nixpkgs/nixos/modules/installer/tools/tools.nix \ + -iA config.system.build.$i + done + + # TODO following fail when aborted in-between + if ! test -d /int; then + mkdir -p /int + mount --bind /int /mnt + fi + if ! test -d /mnt/boot; then + mkdir -p /mnt/boot + mount /dev/sda1 /mnt/boot + fi + + mkdir -p /mnt/etc/nixos + rsync -zvrlptD --delete-excluded /etc/nixos/ /mnt/etc/nixos/ + + mkdir -m 0444 -p /mnt/var/empty + + ln -s $main /mnt/etc/nixos/configuration.nix + nixos-install \ + -I secrets=/etc/nixos/secrets + + find / \ + 1> /root/pre-rsync-find.out \ + 2> /root/pre-rsync-find.err + + rsync -va --force /int/ / + + # find / -type f -mtime +1 -exec rm -v {} \; 2>&1 > rm.log + # ^ too aggressive, kills journal which is bad + # shutdown -r now + # nix-channel --add https://nixos.org/channels/nixos-unstable nixos + # nix-channel --remove nixpkgs + # nix-channel --update + +)' \ + | sshpass -e ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "root@$address" \ + -T /usr/bin/env \ + nix_url="$nix_url" \ + nix_basename="$(basename $nix_url .tar.bz2)" \ + nix_sha256="$nix_sha256" \ + nix_find_sha1sum="$nix_find_sha1sum" \ + main="$main" \ + /bin/sh diff --git a/bin/infest-cac b/bin/infest-cac new file mode 100755 index 000000000..d7d7bb96c --- /dev/null +++ b/bin/infest-cac @@ -0,0 +1,21 @@ +#! /bin/sh +set -euf + +server=$(cac-get-server-by servername "$1") +hostname=$2 + +serverstatus=$(echo $server | jq -r .status) +case $serverstatus in + 'Powered On') : ;; + *) + echo $0: bad server status: $serverstatus >&2 + exit 2 +esac + +template=$(echo $server | jq -r .template) +case $template in + 'CentOS-7-64bit') infest-"$template" "$server" "$hostname";; + *) + echo $0: bad template: $template >&2 + exit 3 +esac diff --git a/bin/netmask-to-prefix b/bin/netmask-to-prefix new file mode 100755 index 000000000..1c4dbeb28 --- /dev/null +++ b/bin/netmask-to-prefix @@ -0,0 +1,12 @@ +#! /bin/sh +set -euf + +netmask=$1 + +binaryNetmask=$(echo $1 | sed 's/^/obase=2;/;s/\./;/g' | bc | tr -d \\n) +binaryPrefix=$(echo $binaryNetmask | sed -n 's/^\(1*\)0*$/\1/p') +if ! echo $binaryPrefix | grep -q .; then + echo $0: bad netmask: $netmask >&2 + exit 4 +fi +printf %s $binaryPrefix | tr -d 0 | wc -c diff --git a/bin/nixos-build b/bin/nixos-build new file mode 100755 index 000000000..a0c9551fa --- /dev/null +++ b/bin/nixos-build @@ -0,0 +1,25 @@ +#! /bin/sh +# +# build : hostname -> system-path +# +set -euf + +host=$1 + +#target=root@$host + +nixpkgs=$nixpkgs_root/$host +nixos_config=$config_root/modules/$host +secrets_nix=$secrets_root/$host/nix +secrets_rsync=$secrets_root/$host/rsync + +nixos-fetch-git "$host" + +nix-build \ + -I "$nixpkgs" \ + -I nixos-config="$nixos_config" \ + -I retiolum-hosts="$retiolum_hosts" \ + -I secrets="$secrets_nix" \ + -A system \ + --no-out-link \ + '<nixos>' diff --git a/bin/nixos-deploy b/bin/nixos-deploy new file mode 100755 index 000000000..6b8418696 --- /dev/null +++ b/bin/nixos-deploy @@ -0,0 +1,16 @@ +#! /bin/sh +# +# deploy +# +set -euf + +host=$1 +system=${2-$(nixos-build "$host")} + +target=root@$host + +nix-copy-closure --gzip --to "$target" "$system" + +copy-secrets "$host" + +ssh ${NIX_SSHOPTS-} "$target" "$system/bin/switch-to-configuration" switch diff --git a/bin/nixos-fetch-git b/bin/nixos-fetch-git new file mode 100755 index 000000000..7002208b5 --- /dev/null +++ b/bin/nixos-fetch-git @@ -0,0 +1,32 @@ +#! /bin/sh +# +# nixos-fetch-git : nixos-config -> ... +# +set -euf + +host=$1 + +target=root@$host + +git_rev=$(nixos-query "$host" nixpkgs.rev) +git_url=$(nixos-query "$host" nixpkgs.url) + +worktree=$nixpkgs_root/$host + +if [ ! -d "$worktree" ]; then + mkdir -p "$worktree" +fi + +cd "$worktree" + +git init -q + +if ! current_url=$(git config remote.src.url); then + git remote add src "$git_url" +elif [ "$current_url" != "$git_url" ]; then + git remote set-url src "$git_url" +fi + +git fetch src + +git checkout "$git_rev" diff --git a/bin/nixos-query b/bin/nixos-query new file mode 100755 index 000000000..65b5c9672 --- /dev/null +++ b/bin/nixos-query @@ -0,0 +1,21 @@ +#! /bin/sh +set -euf + +host=$1 +attr=$2 + +nixpkgs=$nixpkgs_root/$host +nixos_config=$config_root/modules/$host +secrets_nix=$secrets_root/$host/nix +secrets_rsync=$secrets_root/$host/rsync + +nix-instantiate \ + -I "$nixpkgs" \ + -I nixos-config="$nixos_config" \ + -I retiolum-hosts="$retiolum_hosts" \ + -I secrets="$secrets_nix" \ + -A config."$attr" \ + '<nixos>' \ + --eval \ + --json \ + | jq -r . diff --git a/bin/urlencode b/bin/urlencode new file mode 100755 index 000000000..02ca03075 --- /dev/null +++ b/bin/urlencode @@ -0,0 +1,35 @@ +#! /bin/sh +set -euf +exec sed ' + s/%/%25/g + s/ /%20/g + s/!/%21/g + s/"/%22/g + s/#/%23/g + s/\$/%24/g + s/\&/%26/g + s/'\''/%27/g + s/(/%28/g + s/)/%29/g + s/\*/%2a/g + s/+/%2b/g + s/,/%2c/g + s/-/%2d/g + s/\./%2e/g + s/\//%2f/g + s/:/%3a/g + s/;/%3b/g + s//%3e/g + s/?/%3f/g + s/@/%40/g + s/\[/%5b/g + s/\\/%5c/g + s/\]/%5d/g + s/\^/%5e/g + s/_/%5f/g + s/`/%60/g + s/{/%7b/g + s/|/%7c/g + s/}/%7d/g + s/~/%7e/g +' @@ -1,15 +1,16 @@ #! /bin/sh # -# usage: ./deploy HOST [[USER@]HOST] +# usage: ./deploy HOST # set -euf -. ./lib/prelude.sh - -user=root host=$1 -config=./modules/$host/default.nix -target=${2-$user@$host} +export PATH="$PWD/bin:$PATH" +#export nixpkgs=/var/nixpkgs +export nixpkgs_root=$PWD/tmp/nixpkgs +export config_root=$PWD +export retiolum_hosts=$PWD/hosts +export secrets_root=$PWD/secrets -verbose deploy "$config" "$target" +exec nixos-deploy "$host" @@ -1,187 +1,15 @@ #! /bin/sh -set -xeuf +# +# usage: ./infest cac-servername hostname +# +set -euf -. ./lib/prelude.sh -. ./lib/cac.sh -. ./lib/cacnixos.sh +PATH="$PWD/bin${PATH+:$PATH}" +export PATH nix_url=https://nixos.org/releases/nix/nix-1.8/nix-1.8-x86_64-linux.tar.bz2 nix_sha256=52fab207b4ce4d098a12d85357d0353e972c492bab0aa9e08e1600363e76fefb nix_find_sha1sum=86f8775bd4f0841edd4c816df861cebf509d58c3 +export nix_url nix_sha256 nix_find_sha1sum -# This is somewhat required because cloudatcost requires whitelisting -# of hosts. If you whitelist your localhost, then leave this empty. -# cac_via= -# -# cac_key= -# cac_login= -# cac_servername= - -# hostname= - -main() { - server=$(cac_getserver_by_servername "$cac_servername") - - serverstatus=$(echo $server | jq -r .status) - case $serverstatus in - 'Powered On') : ;; - *) - echo $0: bad server status: $serverstatus >&2 - exit 2 - esac - - template=$(echo $server | jq -r .template) - case $template in - 'CentOS-7-64bit') infest_centos7_64bit "$server";; - *) - echo $0: bad template: $template >&2 - exit 3 - esac -} - - -infest_centos7_64bit() { - server=$1 - address=$(echo $server | jq -r .ip) - RSYNC_RSH='sshpass -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' - SSHPASS=$(echo $server | jq -r .rootpass) - export SSHPASS - export RSYNC_RSH - - main="modules/$hostname/default.nix" - target="root@$address" - - cacnixos_networking "$server" $hostname \ - > modules/$hostname/networking.nix - - echo '( - set -xeuf - type bzip2 || yum install -y bzip2 |