From 14ddb767eb10dbe43d3112c4b4674f6c1d4ff32a Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 11:18:00 +0100 Subject: k 5 mv krebs-ci test/infest-cac-centos7 --- krebs/5pkgs/test/infest-cac-centos7/default.nix | 39 ++++++++ krebs/5pkgs/test/infest-cac-centos7/notes | 117 ++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 krebs/5pkgs/test/infest-cac-centos7/default.nix create mode 100755 krebs/5pkgs/test/infest-cac-centos7/notes (limited to 'krebs/5pkgs/test/infest-cac-centos7') diff --git a/krebs/5pkgs/test/infest-cac-centos7/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix new file mode 100644 index 000000000..7f2e3f231 --- /dev/null +++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix @@ -0,0 +1,39 @@ +{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }: + +stdenv.mkDerivation rec { + name = "${shortname}-${version}"; + shortname = "infest-cac-centos7"; + version = "0.2.0"; + + src = ./notes; + + phases = [ + "installPhase" + ]; + buildInputs = [ makeWrapper ]; + + path = stdenv.lib.makeSearchPath "bin" [ + coreutils + cac + cacpanel + gnumake + gnused + jq + openssh + ]; + + installPhase = + '' + mkdir -p $out/bin + cp ${src} $out/bin/${shortname} + chmod +x $out/bin/${shortname} + wrapProgram $out/bin/${shortname} \ + --prefix PATH : ${path} + ''; + meta = with stdenv.lib; { + homepage = http://krebsco.de; + description = "Krebs CI Scripts"; + license = licenses.wtfpl; + maintainers = [ maintainers.makefu ]; + }; +} diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes new file mode 100755 index 000000000..1e350084c --- /dev/null +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -0,0 +1,117 @@ +#! /bin/sh + +# nix-shell -p gnumake jq openssh cac cacpanel +set -eufx + +# 2 secrets are required: + +krebs_cred=${krebs_cred-./cac.json} +retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} + +# Sanity +if test ! -r "$krebs_cred";then + echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 +fi +if test ! -r "$retiolum_key";then + echo "\$retiolum_key=$retiolum_key must be readable"; exit 1 +fi + +krebs_secrets=$(mktemp -d) +sec_file=$krebs_secrets/cac_config +krebs_ssh=$krebs_secrets/tempssh +export cac_resources_cache=$krebs_secrets/res_cache.json +export cac_servers_cache=$krebs_secrets/servers_cache.json +export cac_tasks_cache=$krebs_secrets/tasks_cache.json +export cac_templates_cache=$krebs_secrets/templates_cache.json +# we need to receive this key from buildmaster to speed up tinc bootstrap +TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT" +trap "$TRAP" INT TERM EXIT + +cat > $sec_file <&1\ + | jq -r .servername) + +id=servername:$name +trap "cac delete $id;$TRAP;exit" INT TERM EXIT +# TODO: timeout? + +wait_login_cac(){ + # timeout + for t in `seq 180`;do + # now we have a working cac server + if cac ssh $1 -o ConnectTimeout=10 \ + -o BatchMode=yes \ + cat /etc/redhat-release | \ + grep CentOS ;then + return 0 + fi + sleep 10 + done + return 1 +} +# die on timeout +wait_login_cac $id + +mkdir -p shared/2configs/temp +cac generatenetworking $id > \ + shared/2configs/temp/networking.nix +# new temporary ssh key we will use to log in after infest +ssh-keygen -f $krebs_ssh -N "" +cp $retiolum_key $krebs_secrets/retiolum.rsa_key.priv +# we override the directories for secrets and stockholm +# additionally we set the ssh key we generated +ip=$(cac getserver $id | jq -r .ip) + +cat > shared/2configs/temp/dirs.nix < $krebs_secrets/infest +sh -x $krebs_secrets/infest + +# TODO: generate secrets directory $krebs_secrets for nix import +cac powerop $id reset + +wait_login(){ + # timeout + for t in `seq 20`;do + # now we have a working cac server + if ssh -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -i $krebs_ssh \ + -o ConnectTimeout=10 \ + -o BatchMode=yes \ + root@$1 nixos-version ;then + return 0 + fi + sleep 10 + done + return 1 +} +wait_login $ip -- cgit v1.2.3 From cf3391704d88c49afba652715e1153888bf46099 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 23 Dec 2015 16:02:58 +0100 Subject: k 5 test/infest*: remove batch mode from cac ssh call this leads to "permission denied" --- krebs/5pkgs/test/infest-cac-centos7/notes | 1 - 1 file changed, 1 deletion(-) (limited to 'krebs/5pkgs/test/infest-cac-centos7') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 1e350084c..5fd0cae61 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -53,7 +53,6 @@ wait_login_cac(){ for t in `seq 180`;do # now we have a working cac server if cac ssh $1 -o ConnectTimeout=10 \ - -o BatchMode=yes \ cat /etc/redhat-release | \ grep CentOS ;then return 0 -- cgit v1.2.3 From 6e4351044195f1f3b5708785e760b9d118e2c229 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 02:20:24 +0100 Subject: k 5 test/infest*: up limit of final connect --- krebs/5pkgs/test/infest-cac-centos7/notes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'krebs/5pkgs/test/infest-cac-centos7') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 5fd0cae61..5bb5de2c4 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -99,7 +99,7 @@ cac powerop $id reset wait_login(){ # timeout - for t in `seq 20`;do + for t in `seq 90`;do # now we have a working cac server if ssh -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -- cgit v1.2.3 From 9ae664209328f6030bf3773e09dce7bcd14e82b4 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 24 Dec 2015 20:37:04 +0100 Subject: k 5 cacpanel: bump version to 0.2.3 --- krebs/5pkgs/test/infest-cac-centos7/notes | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'krebs/5pkgs/test/infest-cac-centos7') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 5bb5de2c4..cfb074423 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -29,11 +29,11 @@ trap "$TRAP" INT TERM EXIT cat > $sec_file < Date: Mon, 28 Dec 2015 13:23:34 +0100 Subject: k 5 test/infest-cac-centos7: use defer, loop until we get a working cac box --- krebs/5pkgs/test/infest-cac-centos7/notes | 77 ++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) (limited to 'krebs/5pkgs/test/infest-cac-centos7') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index cfb074423..3f4fcd859 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -8,6 +8,17 @@ set -eufx krebs_cred=${krebs_cred-./cac.json} retiolum_key=${retiolum_key-./retiolum.rsa_key.priv} +clear_defer(){ + echo "${trapstr:-exit}" + trap - INT TERM EXIT KILL +} +defer(){ + if test -z "${debug:-}"; then + trapstr="$1;${trapstr:-exit}" + trap "$trapstr" INT TERM EXIT KILL + fi +} + # Sanity if test ! -r "$krebs_cred";then echo "\$krebs_cred=$krebs_cred must be readable"; exit 1 @@ -24,8 +35,8 @@ export cac_servers_cache=$krebs_secrets/servers_cache.json export cac_tasks_cache=$krebs_secrets/tasks_cache.json export cac_templates_cache=$krebs_secrets/templates_cache.json # we need to receive this key from buildmaster to speed up tinc bootstrap -TRAP="rm -r $krebs_secrets;trap - INT TERM EXIT" -trap "$TRAP" INT TERM EXIT +defer "trap - INT TERM EXIT" +defer "rm -r $krebs_secrets" cat > $sec_file <&1\ - | jq -r .servername) - -id=servername:$name -trap "cac delete $id;$TRAP;exit" INT TERM EXIT -# TODO: timeout? - -wait_login_cac(){ - # timeout - for t in `seq 180`;do - # now we have a working cac server - if cac ssh $1 -o ConnectTimeout=10 \ - cat /etc/redhat-release | \ - grep CentOS ;then - return 0 - fi - sleep 10 - done - return 1 -} -# die on timeout -wait_login_cac $id +# preserve old trap +old_trapstr=$(clear_defer) +while true;do + # Template 26: CentOS7 + # TODO: use cac templates to determine the real Centos7 template in case it changes + name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\ + | jq -r .servername) + id=servername:$name + + clear_defer >/dev/null + defer "cac delete $id" + + # TODO: timeout? + + wait_login_cac(){ + # we wait for 15 minutes + for t in `seq 90`;do + # now we have a working cac server + if cac ssh $1 -o ConnectTimeout=10 \ + cat /etc/redhat-release | \ + grep CentOS ;then + return 0 + fi + sleep 10 + done + return 1 + } + # die on timeout + if ! wait_login_cac $id;then + echo "unable to boot a working system within time frame, retrying..." >&2 + echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" + eval "$(clear_defer)" + else + echo "got a working system" >&2 + fi +done +clear_defer >/dev/null +defer "cac delete $id;$old_trapstr" mkdir -p shared/2configs/temp cac generatenetworking $id > \ -- cgit v1.2.3 From 1798dbaf47fea7793545be2bc78ac5f1c8e27e18 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 30 Dec 2015 04:56:53 +0100 Subject: k 5 test: fix endless loop in test --- krebs/5pkgs/test/infest-cac-centos7/notes | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'krebs/5pkgs/test/infest-cac-centos7') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index 3f4fcd859..eee0bfc34 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -81,9 +81,10 @@ while true;do if ! wait_login_cac $id;then echo "unable to boot a working system within time frame, retrying..." >&2 echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" - eval "$(clear_defer)" + eval "$(clear_defer | sed 's/;exit//')" else echo "got a working system" >&2 + break fi done clear_defer >/dev/null -- cgit v1.2.3 From f0e802d2593ebe7101968deb3593f1c120f552fd Mon Sep 17 00:00:00 2001 From: makefu Date: Sat, 2 Jan 2016 21:36:51 +0100 Subject: k 5 test/infest-cac-centos7: add timeouts, error handling --- krebs/5pkgs/test/infest-cac-centos7/notes | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'krebs/5pkgs/test/infest-cac-centos7') diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes index eee0bfc34..6bfb6906e 100755 --- a/krebs/5pkgs/test/infest-cac-centos7/notes +++ b/krebs/5pkgs/test/infest-cac-centos7/notes @@ -55,9 +55,16 @@ old_trapstr=$(clear_defer) while true;do # Template 26: CentOS7 # TODO: use cac templates to determine the real Centos7 template in case it changes - name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\ - | jq -r .servername) - id=servername:$name + out=$(cac build cpu=1 ram=512 storage=10 os=26 2>&1) + if name=$(echo "$out" | jq -r .servername);then + id=servername:$name + echo "got a working machine, id=$id" + else + echo "Unable to build a virtual machine, retrying in 15 seconds" >&2 + echo "Output of build program: $out" >&2 + sleep 15 + continue + fi clear_defer >/dev/null defer "cac delete $id" @@ -65,8 +72,8 @@ while true;do # TODO: timeout? wait_login_cac(){ - # we wait for 15 minutes - for t in `seq 90`;do + # we wait for 30 minutes + for t in `seq 180`;do # now we have a working cac server if cac ssh $1 -o ConnectTimeout=10 \ cat /etc/redhat-release | \ @@ -82,6 +89,7 @@ while true;do echo "unable to boot a working system within time frame, retrying..." >&2 echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)" eval "$(clear_defer | sed 's/;exit//')" + sleep 15 else echo "got a working system" >&2 break -- cgit v1.2.3