From 14fd045f5bab7252d3fa9d4c909b471312b5399d Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 11 Jun 2016 00:26:21 +0200 Subject: l 2 buildbot-standalone: add build-all target --- lass/2configs/buildbot-standalone.nix | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) (limited to 'lass/2configs/buildbot-standalone.nix') diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index 604d0728d..d1be464f3 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -29,16 +29,26 @@ name="fast-all-branches", builderNames=["fast-tests"])) ''; + build-all-scheduler = '' + # build all lass hosts + sched.append(schedulers.SingleBranchScheduler( + ## only master + change_filter=util.ChangeFilter(branch_re="master"), + # treeStableTimer=10, + name="prism-master", + builderNames=["build-all"])) + ''; }; builder_pre = '' # prepare grab_repo step for stockholm grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') - env = {"LOGNAME": "lass", "NIX_REMOTE": "daemon"} + # TODO: get nixpkgs/stockholm paths from krebs + env = {"LOGNAME": "lass", "NIX_REMOTE": "daemon", "dummy_secrets": "true", "NIX_PATH": "nixpkgs=/var/src/nixpkgs:stockholm=/var/src/stockholm"} # prepare nix-shell # the dependencies which are used by the test script - deps = [ "gnumake", "jq","nix","rsync" ] + deps = [ "gnumake", "jq", "nix", "rsync" ] # TODO: --pure , prepare ENV in nix-shell command: # SSL_CERT_FILE,LOGNAME,NIX_REMOTE nixshell = ["nix-shell", @@ -51,6 +61,24 @@ factory.addStep(steps.ShellCommand(**kwargs)) ''; builder = { + build-all = '' + f = util.BuildFactory() + f.addStep(grab_repo) + #TODO: get hosts via krebs + for i in [ "mors", "uriel", "shodan", "helios", "cloudkrebs", "echelon", "dishfire", "prism" ]: + addShell(f,name="build-{}".format(i),env=env, + command=nixshell + \ + ["nix-build \ + --show-trace --no-out-link \ + -I nixos-config=./lass/1systems/{}.nix \ + -I secrets=/var/src/stockholm/lass/2configs/tests/dummy-secrets \ + -A config.system.build.toplevel".format(i)]) + + bu.append(util.BuilderConfig(name="build-all", + slavenames=slavenames, + factory=f)) + + ''; fast-tests = '' f = util.BuildFactory() f.addStep(grab_repo) @@ -93,7 +121,7 @@ password = "lasspass"; packages = with pkgs;[ git nix gnumake jq rsync ]; extraEnviron = { - NIX_PATH="nixpkgs=/var/src/nixpkgs:nixos-config=./shared/1systems/wolf.nix"; + NIX_PATH="nixpkgs=/var/src/nixpkgs"; }; }; krebs.iptables = { -- cgit v1.2.3 From a78f3e3ebe7a991caf6c3718929c360891871c37 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 11 Jun 2016 10:55:40 +0200 Subject: l 2 buildbot-standalone: use git dummy-secrets --- lass/2configs/buildbot-standalone.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lass/2configs/buildbot-standalone.nix') diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index d1be464f3..151ce562b 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -44,7 +44,7 @@ grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') # TODO: get nixpkgs/stockholm paths from krebs - env = {"LOGNAME": "lass", "NIX_REMOTE": "daemon", "dummy_secrets": "true", "NIX_PATH": "nixpkgs=/var/src/nixpkgs:stockholm=/var/src/stockholm"} + env = {"LOGNAME": "lass", "NIX_REMOTE": "daemon", "dummy_secrets": "true"} # prepare nix-shell # the dependencies which are used by the test script @@ -71,7 +71,8 @@ ["nix-build \ --show-trace --no-out-link \ -I nixos-config=./lass/1systems/{}.nix \ - -I secrets=/var/src/stockholm/lass/2configs/tests/dummy-secrets \ + -I secrets=./lass/2configs/tests/dummy-secrets \ + -I stockholm=. \ -A config.system.build.toplevel".format(i)]) bu.append(util.BuilderConfig(name="build-all", -- cgit v1.2.3 From 0f3df5c0f417be6011c833203f9f4fbd346aa64b Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 14 Jun 2016 13:26:17 +0200 Subject: l 2 buildbot-standalone: add treeStableTimer --- lass/2configs/buildbot-standalone.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lass/2configs/buildbot-standalone.nix') diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index 151ce562b..b70cf9da8 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -25,7 +25,7 @@ sched.append(schedulers.SingleBranchScheduler( ## all branches change_filter=util.ChangeFilter(branch_re=".*"), - # treeStableTimer=10, + treeStableTimer=10, name="fast-all-branches", builderNames=["fast-tests"])) ''; @@ -34,7 +34,7 @@ sched.append(schedulers.SingleBranchScheduler( ## only master change_filter=util.ChangeFilter(branch_re="master"), - # treeStableTimer=10, + treeStableTimer=10, name="prism-master", builderNames=["build-all"])) ''; -- cgit v1.2.3 From 30a80d67df7a41baab1fc3557d220c0cbbd5cf0c Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 18 Jun 2016 13:26:22 +0200 Subject: l 2 buildbot-standalone: use make test --- lass/2configs/buildbot-standalone.nix | 60 ++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 12 deletions(-) (limited to 'lass/2configs/buildbot-standalone.nix') diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index b70cf9da8..e36008335 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -1,6 +1,14 @@ { lib, config, pkgs, ... }: -{ - krebs.buildbot.master = let + +with config.krebs.lib; + +let + sshWrapper = pkgs.writeDash "ssh-wrapper" '' + ${pkgs.openssh}/bin/ssh -i ${shell.escape config.lass.build-ssh-privkey.path} "$@" + ''; + +in { + config.krebs.buildbot.master = let stockholm-mirror-url = http://cgit.prism/stockholm ; in { slaves = { @@ -44,11 +52,15 @@ grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') # TODO: get nixpkgs/stockholm paths from krebs - env = {"LOGNAME": "lass", "NIX_REMOTE": "daemon", "dummy_secrets": "true"} + env = { + "LOGNAME": "lass", + "NIX_REMOTE": "daemon", + "dummy_secrets": "true", + } # prepare nix-shell # the dependencies which are used by the test script - deps = [ "gnumake", "jq", "nix", "rsync" ] + deps = [ "gnumake", "jq", "nix", "rsync", "proot" ] # TODO: --pure , prepare ENV in nix-shell command: # SSL_CERT_FILE,LOGNAME,NIX_REMOTE nixshell = ["nix-shell", @@ -68,12 +80,11 @@ for i in [ "mors", "uriel", "shodan", "helios", "cloudkrebs", "echelon", "dishfire", "prism" ]: addShell(f,name="build-{}".format(i),env=env, command=nixshell + \ - ["nix-build \ - --show-trace --no-out-link \ - -I nixos-config=./lass/1systems/{}.nix \ - -I secrets=./lass/2configs/tests/dummy-secrets \ - -I stockholm=. \ - -A config.system.build.toplevel".format(i)]) + ["make \ + test \ + ssh=${sshWrapper} \ + target=build@localhost:${config.users.users.build.home}/testbuild \ + system={}".format(i)]) bu.append(util.BuilderConfig(name="build-all", slavenames=slavenames, @@ -115,7 +126,7 @@ }; }; - krebs.buildbot.slave = { + config.krebs.buildbot.slave = { enable = true; masterhost = "localhost"; username = "testslave"; @@ -125,7 +136,7 @@ NIX_PATH="nixpkgs=/var/src/nixpkgs"; }; }; - krebs.iptables = { + config.krebs.iptables = { tables = { filter.INPUT.rules = [ { predicate = "-p tcp --dport 8010"; target = "ACCEPT"; } @@ -133,4 +144,29 @@ ]; }; }; + + #ssh workaround for make test + options.lass.build-ssh-privkey = mkOption { + type = types.secret-file; + default = { + path = "${config.users.users.buildbotSlave.home}/ssh.privkey"; + owner = { inherit (config.users.users.buildbotSlave ) name uid;}; + source-path = toString + "/build.ssh.key"; + }; + }; + config.krebs.secret.files = { + build-ssh-privkey = config.lass.build-ssh-privkey; + }; + config.users.users = { + build = { + name = "build"; + uid = genid "build"; + home = "/home/build"; + useDefaultShell = true; + createHome = true; + openssh.authorizedKeys.keys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDiV0Xn60aVLHC/jGJknlrcxSvKd/MVeh2tjBpxSBT3II9XQGZhID2Gdh84eAtoWyxGVFQx96zCHSuc7tfE2YP2LhXnwaxHTeDc8nlMsdww53lRkxihZIEV7QHc/3LRcFMkFyxdszeUfhWz8PbJGL2GYT+s6CqoPwwa68zF33U1wrMOAPsf/NdpSN4alsqmjFc2STBjnOd9dXNQn1VEJQqGLG3kR3WkCuwMcTLS5eu0KLwG4i89Twjy+TGp2QsF5K6pNE+ZepwaycRgfYzGcPTn5d6YQXBgcKgHMoSJsK8wqpr0+eFPCDiEA3HDnf76E4mX4t6/9QkMXCLmvs0IO/WP lass@mors" + ]; + }; + }; } -- cgit v1.2.3 From ae3a2a954d767127ae1c144685293c03a01cf1c8 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 18 Jun 2016 19:20:26 +0200 Subject: l 2 buildbot: build-all with method=build --- lass/2configs/buildbot-standalone.nix | 1 + 1 file changed, 1 insertion(+) (limited to 'lass/2configs/buildbot-standalone.nix') diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index e36008335..7d3c236a6 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -84,6 +84,7 @@ in { test \ ssh=${sshWrapper} \ target=build@localhost:${config.users.users.build.home}/testbuild \ + method=build \ system={}".format(i)]) bu.append(util.BuilderConfig(name="build-all", -- cgit v1.2.3 From 3d1f3e697db96491adbe49fc516489fab81f6617 Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 23 Jun 2016 17:48:42 +0200 Subject: l 2 buildbot-standalone: build-all -> build-lass --- lass/2configs/buildbot-standalone.nix | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'lass/2configs/buildbot-standalone.nix') diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index 7d3c236a6..8fe2b4074 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -37,14 +37,14 @@ in { name="fast-all-branches", builderNames=["fast-tests"])) ''; - build-all-scheduler = '' + build-lass-scheduler = '' # build all lass hosts sched.append(schedulers.SingleBranchScheduler( ## only master change_filter=util.ChangeFilter(branch_re="master"), treeStableTimer=10, name="prism-master", - builderNames=["build-all"])) + builderNames=["build-lass"])) ''; }; builder_pre = '' @@ -52,7 +52,7 @@ in { grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') # TODO: get nixpkgs/stockholm paths from krebs - env = { + env_lass = { "LOGNAME": "lass", "NIX_REMOTE": "daemon", "dummy_secrets": "true", @@ -73,12 +73,12 @@ in { factory.addStep(steps.ShellCommand(**kwargs)) ''; builder = { - build-all = '' + build-lass = '' f = util.BuildFactory() f.addStep(grab_repo) #TODO: get hosts via krebs for i in [ "mors", "uriel", "shodan", "helios", "cloudkrebs", "echelon", "dishfire", "prism" ]: - addShell(f,name="build-{}".format(i),env=env, + addShell(f,name="build-{}".format(i),env=env_lass, command=nixshell + \ ["make \ test \ @@ -87,7 +87,7 @@ in { method=build \ system={}".format(i)]) - bu.append(util.BuilderConfig(name="build-all", + bu.append(util.BuilderConfig(name="build-lass", slavenames=slavenames, factory=f)) @@ -96,12 +96,12 @@ in { f = util.BuildFactory() f.addStep(grab_repo) for i in [ "prism", "mors", "echelon" ]: - addShell(f,name="populate-{}".format(i),env=env, + addShell(f,name="populate-{}".format(i),env=env_lass, command=nixshell + \ ["{}( make system={} eval.config.krebs.build.populate \ | jq -er .)".format("!" if "failing" in i else "",i)]) - addShell(f,name="build-test-minimal",env=env, + addShell(f,name="build-test-minimal",env=env_lass, command=nixshell + \ ["nix-instantiate \ --show-trace --eval --strict --json \ -- cgit v1.2.3 From fa3acf32fc2e4c72a13b59f167d75a6c5728a7bd Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 25 Jun 2016 09:55:36 +0200 Subject: l 2 buildbot: build-lass from /newest --- lass/2configs/buildbot-standalone.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lass/2configs/buildbot-standalone.nix') diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index 8fe2b4074..66e7e47b0 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -40,8 +40,7 @@ in { build-lass-scheduler = '' # build all lass hosts sched.append(schedulers.SingleBranchScheduler( - ## only master - change_filter=util.ChangeFilter(branch_re="master"), + change_filter=util.ChangeFilter(branch_re="newest"), treeStableTimer=10, name="prism-master", builderNames=["build-lass"])) -- cgit v1.2.3 From 28b539875032effd3770bb97355ee9d2177dd1aa Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 25 Jun 2016 09:58:00 +0200 Subject: l 2 buildbot: add makefu tests --- lass/2configs/buildbot-standalone.nix | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'lass/2configs/buildbot-standalone.nix') diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index 66e7e47b0..fefe01e0a 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -37,6 +37,14 @@ in { name="fast-all-branches", builderNames=["fast-tests"])) ''; + build-makefu-scheduler = '' + # build makefu hosts + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch_re="newest"), + treeStableTimer=10, + name="prism-newest", + builderNames=["build-makefu"])) + ''; build-lass-scheduler = '' # build all lass hosts sched.append(schedulers.SingleBranchScheduler( @@ -56,6 +64,11 @@ in { "NIX_REMOTE": "daemon", "dummy_secrets": "true", } + env_makefu = { + "LOGNAME": "makefu", + "NIX_REMOTE": "daemon", + "dummy_secrets": "true", + } # prepare nix-shell # the dependencies which are used by the test script @@ -90,6 +103,25 @@ in { slavenames=slavenames, factory=f)) + ''; + build-makefu = '' + f = util.BuildFactory() + f.addStep(grab_repo) + #TODO: get hosts via krebs + for i in [ "pornocauster", "wry" ]: + addShell(f,name="build-{}".format(i),env=env_makefu, + command=nixshell + \ + ["make \ + test \ + ssh=${sshWrapper} \ + target=build@localhost:${config.users.users.build.home}/testbuild \ + method=build \ + system={}".format(i)]) + + bu.append(util.BuilderConfig(name="build-makefu", + slavenames=slavenames, + factory=f)) + ''; fast-tests = '' f = util.BuildFactory() -- cgit v1.2.3 From 43a8f0977526d48057d0da64e26ff9ae5a0a11a4 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 25 Jun 2016 10:39:35 +0200 Subject: l 2 buildbot: build on all branches --- lass/2configs/buildbot-standalone.nix | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) (limited to 'lass/2configs/buildbot-standalone.nix') diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index fefe01e0a..46d50faac 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -37,21 +37,13 @@ in { name="fast-all-branches", builderNames=["fast-tests"])) ''; - build-makefu-scheduler = '' - # build makefu hosts + build-scheduler = '' + # build all hosts sched.append(schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch_re="newest"), - treeStableTimer=10, - name="prism-newest", - builderNames=["build-makefu"])) - ''; - build-lass-scheduler = '' - # build all lass hosts - sched.append(schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch_re="newest"), + change_filter=util.ChangeFilter(branch_re=".*"), treeStableTimer=10, - name="prism-master", - builderNames=["build-lass"])) + name="prism-all-branches", + builderNames=["build-lass", "build-makefu"])) ''; }; builder_pre = '' -- cgit v1.2.3 From 7089eea04132c062fb9e3b4fe53f09542238e6e9 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 25 Jun 2016 11:01:56 +0200 Subject: l 2 buildbot: merge build-lass and build-makefu --- lass/2configs/buildbot-standalone.nix | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) (limited to 'lass/2configs/buildbot-standalone.nix') diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index 46d50faac..04bdcf9d8 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -43,7 +43,7 @@ in { change_filter=util.ChangeFilter(branch_re=".*"), treeStableTimer=10, name="prism-all-branches", - builderNames=["build-lass", "build-makefu"])) + builderNames=["build-all"])) ''; }; builder_pre = '' @@ -77,10 +77,9 @@ in { factory.addStep(steps.ShellCommand(**kwargs)) ''; builder = { - build-lass = '' + build-all = '' f = util.BuildFactory() f.addStep(grab_repo) - #TODO: get hosts via krebs for i in [ "mors", "uriel", "shodan", "helios", "cloudkrebs", "echelon", "dishfire", "prism" ]: addShell(f,name="build-{}".format(i),env=env_lass, command=nixshell + \ @@ -91,15 +90,6 @@ in { method=build \ system={}".format(i)]) - bu.append(util.BuilderConfig(name="build-lass", - slavenames=slavenames, - factory=f)) - - ''; - build-makefu = '' - f = util.BuildFactory() - f.addStep(grab_repo) - #TODO: get hosts via krebs for i in [ "pornocauster", "wry" ]: addShell(f,name="build-{}".format(i),env=env_makefu, command=nixshell + \ @@ -110,11 +100,12 @@ in { method=build \ system={}".format(i)]) - bu.append(util.BuilderConfig(name="build-makefu", + bu.append(util.BuilderConfig(name="build-all", slavenames=slavenames, factory=f)) ''; + fast-tests = '' f = util.BuildFactory() f.addStep(grab_repo) -- cgit v1.2.3