diff options
36 files changed, 287 insertions, 313 deletions
diff --git a/krebs/1systems/hotdog/config.nix b/krebs/1systems/hotdog/config.nix index 662e094d1..18b751a7e 100644 --- a/krebs/1systems/hotdog/config.nix +++ b/krebs/1systems/hotdog/config.nix @@ -9,17 +9,17 @@ <stockholm/krebs> <stockholm/krebs/2configs> - <stockholm/krebs/2configs/buildbot-all.nix> + <stockholm/krebs/2configs/buildbot-stockholm.nix> <stockholm/krebs/2configs/gitlab-runner-shackspace.nix> <stockholm/krebs/2configs/binary-cache/nixos.nix> <stockholm/krebs/2configs/ircd.nix> <stockholm/krebs/2configs/reaktor-retiolum.nix> + <stockholm/krebs/2configs/reaktor-krebs.nix> ]; krebs.build.host = config.krebs.hosts.hotdog; boot.isContainer = true; networking.useDHCP = false; - krebs.ci.stockholmSrc = "http://cgit.prism.r/stockholm"; environment.variables.NIX_REMOTE = "daemon"; } diff --git a/krebs/2configs/buildbot-all.nix b/krebs/2configs/buildbot-all.nix deleted file mode 100644 index d85cde175..000000000 --- a/krebs/2configs/buildbot-all.nix +++ /dev/null @@ -1,9 +0,0 @@ -with import <stockholm/lib>; -{ lib, config, pkgs, ... }: -{ - networking.firewall.allowedTCPPorts = [ 80 8010 9989 ]; - krebs.ci.enable = true; - krebs.ci.treeStableTimer = 1; - krebs.ci.hosts = filter (getAttr "ci") (attrValues config.krebs.hosts); -} - diff --git a/krebs/2configs/buildbot-krebs.nix b/krebs/2configs/buildbot-krebs.nix deleted file mode 100644 index a09b3b98b..000000000 --- a/krebs/2configs/buildbot-krebs.nix +++ /dev/null @@ -1,12 +0,0 @@ -with import <stockholm/lib>; -{ lib, config, pkgs, ... }: -{ - imports = [ - <stockholm/krebs/2configs/repo-sync.nix> - ]; - - networking.firewall.allowedTCPPorts = [ 80 8010 9989 ]; - krebs.ci.enable = true; - krebs.ci.treeStableTimer = 120; - krebs.ci.hosts = [ config.krebs.build.host ]; -} diff --git a/krebs/2configs/buildbot-stockholm.nix b/krebs/2configs/buildbot-stockholm.nix new file mode 100644 index 000000000..04b1c999f --- /dev/null +++ b/krebs/2configs/buildbot-stockholm.nix @@ -0,0 +1,178 @@ +{ config, pkgs, ... }: with import <stockholm/lib>; + +let + + hostname = config.networking.hostName; + +in +{ + networking.firewall.allowedTCPPorts = [ 80 ]; + services.nginx = { + enable = true; + virtualHosts.build = { + serverAliases = [ "build.${hostname}.r" ]; + locations."/".extraConfig = '' + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port}; + ''; + }; + }; + + krebs.buildbot.master = { + slaves = { + testslave = "lasspass"; + }; + change_source.stockholm = '' + stockholm_repo = 'http://cgit.prism.r/stockholm' + cs.append( + changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branches=True, + project='stockholm', + pollinterval=10 + ) + ) + ''; + scheduler = { + auto-scheduler = '' + sched.append( + schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch_re=".*"), + treeStableTimer=60, + name="build-all-branches", + builderNames=[ + "hosts", + ] + ) + ) + ''; + force-scheduler = '' + sched.append( + schedulers.ForceScheduler( + name="hosts", + builderNames=[ + "hosts", + ] + ) + ) + ''; + }; + builder_pre = '' + # prepare grab_repo step for stockholm + grab_repo = steps.Git( + repourl=stockholm_repo, + mode='full', + ) + ''; + builder = { + hosts = '' + from buildbot import interfaces + from buildbot.steps.shell import ShellCommand + + class StepToStartMoreSteps(ShellCommand): + def __init__(self, **kwargs): + ShellCommand.__init__(self, **kwargs) + + def addBuildSteps(self, steps_factories): + for sf in steps_factories: + step = interfaces.IBuildStepFactory(sf).buildStep() + step.setBuild(self.build) + step.setBuildSlave(self.build.slavebuilder.slave) + step_status = self.build.build_status.addStepWithName(step.name) + step.setStepStatus(step_status) + self.build.steps.append(step) + + def start(self): + props = self.build.getProperties() + hosts = json.loads(props.getProperty('hosts_json')) + for host in hosts: + user = hosts[host]['owner'] + + self.addBuildSteps([steps.ShellCommand( + name=str(host), + env={ + "NIX_PATH": "secrets=/var/src/stockholm/null:stockholm=./:/var/src", + "NIX_REMOTE": "daemon", + "dummy_secrets": "true", + }, + command=[ + "nix-shell", "-I", "stockholm=.", "--run", " ".join(["test", + "--user={}".format(user), + "--system={}".format(host), + "--force-populate", + "--target=$LOGNAME@${config.krebs.build.host.name}$HOME/{}".format(user), + ]) + ], + timeout=90001, + workdir='build', # TODO figure out why we need this? + )]) + + ShellCommand.start(self) + + + f = util.BuildFactory() + f.addStep(grab_repo) + + f.addStep(steps.SetPropertyFromCommand( + env={ + "NIX_PATH": "secrets=/var/src/stockholm/null:stockholm=./:/var/src", + "NIX_REMOTE": "daemon", + }, + name="get_hosts", + command=["nix-instantiate", "--json", "--strict", "--eval", "-E", """ + with import <nixpkgs> {}; + let + eval-config = cfg: + import <nixpkgs/nixos/lib/eval-config.nix> { + modules = [ + (import cfg) + ]; + } + ; + + system = eval-config ./krebs/1systems/hotdog/config.nix; # TODO put a better config here + + ci-systems = lib.filterAttrs (_: v: v.ci) system.config.krebs.hosts; + + filtered-attrs = lib.mapAttrs ( n: v: { + owner = v.owner.name; + }) ci-systems; + + in filtered-attrs + """], + property="hosts_json" + )) + f.addStep(StepToStartMoreSteps(command=["echo"])) # TODO remove dummy command from here + + bu.append( + util.BuilderConfig( + name="hosts", + slavenames=slavenames, + factory=f + ) + ) + ''; + }; + enable = true; + web.enable = true; + irc = { + enable = true; + nick = "build|${hostname}"; + server = "irc.r"; + channels = [ "noise" "xxx" ]; + allowForce = true; + }; + extraConfig = '' + c['buildbotURL'] = "http://build.${hostname}.r/" + ''; + }; + + krebs.buildbot.slave = { + enable = true; + masterhost = "localhost"; + username = "testslave"; + password = "lasspass"; + packages = with pkgs; [ gnumake jq nix populate ]; + }; +} diff --git a/krebs/2configs/default.nix b/krebs/2configs/default.nix index 90aaa254a..7b970923d 100644 --- a/krebs/2configs/default.nix +++ b/krebs/2configs/default.nix @@ -50,6 +50,7 @@ with import <stockholm/lib>; users.extraUsers.root.openssh.authorizedKeys.keys = [ # TODO config.krebs.users.lass.pubkey + config.krebs.users.lass-mors.pubkey config.krebs.users.makefu.pubkey # TODO HARDER: config.krebs.users.makefu-omo.pubkey diff --git a/krebs/2configs/news-spam.nix b/krebs/2configs/news-spam.nix index 69c503bf9..a3f39b40e 100644 --- a/krebs/2configs/news-spam.nix +++ b/krebs/2configs/news-spam.nix @@ -94,7 +94,7 @@ [SPAM]npr_world|http://www.npr.org/rss/rss.php?id=1004|#snews [SPAM]nsa|https://www.nsa.gov/rss.xml|#snews #bullerei [SPAM]nytimes|http://rss.nytimes.com/services/xml/rss/nyt/World.xml|#snews - [SPAM]painload|https://github.com/krebscode/painload/commits/master.atom|#snews + [SPAM]painload|https://github.com/krebs/painload/commits/master.atom|#snews [SPAM]phys|http://phys.org/rss-feed/|#snews [SPAM]piraten|https://www.piratenpartei.de/feed/|#snews [SPAM]polizei_berlin|http://www.berlin.de/polizei/presse-fahndung/_rss_presse.xml|#snews diff --git a/krebs/2configs/news.nix b/krebs/2configs/news.nix index 49a5e3459..6c59f4d84 100644 --- a/krebs/2configs/news.nix +++ b/krebs/2configs/news.nix @@ -8,7 +8,7 @@ ethereum|http://blog.ethereum.org/feed|#news LtU|http://lambda-the-ultimate.org/rss.xml|#news mongrel2_master|https://github.com/zedshaw/mongrel2/commits/master.atom|#news - painload|https://github.com/krebscode/painload/commits/master.atom|#news + painload|https://github.com/krebs/painload/commits/master.atom|#news reddit_haskell|http://www.reddit.com/r/haskell/.rss|#news reddit_nix|http://www.reddit.com/r/nixos/.rss|#news shackspace|http://shackspace.de/atom.xml|#news @@ -16,7 +16,7 @@ vimperator|https://sites.google.com/a/vimperator.org/www/blog/posts.xml|#news weechat|http://dev.weechat.org/feed/atom|#news xkcd|https://xkcd.com/rss.xml|#news - painload|https://github.com/krebscode/painload/commits/master.atom|#news + painload|https://github.com/krebs/painload/commits/master.atom|#news ''; }; } diff --git a/krebs/2configs/reaktor-krebs.nix b/krebs/2configs/reaktor-krebs.nix index 6b17b457d..fa51b84f0 100644 --- a/krebs/2configs/reaktor-krebs.nix +++ b/krebs/2configs/reaktor-krebs.nix @@ -13,13 +13,8 @@ with import <stockholm/lib>; }; plugins = with pkgs.ReaktorPlugins; [ sed-plugin - wiki-todo-add - wiki-todo-done - wiki-todo-show - ]; + ] ++ + (attrValues (todo "agenda")) + ; }; - services.nginx.virtualHosts."lassul.us".locations."/wiki-todo".extraConfig = '' - default_type "text/plain"; - alias /var/lib/Reaktor/state/wiki-todo; - ''; } diff --git a/krebs/2configs/reaktor-retiolum.nix b/krebs/2configs/reaktor-retiolum.nix index 144b7d484..b32d39b7e 100644 --- a/krebs/2configs/reaktor-retiolum.nix +++ b/krebs/2configs/reaktor-retiolum.nix @@ -10,6 +10,8 @@ with import <stockholm/lib>; }; plugins = with pkgs.ReaktorPlugins; [ sed-plugin - ]; + ] ++ + (attrValues (todo "agenda")) + ; }; } diff --git a/krebs/3modules/ci.nix b/krebs/3modules/ci.nix deleted file mode 100644 index e97aa16eb..000000000 --- a/krebs/3modules/ci.nix +++ /dev/null @@ -1,192 +0,0 @@ -{ config, pkgs, ... }: -with import <stockholm/lib>; -let - cfg = config.krebs.ci; - - hostname = config.networking.hostName; -in -{ - options.krebs.ci = { - enable = mkEnableOption "krebs continous integration"; - stockholmSrc = mkOption { - type = types.str; - default = "http://cgit.${hostname}.r/stockholm"; - }; - treeStableTimer = mkOption { - type = types.int; - default = 10; - description = "how long to wait until we test changes (in minutes)"; - }; - hosts = mkOption { - type = types.listOf types.host; - default = []; - description = '' - List of hosts that should be build - ''; - }; - tests = mkOption { - type = types.listOf types.str; - default = []; - description = '' - List of tests that should be build - ''; - }; - }; - - config = mkIf cfg.enable { - services.nginx = { - enable = true; - virtualHosts.build = { - serverAliases = [ "build.${hostname}.r" ]; - locations."/".extraConfig = '' - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port}; - ''; - }; - }; - - krebs.buildbot.master = { - slaves = { - testslave = "lasspass"; - }; - change_source.stockholm = '' - stockholm_repo = '${cfg.stockholmSrc}' - cs.append( - changes.GitPoller( - stockholm_repo, - workdir='stockholm-poller', branches=True, - project='stockholm', - pollinterval=10 - ) - ) - ''; - scheduler = { - build-scheduler = '' - sched.append( - schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch_re=".*"), - treeStableTimer=${toString cfg.treeStableTimer}*60, - name="build-all-branches", - builderNames=[ - ${optionalString (cfg.hosts != []) ''"hosts",''} - ${optionalString (cfg.tests != []) ''"tests",''} - ] - ) - ) - ''; - force-scheduler = '' - sched.append( - schedulers.ForceScheduler( - name="force", - builderNames=[ - ${optionalString (cfg.hosts != []) ''"hosts",''} - ${optionalString (cfg.tests != []) ''"tests",''} - ] - ) - ) - ''; - }; - builder_pre = '' - # prepare grab_repo step for stockholm - grab_repo = steps.Git( - repourl=stockholm_repo, - mode='full' - ) - - # prepare addShell function - def addShell(factory,**kwargs): - factory.addStep(steps.ShellCommand(**kwargs)) - ''; - builder = { - hosts = mkIf (cfg.hosts != []) '' - f = util.BuildFactory() - f.addStep(grab_repo) - - def build_host(user, host): - addShell(f, - name="{}".format(host), - env={ - "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src", - "NIX_REMOTE": "daemon", - "dummy_secrets": "true", - }, - command=[ - "nix-shell", "-I", "stockholm=.", "--run", " ".join(["test", - "--user={}".format(user), - "--system={}".format(host), - "--force-populate", - "--target=$LOGNAME@${config.krebs.build.host.name}$HOME/{}".format(user), - ]) - ], - timeout=90001 - ) - - ${concatMapStringsSep "\n" (host: - "build_host(\"${host.owner.name}\", \"${host.name}\")" - ) cfg.hosts} - - bu.append( - util.BuilderConfig( - name="hosts", - slavenames=slavenames, - factory=f - ) - ) - ''; - tests = mkIf (cfg.tests != []) '' - f = util.BuildFactory() - f.addStep(grab_repo) - - def run_test(test): - addShell(f, - name="{}".format(test), - env={ - "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src", - "NIX_REMOTE": "daemon", - "dummy_secrets": "true", - }, - command=[ - "nix-build", "-I", "stockholm=.", "krebs/0tests", - "-A", "{}".format(test) - ], - timeout=90001 - ) - - ${concatMapStringsSep "\n" (test: - "run_test(\"${test}\")" - ) cfg.tests} - - bu.append( - util.BuilderConfig( - name="tests", - slavenames=slavenames, - factory=f - ) - ) - ''; - }; - enable = true; - web.enable = true; - irc = { - enable = true; - nick = "build|${hostname}"; - server = "irc.r"; - channels = [ "xxx" "noise" ]; - allowForce = true; - }; - extraConfig = '' - c['buildbotURL'] = "http://build.${hostname}.r/" - ''; - }; - - krebs.buildbot.slave = { - enable = true; - masterhost = "localhost"; - username = "testslave"; - password = "lasspass"; - packages = with pkgs; [ gnumake jq nix populate ]; - }; - - }; -} diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index 6fe01e466..afc96e9ee 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -13,7 +13,6 @@ let ./buildbot/master.nix ./buildbot/slave.nix ./build.nix - ./ci.nix ./current.nix ./exim.nix ./exim-retiolum.nix diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix index 8dbdc7e65..71670d336 100644 --- a/krebs/3modules/tv/default.nix +++ b/krebs/3modules/tv/default.nix @@ -32,30 +32,6 @@ with import <stockholm/lib>; ssh.privkey.path = <secrets/ssh.id_rsa>; ssh.pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDP9JS2Nyjx4Pn+/4MrFi1EvBBYVKkGm2Q4lhgaAiSuiGLol53OSsL2KIo01mbcSSBWow9QpQpn8KDoRnT2aMLDrdTFqL20ztDLOXmtrSsz3flgCjmW4f6uOaoZF0RNjAybd1coqwSJ7EINugwoqOsg1zzN2qeIGKYFvqFIKibYFAnQ8hcksmkvPdIO5O8CbdIiP9sZSrSDp0ZyLK2T0PML2jensVZOeqSPulQDFqLsbmavpVLkpDjdzzPRwbZWNB4++YeipbYNOkX4GR1EB4wMZ93IbBV7kpJtib2Zb2AnUf7UW37hxWBjILdstj9ClwNOQggn8kD9ub7YxBzH1dz0Xd8a0mPOAWIDJz9MypXgFRc3vdvPB/W1I4Se0CLbgOkORun9CkgijKr9oEY8JNt8HFd6viZcAaQxOyIm6PNHZTnHfdSc7bIBS2n3e3IZBv0fTd77knGLXg402aTuu2bm/kxsKivxsILXIaGbeXe4ceN3Fynr3FzSM2bUkzHb0mAHu1BQ9YaX0xzCwjVueA5nzGls7ODSFkXsiBfg2FvMN/sTLFca6tnwyqcnD6nujoiS5+BxjDWPgnZYqCaW3B/IkpTsRMsX6QrfhOFcsP8qlJ2Cp82orWoDK/D0vZ9pdzAc6PFGga0RofuJKY2yiq+SRZ7/e9E6VncIVCYZ1OfN0Q=="; }; - kaepsele = { - external = true; - nets = { - retiolum = { - ip4.addr = "10.243.166.2"; - ip6.addr = "42:b9d:6660:d07c:2bb7:4e91:1a01:2e7d"; - aliases = [ - "kaepsele.r" - ]; - tinc.pubkey = '' - -----BEGIN RSA PUBLIC KEY----- - MIIBCgKCAQEA4+kDaKhCBNlpHqRCA2R6c4UEFk0OaiPwHvjmBBjpihTJVyffIEYm - QFZ5ZNkaVumSOAgKk9ygppO9WsNasl1ag+IRWik9oupdzEkNjgvOMBVJGhcwGZGF - 6UEY5sdA1n0qg74og5BGSiXUBiaahVM0rAfCNk8gV3qrot5kWJMQLb9BKabJ56eb - JrgWepxuVaw3BoEhz6uusuvw5i1IF382L8R11hlvyefifXONFOAUjCrCr0bCb4uK - ZZcRUU35pbHLDXXTOrOarOO1tuVGu85VXo3S1sLaaouHYjhTVT8bxqbwcNhxBXYf - ONLv0f7G5XwecgUNbE6ZTfjV5PQKaww3lwIDAQAB - -----END RSA PUBLIC KEY----- - ''; - }; - }; - ssh.privkey.path = <secrets/ssh.id_ed25519>; - ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC5Wr36T0MmB8pnSO5/pw9/Dfe5+IMgVHOhm6EUa55jj"; - }; mu = { ci = true; cores = 2; diff --git a/krebs/5pkgs/simple/Reaktor/plugins.nix b/krebs/5pkgs/simple/Reaktor/plugins.nix index f3b771190..c39e39799 100644 --- a/krebs/5pkgs/simple/Reaktor/plugins.nix +++ b/krebs/5pkgs/simple/Reaktor/plugins.nix @@ -141,25 +141,27 @@ rec { ''; }); - wiki-todo-add = buildSimpleReaktorPlugin "wiki-todo-add" { - pattern = "^wiki-todo: (?P<args>.*)$$"; - script = pkgs.writeDash "wiki-todo-add" '' - echo "$*" >> wiki-todo - echo "added todo. check on http://lassul.us/wiki-todo" - ''; - }; - wiki-todo-done = buildSimpleReaktorPlugin "wiki-todo-done" { - pattern = "^wiki-done: (?P<args>.*)$$"; - script = pkgs.writeDash "wiki-todo-done" '' - ${pkgs.gnugrep}/bin/grep -Fvxe "$*" wiki-todo > wiki-todo.tmp - ${pkgs.coreutils}/bin/mv wiki-todo.tmp wiki-todo - echo "thank you for resolving todo: $*" - ''; - }; - wiki-todo-show = buildSimpleReaktorPlugin "wiki-todo" { - pattern = "^wiki-show$"; - script = pkgs.writeDash "wiki-show" '' - ${pkgs.coreutils}/bin/cat wiki-todo - ''; + todo = name: { + add = buildSimpleReaktorPlugin "${name}-add" { + pattern = "^${name}-add: (?P<args>.*)$$"; + script = pkgs.writeDash "${name}-add" '' + echo "$*" >> ${name}-todo + echo "added ${name} todo" + ''; + }; + delete = buildSimpleReaktorPlugin "${name}-delete" { + pattern = "^${name}-delete: (?P<args>.*)$$"; + script = pkgs.writeDash "${name}-delete" '' + ${pkgs.gnugrep}/bin/grep -Fvxe "$*" ${name}-todo > ${name}-todo.tmp + ${pkgs.coreutils}/bin/mv ${name}-todo.tmp ${name}-todo + echo "removed ${name} todo: $*" + ''; + }; + show = buildSimpleReaktorPlugin "${name}-show" { + pattern = "^${name}-show$"; + script = pkgs.writeDash "${name}-show" '' + ${pkgs.coreutils}/bin/cat ${name}-todo + ''; + }; }; } diff --git a/krebs/5pkgs/simple/buildbot-classic/default.nix b/krebs/5pkgs/simple/buildbot-classic/default.nix index 3cb691f53..e6bafbd25 100644 --- a/krebs/5pkgs/simple/buildbot-classic/default.nix +++ b/krebs/5pkgs/simple/buildbot-classic/default.nix @@ -7,7 +7,7 @@ python2Packages.buildPythonApplication rec { patches = []; src = fetchFromGitHub { - owner = "krebscode"; + owner = "krebs"; repo = "buildbot-classic"; rev = version; sha256 = "0g686n6m0cjfyympl0ksansllx503gby3hx9gmc8hiyx6x5fkjha"; diff --git a/krebs/5pkgs/simple/ejabberd/default.nix b/krebs/5pkgs/simple/ejabberd/default.nix index 9e4ed3df5..b4ab13b43 100644 --- a/krebs/5pkgs/simple/ejabberd/default.nix +++ b/krebs/5pkgs/simple/ejabberd/default.nix @@ -1,6 +1,5 @@ { stdenv, writeScriptBin, lib, fetchurl, git, cacert , erlang, openssl, expat, libyaml, bash, gnused, gnugrep, coreutils, utillinux, procps, gd -, flock , withMysql ? false , withPgsql ? false , withSqlite ? false, sqlite @@ -106,7 +105,7 @@ in stdenv.mkDerivation rec { postInstall = '' sed -i \ -e '2iexport PATH=${ctlpath}:$PATH' \ - -e 's,\(^ *FLOCK=\).*,\1${flock}/bin/flock,' \ + -e 's,\(^ *FLOCK=\).*,\1${utillinux}/bin/flock,' \ -e 's,\(^ *JOT=\).*,\1,' \ -e 's,\(^ *CONNLOCKDIR=\).*,\1/var/lock/ejabberdctl,' \ $out/sbin/ejabberdctl |