Merge remote-tracking branch 'gum/master'

10 files changed, 254 insertions, 111 deletions

diff --git a/krebs/3modules/buildbot/buildbot-worker.patch b/krebs/3modules/buildbot/buildbot-worker.patch
new file mode 100644
index 000000000..df6f7ed37
--- /dev/null
+++ b/krebs/3modules/buildbot/buildbot-worker.patch
@@ -0,0 +1,11 @@
+--- ./buildbot_worker/scripts/logwatcher.py  2016-11-10 23:25:46.956000000 +0100
++++ ./buildbot_worker/scripts/logwatcher.py.fix  2016-11-10 23:24:33.225000000 +0100
+@@ -76,7 +76,7 @@
+         if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'):
+             tailBin = "/usr/xpg4/bin/tail"
+         else:
+-            tailBin = "/usr/bin/tail"
++            tailBin = "tail"
+         self.p = reactor.spawnProcess(self.pp, tailBin,
+                                       ("tail", "-f", "-n", "0", self.logfile),
+                                       env=os.environ,
diff --git a/krebs/3modules/buildbot/buildbot.patch b/krebs/3modules/buildbot/buildbot.patch
new file mode 100644
index 000000000..3a5794d82
--- /dev/null
+++ b/krebs/3modules/buildbot/buildbot.patch
@@ -0,0 +1,11 @@
+--- ./buildbot/scripts/logwatcher.py  2016-11-10 23:25:46.956000000 +0100
++++ ./buildbot/scripts/logwatcher.py.fix  2016-11-10 23:24:33.225000000 +0100
+@@ -76,7 +76,7 @@
+         if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'):
+             tailBin = "/usr/xpg4/bin/tail"
+         else:
+-            tailBin = "/usr/bin/tail"
++            tailBin = "tail"
+         self.p = reactor.spawnProcess(self.pp, tailBin,
+                                       ("tail", "-f", "-n", "0", self.logfile),
+                                       env=os.environ,
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 9e144ee0e..b31661572 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -3,14 +3,10 @@
 with import <stockholm/lib>;
 let
 
-  # https://github.com/NixOS/nixpkgs/issues/14026
-  nixpkgs-fix = import (pkgs.fetchgit {
-    url = https://github.com/nixos/nixpkgs;
-    rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
-    sha256 = "87e0724910a6df0371f883f99a8cf42e366fb4119f676f6f74ffb404beca2632";
-  }) {};
-
-  buildbot = nixpkgs-fix.buildbot;
+  buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-full (old:{
+    patches = [ ./buildbot.patch ];
+    propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ];
+  });
   buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
     # -*- python -*-
     from buildbot.plugins import *
@@ -18,11 +14,11 @@ let
     import json
     c = BuildmasterConfig = {}
 
-    c['slaves'] = []
-    slaves = json.loads('${builtins.toJSON cfg.slaves}')
-    slavenames = [ s for s in slaves ]
-    for k,v in slaves.items():
-      c['slaves'].append(buildslave.BuildSlave(k, v))
+    c['workers'] = []
+    workers = json.loads('${builtins.toJSON cfg.workers}')
+    workernames = [ s for s in workers ]
+    for k,v in workers.items():
+      c['workers'].append(worker.Worker(k, v))
 
     # TODO: configure protocols?
     c['protocols'] = {'pb': {'port': 9989}}
@@ -63,32 +59,46 @@ let
 
 
     ####### Status
-    c['status'] = st = []
+    c['services'] = []
 
     # If you want to configure this url, override with extraConfig
     c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
 
     ${optionalString (cfg.web.enable) ''
-      from buildbot.status import html
-      from buildbot.status.web import authz, auth
-      authz_cfg=authz.Authz(
-          auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
-          # TODO: configure harder
-          gracefulShutdown = False,
-          forceBuild = 'auth',
-          forceAllBuilds = 'auth',
-          pingBuilder = False,
-          stopBuild = 'auth',
-          stopAllBuilds = 'auth',
-          cancelPendingBuild = 'auth'
-      )
+      from buildbot.plugins import util
+
+      #authz_cfg=authz.Authz(
+      #    auth=auth.BasicAuth([  ]),
+      #    # TODO: configure harder
+      #    gracefulShutdown = False,
+      #    forceBuild = 'auth',
+      #    forceAllBuilds = 'auth',
+      #    pingBuilder = False,
+      #    stopBuild = 'auth',
+      #    stopAllBuilds = 'auth',
+      #    cancelPendingBuild = 'auth'
+      #)
       # TODO: configure krebs.nginx
-      st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
+      c['www'] = dict(
+        port = ${toString cfg.web.port},
+        plugins = { 'waterfall_view':{}, 'console_view':{} }
+      )
+      c['www']['auth'] = util.UserPasswordAuth({"${cfg.web.username}":"${cfg.web.password}"})
+      c['www']['authz'] = util.Authz(
+        allowRules = [
+          util.StopBuildEndpointMatcher(role="admins"),
+          util.ForceBuildEndpointMatcher(role="admins"),
+          util.RebuildBuildEndpointMatcher(role="admins")
+        ],
+        roleMatchers = [
+          util.RolesFromEmails(admins=["${cfg.web.username}"])
+        ]
+      )
       ''}
 
     ${optionalString (cfg.irc.enable) ''
-      from buildbot.status import words
-      irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
+      from buildbot.plugins import reporters
+      irc = reporters.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
                       channels=${builtins.toJSON cfg.irc.channels},
                       notify_events={
                         'success': 1,
@@ -97,7 +107,7 @@ let
                         'successToFailure': 1,
                         'failureToSuccess': 1,
                       }${optionalString cfg.irc.allowForce ",allowForce=True"})
-      c['status'].append(irc)
+      c['services'].append(irc)
       ''}
 
     ${ concatStringsSep "\n"
@@ -150,12 +160,12 @@ let
       '';
     };
 
-    slaves = mkOption {
+    workers = mkOption {
       default = {};
       type = types.attrsOf types.str;
       description = ''
-        Attrset of slavenames with their passwords
-        slavename = slavepassword
+        Attrset of workernames with their passwords
+        workername = workerpassword
       '';
     };
 
@@ -283,8 +293,12 @@ let
         options = {
           enable = mkEnableOption "Buildbot Master IRC Status";
           channels = mkOption {
-            default = [ "nix-buildbot-meetup" ];
-            type = with types; listOf str;
+            default = [ { channel = "nix-buildbot-meetup";} ];
+            example = literalExample ''[
+              {channel = "nix-buildbot-meetup";}
+              {channel = "nix-buildbot-lol"; "password" = "lol";}
+            ]'';
+            type = with types; listOf (attrsOf str);
             description = ''
               irc channels the bot should connect to
             '';
@@ -333,7 +347,7 @@ let
     };
 
     users.extraGroups.buildbotMaster = {
-      gid = 672626386;
+      gid = genid "buildbotMaster";
     };
 
     systemd.services.buildbotMaster = {
@@ -350,8 +364,6 @@ let
         secretsdir = shell.escape (toString <secrets>);
       in {
         PermissionsStartOnly = true;
-        Type = "forking";
-        PIDFile = "${workdir}/twistd.pid";
         # TODO: maybe also prepare buildbot.tac?
         ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
           set -efux
@@ -375,9 +387,8 @@ let
           chmod 700 -R ${workdir}
           chown buildbotMaster:buildbotMaster -R ${workdir}
         '';
-        ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
-        ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
-        ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
+        ExecStart = "${buildbot}/bin/buildbot start --nodaemon ${workdir}";
+        # ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
         PrivateTmp = "true";
         User = "buildbotMaster";
         Restart = "always";
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
index 650594a6c..95b547081 100644
--- a/krebs/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -2,53 +2,21 @@
 
 with import <stockholm/lib>;
 let
-  buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
-    import os
-
-    from buildslave.bot import BuildSlave
-    from twisted.application import service
-
-    basedir = '${cfg.workDir}'
-    rotateLength = 10000000
-    maxRotatedFiles = 10
-
-    application = service.Application('buildslave')
-
-    from twisted.python.logfile import LogFile
-    from twisted.python.log import ILogObserver, FileLogObserver
-    logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
-                                  maxRotatedFiles=maxRotatedFiles)
-    application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
-
-    buildmaster_host = '${cfg.masterhost}'
-    # TODO: masterport?
-    port = 9989
-    slavename = '${cfg.username}'
-    passwd = '${cfg.password}'
-    keepalive = 600
-    usepty = 0
-    umask = None
-    maxdelay = 300
-    allow_shutdown = None
-
-    ${cfg.extraConfig}
-
-    s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
-                  keepalive, usepty, umask=umask, maxdelay=maxdelay,
-                  allow_shutdown=allow_shutdown)
-    s.setServiceParent(application)
-    '';
   default-packages = [ pkgs.git pkgs.bash ];
-  cfg = config.krebs.buildbot.slave;
+  buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-worker (old:{
+    patches = [ ./buildbot-worker.patch ];
+    propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ];
+  });
+  cfg = config.krebs.buildbot.worker;
 
   api = {
-    enable = mkEnableOption "Buildbot Slave";
+    enable = mkEnableOption "Buildbot worker";
 
     workDir = mkOption {
-      default = "/var/lib/buildbot/slave";
+      default = "/var/lib/buildbot/worker";
       type = types.str;
       description = ''
-        Path to build bot slave directory.
+        Path to build bot worker directory.
         Will be created on startup.
       '';
     };
@@ -64,30 +32,30 @@ let
     username = mkOption {
       type = types.str;
       description = ''
-        slavename used to authenticate with master
+        workername used to authenticate with master
       '';
     };
 
     password = mkOption {
       type = types.str;
       description = ''
-        slave password used to authenticate with master
+        worker password used to authenticate with master
       '';
     };
 
     contact = mkOption {
-      default = "nix slave <buildslave@${config.networking.hostName}>";
+      default = "nix worker <buildworker@${config.networking.hostName}>";
       type = types.str;
       description = ''
-        contact to be announced by buildslave
+        contact to be announced by buildworker
       '';
     };
 
     description = mkOption {
-      default = "Nix Generated BuildSlave";
+      default = "Nix Generated Buildworker";
       type = types.str;
       description = ''
-        description for hostto be announced by buildslave
+        description for hostto be announced by buildworker
       '';
     };
 
@@ -95,7 +63,7 @@ let
       default = [ pkgs.git ];
       type = with types; listOf package;
       description = ''
-        packages which should be in path for buildslave
+        packages which should be in path for buildworker
       '';
     };
 
@@ -106,7 +74,7 @@ let
       };
       type = types.attrsOf types.str;
       description = ''
-        extra environment variables to be provided to the buildslave service
+        extra environment variables to be provided to the buildworker service
         if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
       '';
     };
@@ -119,26 +87,26 @@ let
         keepalive = 600
       '';
       description = ''
-        extra config evaluated before calling BuildSlave init in .tac file
+        extra config evaluated before calling Buildworker init in .tac file
       '';
     };
   };
 
   imp = {
 
-    users.extraUsers.buildbotSlave = {
-      uid = genid "buildbotSlave";
-      description = "Buildbot Slave";
+    users.extraUsers.buildbotworker = {
+      uid = genid "buildbotworker";
+      description = "Buildbot worker";
       home = cfg.workDir;
       createHome = false;
     };
 
-    users.extraGroups.buildbotSlave = {
-      gid = 1408105834;
+    users.extraGroups.buildbotworker = {
+      gid = genid "buildbotworker";
     };
 
-    systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
-      description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
+    systemd.services."buildbotworker-${cfg.username}-${cfg.masterhost}" = {
+      description = "Buildbot worker for ${cfg.username}@${cfg.masterhost}";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
       path = default-packages ++ cfg.packages;
@@ -152,27 +120,28 @@ let
         workdir = shell.escape cfg.workDir;
         contact = shell.escape cfg.contact;
         description = shell.escape cfg.description;
-        buildbot = pkgs.buildbot-slave;
-        # TODO:make this
+        masterhost = shell.escape cfg.masterhost;
+        username = shell.escape cfg.username;
+        password = shell.escape cfg.password;
       in {
         PermissionsStartOnly = true;
         Type = "forking";
         PIDFile = "${workdir}/twistd.pid";
-        # TODO: maybe also prepare buildbot.tac?
-        ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
+        ExecStartPre = pkgs.writeDash "buildbot-slave-init" ''
           set -efux
           mkdir -p ${workdir}/info
-          cp ${buildbot-slave-init} ${workdir}/buildbot.tac
+          # TODO: cleanup .tac file?
+          ${buildbot}/bin/buildbot-worker create-worker ${workdir} ${masterhost} ${username} ${password}
           echo ${contact} > ${workdir}/info/admin
           echo ${description} > ${workdir}/info/host
 
-          chown buildbotSlave:buildbotSlave -R ${workdir}
+          chown buildbotworker:buildbotworker -R ${workdir}
           chmod 700 -R ${workdir}
         '';
-        ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
-        ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
+        ExecStart = "${buildbot}/bin/buildbot-worker start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildbot-worker stop ${workdir}";
         PrivateTmp = "true";
-        User = "buildbotSlave";
+        User = "buildbotworker";
         Restart = "always";
         RestartSec = "10";
       };
@@ -180,6 +149,6 @@ let
   };
 in
 {
-  options.krebs.buildbot.slave = api;
+  options.krebs.buildbot.worker = api;
   config = lib.mkIf cfg.enable imp;
 }
diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix
index a483db32c..242373ced 100644
--- a/krebs/5pkgs/Reaktor/plugins.nix
+++ b/krebs/5pkgs/Reaktor/plugins.nix
@@ -59,7 +59,7 @@ rec {
   };
 
   sed-plugin = buildSimpleReaktorPlugin "sed-plugin" {
-    path = [ pkgs.gnused pkgs.python3 ];
+    path = [ pkgs.gnused pkgs.proot pkgs.python3 ];
     # only support s///gi the plugin needs to see every msg
     # TODO: this will eat up the last regex, fix Reaktor to support fallthru
     append_rule = true;
diff --git a/krebs/5pkgs/Reaktor/scripts/sed-plugin.py b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py
index 8103c9585..6039aeb43 100644
--- a/krebs/5pkgs/Reaktor/scripts/sed-plugin.py
+++ b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py
@@ -34,9 +34,22 @@ if m:
         flagstr = ''
     last = d.get(usr,None)
     if last:
-        #print(re.sub(fn,tn,last,count=count,flags=flags))
         from subprocess import Popen,PIPE
-        p = Popen(['sed','s/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE )
+        import shutil
+        from os.path import realpath
+        # sed only needs stdin/stdout, we protect state_dir with this
+        # input to read/write arbitrary files:
+        #   s/.\/\/; w /tmp/i       (props to waldi)
+        # conclusion: sed is untrusted and we handle it like this
+        p = Popen(['proot',
+            # '-v','1',
+            '-w','/', # cwd is root
+            '-b','/nix/store', # mount important folders
+            '-b','/usr',
+            '-b','/bin',
+            '-r','/var/empty', # chroot to /var/empty
+            realpath(shutil.which('sed')),
+            's/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE )
         so,se = p.communicate(bytes("{}\n".format(last),"UTF-8"))
         if p.returncode:
             print("something went wrong when trying to process your regex: {}".format(se.decode()))
diff --git a/krebs/5pkgs/buildbot/default.nix b/krebs/5pkgs/buildbot/default.nix
new file mode 100644
index 000000000..a0e6bb6a5
--- /dev/null
+++ b/krebs/5pkgs/buildbot/default.nix
@@ -0,0 +1,81 @@
+{ pkgs, stdenv, pythonPackages, fetchurl, coreutils, plugins ? [] }:
+
+pythonPackages.buildPythonApplication (rec {
+  name = "${pname}-${version}";
+  pname = "buildbot";
+  version = "0.9.1";
+  src = fetchurl {
+    url = "mirror://pypi/b/${pname}/${name}.tar.gz";
+    sha256 = "1kk4dlkk4rznwid9xykq2lbzksvkcr4r5kmz9hgh5hswdzv8bwx9";
+  };
+  doCheck = false;
+  buildInputs = with pythonPackages; [
+    lz4
+    txrequests
+    pyjade
+    boto3
+    moto
+    txgithub
+    mock
+    setuptoolsTrial
+    isort
+    pylint
+    astroid
+    pyflakes
+  ];
+
+  propagatedBuildInputs = with pythonPackages; [
+
+    # core
+    twisted
+    jinja2
+    zope_interface
+    future
+    sqlalchemy
+    sqlalchemy_migrate
+    future
+    dateutil
+    txaio
+    autobahn
+
+    # tls
+    pyopenssl
+    service-identity
+    idna
+    pkgs.treq
+
+    # docs
+    sphinx
+    sphinxcontrib-blockdiag
+    sphinxcontrib-spelling
+    pyenchant
+    docutils
+    ramlfications
+    sphinx-jinja
+
+  ] ++ plugins;
+
+  preInstall = ''
+    # writes out a file that can't be read properly
+    sed -i.bak -e '69,84d' buildbot/test/unit/test_www_config.py
+
+    # re-hardcode path to tail
+    sed -i.bak 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py
+  '';
+
+  postFixup = ''
+    mv -v $out/bin/buildbot $out/bin/.wrapped-buildbot
+    echo "#!/bin/sh" > $out/bin/buildbot
+    echo "export PYTHONPATH=$PYTHONPATH" >> $out/bin/buildbot
+    echo "exec $out/bin/.wrapped-buildbot \"\$@\"" >> $out/bin/buildbot
+    chmod -c 555 $out/bin/buildbot
+  '';
+
+  meta = with stdenv.lib; {
+    homepage = http://buildbot.net/;
+    description = "Continuous integration system that automates the build/test cycle";
+    maintainers = with maintainers; [ nand0p ryansydnor ];
+    platforms = platforms.all;
+    license = licenses.gpl2;
+  };
+})
diff --git a/krebs/5pkgs/buildbot/worker.nix b/krebs/5pkgs/buildbot/worker.nix
new file mode 100644
index 000000000..c100de5d2
--- /dev/null
+++ b/krebs/5pkgs/buildbot/worker.nix
@@ -0,0 +1,24 @@
+{ pkgs, stdenv, fetchurl, pythonPackages }:
+pythonPackages.buildPythonApplication (rec {
+  name = "${pname}-${version}";
+  pname = "buildbot-worker";
+  version = "0.9.1";
+
+  doCheck = false;
+  src = fetchurl {
+    url = "mirror://pypi/b/${pname}/${name}.tar.gz";
+    sha256 = "00p9l1qz6mx12npjwsycp8f9a8f2har15ig79pfsg8z7a7yw93hx";
+  };
+
+  buildInputs = with pythonPackages; [ setuptoolsTrial mock ];
+  propagatedBuildInputs = with pythonPackages; [ twisted future pkgs.treq ];
+
+  meta = with stdenv.lib; {
+    homepage = http://buildbot.net/;
+    description = "Buildbot Worker Daemon";
+    maintainers = with maintainers; [ nand0p ryansydnor ];
+    platforms = platforms.all;
+    license = licenses.gpl2;
+  };
+})
+
diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix
index 876f8b9a4..c1ec0f333 100644
--- a/krebs/5pkgs/default.nix
+++ b/krebs/5pkgs/default.nix
@@ -38,6 +38,12 @@ with import <stockholm/lib>;
 
     ReaktorPlugins = callPackage ./Reaktor/plugins.nix {};
 
+    buildbot = callPackage ./buildbot {};
+    buildbot-full = callPackage ./buildbot {
+      plugins = with pkgs.buildbot-plugins; [ www console-view waterfall-view ];
+    };
+    buildbot-worker = callPackage ./buildbot/worker.nix {};
+
     # XXX symlinkJoin changed arguments somewhere around nixpkgs d541e0d
     symlinkJoin = { name, paths, ... }@args: let
       x = pkgs.symlinkJoin args;
diff --git a/krebs/5pkgs/treq/default.nix b/krebs/5pkgs/treq/default.nix
new file mode 100644
index 000000000..20387b9cb
--- /dev/null
+++ b/krebs/5pkgs/treq/default.nix
@@ -0,0 +1,17 @@
+{ stdenv, fetchurl, pythonPackages }:
+
+pythonPackages.buildPythonPackage rec {
+  name = "${pname}-${version}";
+  pname = "treq";
+  version = "15.1.0";
+  src = fetchurl {
+    url = "mirror://pypi/t/${pname}/${name}.tar.gz";
+    sha256= "425a47d5d52a993d51211028fb6ade252e5fbea094e878bb4b644096a7322de8";
+  };
+  propagatedBuildInputs = with pythonPackages; [
+    twisted
+    pyopenssl
+    requests2
+    service-identity
+  ];
+}