Revert "buildbot: 0.8.4 -> 0.9.0rc2"

This reverts commit 6c092cd20ca256b1cefa92aa779380cdb71c1313.

7 files changed, 126 insertions, 226 deletions

diff --git a/krebs/2configs/shared-buildbot.nix b/krebs/2configs/shared-buildbot.nix
index a9e5afc75..b534f0b62 100644
--- a/krebs/2configs/shared-buildbot.nix
+++ b/krebs/2configs/shared-buildbot.nix
@@ -26,11 +26,12 @@ in {
   nix.gc.automatic = true;
   nix.gc.dates = "05:23";
   networking.firewall.allowedTCPPorts = [ 80 8010 9989 ];
+
   krebs.buildbot.master = let
     stockholm-mirror-url = "http://cgit.${hostname}.r/stockholm" ;
   in {
-    workers = {
-      testworker =  "krebspass";
+    slaves = {
+      testslave =  "krebspass";
     };
     change_source.stockholm = ''
   stockholm_repo = '${stockholm-mirror-url}'
@@ -119,7 +120,7 @@ in {
                             system={}".format(i)])
 
         bu.append(util.BuilderConfig(name="fast-tests",
-              workernames=workernames,
+              slavenames=slavenames,
               factory=f))
 
             '';
@@ -131,25 +132,25 @@ in {
 
 
   bu.append(util.BuilderConfig(name="build-local",
-        workernames=workernames,
+        slavenames=slavenames,
         factory=f))
       '';
 #      slow-tests = ''
 #  s = util.BuildFactory()
 #  s.addStep(grab_repo)
 #
-#  # worker needs 2 files:
+#  # slave needs 2 files:
 #  # * cac.json
 #  # * retiolum
-#  s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", workerdest="cac.json"))
-#  s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", workerdest="retiolum.rsa_key.priv"))
+#  s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json"))
+#  s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv"))
 #  addShell(s, name="infest-cac-centos7",env=env,
 #              sigtermTime=60,           # SIGTERM 1 minute before SIGKILL
 #              timeout=10800,             # 3h
 #              command=nixshell + ["infest-cac-centos7"])
 #
 #  bu.append(util.BuilderConfig(name="full-tests",
-#        workernames=workernames,
+#        slavenames=slavenames,
 #        factory=s))
 #      '';
     };
@@ -161,7 +162,7 @@ in {
       enable = true;
       nick = "${hostname}bot";
       server = "ni.r";
-      channels = [ { channel = "retiolum"; } ];
+      channels = [ "retiolum" ];
       allowForce = true;
     };
     extraConfig = ''
@@ -169,10 +170,10 @@ in {
     '';
   };
 
-  krebs.buildbot.worker = {
+  krebs.buildbot.slave = {
     enable = true;
     masterhost = "localhost";
-    username = "testworker";
+    username = "testslave";
     password = "krebspass";
     packages = with pkgs; [ gnumake jq nix populate ];
     # all nix commands will need a working nixpkgs installation
diff --git a/krebs/3modules/buildbot/buildbot-worker.patch b/krebs/3modules/buildbot/buildbot-worker.patch
deleted file mode 100644
index df6f7ed37..000000000
--- a/krebs/3modules/buildbot/buildbot-worker.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- ./buildbot_worker/scripts/logwatcher.py  2016-11-10 23:25:46.956000000 +0100
-+++ ./buildbot_worker/scripts/logwatcher.py.fix  2016-11-10 23:24:33.225000000 +0100
-@@ -76,7 +76,7 @@
-         if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'):
-             tailBin = "/usr/xpg4/bin/tail"
-         else:
--            tailBin = "/usr/bin/tail"
-+            tailBin = "tail"
-         self.p = reactor.spawnProcess(self.pp, tailBin,
-                                       ("tail", "-f", "-n", "0", self.logfile),
-                                       env=os.environ,
diff --git a/krebs/3modules/buildbot/buildbot.patch b/krebs/3modules/buildbot/buildbot.patch
deleted file mode 100644
index 3a5794d82..000000000
--- a/krebs/3modules/buildbot/buildbot.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- ./buildbot/scripts/logwatcher.py  2016-11-10 23:25:46.956000000 +0100
-+++ ./buildbot/scripts/logwatcher.py.fix  2016-11-10 23:24:33.225000000 +0100
-@@ -76,7 +76,7 @@
-         if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'):
-             tailBin = "/usr/xpg4/bin/tail"
-         else:
--            tailBin = "/usr/bin/tail"
-+            tailBin = "tail"
-         self.p = reactor.spawnProcess(self.pp, tailBin,
-                                       ("tail", "-f", "-n", "0", self.logfile),
-                                       env=os.environ,
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index d75e6c880..6c7af6da5 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -3,10 +3,14 @@
 with import <stockholm/lib>;
 let
 
-  buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-full (old:{
-    patches = [ ./buildbot.patch ];
-    propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ];
-  });
+  # https://github.com/NixOS/nixpkgs/issues/14026
+  nixpkgs-fix = import (pkgs.fetchgit {
+    url = https://github.com/nixos/nixpkgs;
+    rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
+    sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8";
+  }) {};
+
+  buildbot = nixpkgs-fix.buildbot;
   buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
     # -*- python -*-
     from buildbot.plugins import *
@@ -14,11 +18,11 @@ let
     import json
     c = BuildmasterConfig = {}
 
-    c['workers'] = []
-    workers = json.loads('${builtins.toJSON cfg.workers}')
-    workernames = [ s for s in workers ]
-    for k,v in workers.items():
-      c['workers'].append(worker.Worker(k, v))
+    c['slaves'] = []
+    slaves = json.loads('${builtins.toJSON cfg.slaves}')
+    slavenames = [ s for s in slaves ]
+    for k,v in slaves.items():
+      c['slaves'].append(buildslave.BuildSlave(k, v))
 
     # TODO: configure protocols?
     c['protocols'] = {'pb': {'port': 9989}}
@@ -59,45 +63,32 @@ let
 
 
     ####### Status
-    c['services'] = []
+    c['status'] = st = []
 
     # If you want to configure this url, override with extraConfig
     c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
 
     ${optionalString (cfg.web.enable) ''
-      from buildbot.plugins import util
-
-      #authz_cfg=authz.Authz(
-      #    auth=auth.BasicAuth([  ]),
-      #    # TODO: configure harder
-      #    gracefulShutdown = False,
-      #    forceBuild = 'auth',
-      #    forceAllBuilds = 'auth',
-      #    pingBuilder = False,
-      #    stopBuild = 'auth',
-      #    stopAllBuilds = 'auth',
-      #    cancelPendingBuild = 'auth'
-      #)
-      c['www'] = dict(
-        port = ${toString cfg.web.port},
-        plugins = { 'waterfall_view':{}, 'console_view':{} }
-      )
-      c['www']['auth'] = util.UserPasswordAuth({"${cfg.web.username}":"${cfg.web.password}"})
-      c['www']['authz'] = util.Authz(
-        allowRules = [
-          util.StopBuildEndpointMatcher(role="admins"),
-          util.ForceBuildEndpointMatcher(role="admins"),
-          util.RebuildBuildEndpointMatcher(role="admins")
-        ],
-        roleMatchers = [
-          util.RolesFromEmails(admins=["${cfg.web.username}"])
-        ]
+      from buildbot.status import html
+      from buildbot.status.web import authz, auth
+      authz_cfg=authz.Authz(
+          auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
+          # TODO: configure harder
+          gracefulShutdown = False,
+          forceBuild = 'auth',
+          forceAllBuilds = 'auth',
+          pingBuilder = False,
+          stopBuild = 'auth',
+          stopAllBuilds = 'auth',
+          cancelPendingBuild = 'auth'
       )
+      # TODO: configure krebs.nginx
+      st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
       ''}
 
     ${optionalString (cfg.irc.enable) ''
-      from buildbot.plugins import reporters
-      irc = reporters.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
+      from buildbot.status import words
+      irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
                       channels=${builtins.toJSON cfg.irc.channels},
                       notify_events={
                         'success': 1,
@@ -106,7 +97,7 @@ let
                         'successToFailure': 1,
                         'failureToSuccess': 1,
                       }${optionalString cfg.irc.allowForce ",allowForce=True"})
-      c['services'].append(irc)
+      c['status'].append(irc)
       ''}
 
     ${ concatStringsSep "\n"
@@ -159,12 +150,12 @@ let
       '';
     };
 
-    workers = mkOption {
+    slaves = mkOption {
       default = {};
       type = types.attrsOf types.str;
       description = ''
-        Attrset of workernames with their passwords
-        workername = workerpassword
+        Attrset of slavenames with their passwords
+        slavename = slavepassword
       '';
     };
 
@@ -292,12 +283,8 @@ let
         options = {
           enable = mkEnableOption "Buildbot Master IRC Status";
           channels = mkOption {
-            default = [ { channel = "nix-buildbot-meetup";} ];
-            example = literalExample ''[
-              {channel = "nix-buildbot-meetup";}
-              {channel = "nix-buildbot-lol"; "password" = "lol";}
-            ]'';
-            type = with types; listOf (attrsOf str);
+            default = [ "nix-buildbot-meetup" ];
+            type = with types; listOf str;
             description = ''
               irc channels the bot should connect to
             '';
@@ -346,7 +333,7 @@ let
     };
 
     users.extraGroups.buildbotMaster = {
-      gid = genid "buildbotMaster";
+      gid = 672626386;
     };
 
     systemd.services.buildbotMaster = {
@@ -363,6 +350,8 @@ let
         secretsdir = shell.escape (toString <secrets>);
       in {
         PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
         # TODO: maybe also prepare buildbot.tac?
         ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
           set -efux
@@ -386,8 +375,9 @@ let
           chmod 700 -R ${workdir}
           chown buildbotMaster:buildbotMaster -R ${workdir}
         '';
-        ExecStart = "${buildbot}/bin/buildbot start --nodaemon ${workdir}";
-        # ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
+        ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
+        ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
         PrivateTmp = "true";
         User = "buildbotMaster";
         Restart = "always";
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
index 95b547081..932923ae5 100644
--- a/krebs/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -2,21 +2,59 @@
 
 with import <stockholm/lib>;
 let
+  nixpkgs-fix = import (pkgs.fetchgit {
+    url = https://github.com/nixos/nixpkgs;
+    rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
+    sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8";
+  }) {};
+
+  buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
+    import os
+
+    from buildslave.bot import BuildSlave
+    from twisted.application import service
+
+    basedir = '${cfg.workDir}'
+    rotateLength = 10000000
+    maxRotatedFiles = 10
+
+    application = service.Application('buildslave')
+
+    from twisted.python.logfile import LogFile
+    from twisted.python.log import ILogObserver, FileLogObserver
+    logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+                                  maxRotatedFiles=maxRotatedFiles)
+    application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+
+    buildmaster_host = '${cfg.masterhost}'
+    # TODO: masterport?
+    port = 9989
+    slavename = '${cfg.username}'
+    passwd = '${cfg.password}'
+    keepalive = 600
+    usepty = 0
+    umask = None
+    maxdelay = 300
+    allow_shutdown = None
+
+    ${cfg.extraConfig}
+
+    s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
+                  keepalive, usepty, umask=umask, maxdelay=maxdelay,
+                  allow_shutdown=allow_shutdown)
+    s.setServiceParent(application)
+    '';
   default-packages = [ pkgs.git pkgs.bash ];
-  buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-worker (old:{
-    patches = [ ./buildbot-worker.patch ];
-    propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ];
-  });
-  cfg = config.krebs.buildbot.worker;
+  cfg = config.krebs.buildbot.slave;
 
   api = {
-    enable = mkEnableOption "Buildbot worker";
+    enable = mkEnableOption "Buildbot Slave";
 
     workDir = mkOption {
-      default = "/var/lib/buildbot/worker";
+      default = "/var/lib/buildbot/slave";
       type = types.str;
       description = ''
-        Path to build bot worker directory.
+        Path to build bot slave directory.
         Will be created on startup.
       '';
     };
@@ -32,30 +70,30 @@ let
     username = mkOption {
       type = types.str;
       description = ''
-        workername used to authenticate with master
+        slavename used to authenticate with master
       '';
     };
 
     password = mkOption {
       type = types.str;
       description = ''
-        worker password used to authenticate with master
+        slave password used to authenticate with master
       '';
     };
 
     contact = mkOption {
-      default = "nix worker <buildworker@${config.networking.hostName}>";
+      default = "nix slave <buildslave@${config.networking.hostName}>";
       type = types.str;
       description = ''
-        contact to be announced by buildworker
+        contact to be announced by buildslave
       '';
     };
 
     description = mkOption {
-      default = "Nix Generated Buildworker";
+      default = "Nix Generated BuildSlave";
       type = types.str;
       description = ''
-        description for hostto be announced by buildworker
+        description for hostto be announced by buildslave
       '';
     };
 
@@ -63,7 +101,7 @@ let
       default = [ pkgs.git ];
       type = with types; listOf package;
       description = ''
-        packages which should be in path for buildworker
+        packages which should be in path for buildslave
       '';
     };
 
@@ -74,7 +112,7 @@ let
       };
       type = types.attrsOf types.str;
       description = ''
-        extra environment variables to be provided to the buildworker service
+        extra environment variables to be provided to the buildslave service
         if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
       '';
     };
@@ -87,26 +125,26 @@ let
         keepalive = 600
       '';
       description = ''
-        extra config evaluated before calling Buildworker init in .tac file
+        extra config evaluated before calling BuildSlave init in .tac file
       '';
     };
   };
 
   imp = {
 
-    users.extraUsers.buildbotworker = {
-      uid = genid "buildbotworker";
-      description = "Buildbot worker";
+    users.extraUsers.buildbotSlave = {
+      uid = genid "buildbotSlave";
+      description = "Buildbot Slave";
       home = cfg.workDir;
       createHome = false;
     };
 
-    users.extraGroups.buildbotworker = {
-      gid = genid "buildbotworker";
+    users.extraGroups.buildbotSlave = {
+      gid = 1408105834;
     };
 
-    systemd.services."buildbotworker-${cfg.username}-${cfg.masterhost}" = {
-      description = "Buildbot worker for ${cfg.username}@${cfg.masterhost}";
+    systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
+      description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
       path = default-packages ++ cfg.packages;
@@ -120,28 +158,27 @@ let
         workdir = shell.escape cfg.workDir;
         contact = shell.escape cfg.contact;
         description = shell.escape cfg.description;
-        masterhost = shell.escape cfg.masterhost;
-        username = shell.escape cfg.username;
-        password = shell.escape cfg.password;
+        buildbot = nixpkgs-fix.buildbot-slave;
+        # TODO:make this
       in {
         PermissionsStartOnly = true;
         Type = "forking";
         PIDFile = "${workdir}/twistd.pid";
-        ExecStartPre = pkgs.writeDash "buildbot-slave-init" ''
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
           set -efux
           mkdir -p ${workdir}/info
-          # TODO: cleanup .tac file?
-          ${buildbot}/bin/buildbot-worker create-worker ${workdir} ${masterhost} ${username} ${password}
+          cp ${buildbot-slave-init} ${workdir}/buildbot.tac
           echo ${contact} > ${workdir}/info/admin
           echo ${description} > ${workdir}/info/host
 
-          chown buildbotworker:buildbotworker -R ${workdir}
+          chown buildbotSlave:buildbotSlave -R ${workdir}
           chmod 700 -R ${workdir}
         '';
-        ExecStart = "${buildbot}/bin/buildbot-worker start ${workdir}";
-        ExecStop = "${buildbot}/bin/buildbot-worker stop ${workdir}";
+        ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
         PrivateTmp = "true";
-        User = "buildbotworker";
+        User = "buildbotSlave";
         Restart = "always";
         RestartSec = "10";
       };
@@ -149,6 +186,6 @@ let
   };
 in
 {
-  options.krebs.buildbot.worker = api;
+  options.krebs.buildbot.slave = api;
   config = lib.mkIf cfg.enable imp;
 }
diff --git a/krebs/5pkgs/simple/buildbot/default.nix b/krebs/5pkgs/simple/buildbot/default.nix
deleted file mode 100644
index 37eea5fd9..000000000
--- a/krebs/5pkgs/simple/buildbot/default.nix
+++ /dev/null
@@ -1,82 +0,0 @@
-{ pkgs, stdenv, pythonPackages, fetchurl, coreutils, plugins ? [] }:
-
-pythonPackages.buildPythonApplication (rec {
-  name = "${pname}-${version}";
-  pname = "buildbot";
-  version = "0.9.4";
-  src = fetchurl {
-    url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-    sha256 = "0wklrn4fszac9wi8zw3vbsznwyff6y57cz0i81zvh46skb6n3086";
-  };
-  doCheck = false;
-  buildInputs = with pythonPackages; [
-    lz4
-    txrequests
-    pyjade
-    boto3
-    moto
-    txgithub
-    mock
-    setuptoolsTrial
-    isort
-    pylint
-    astroid
-    pyflakes
-    pyjwt
-  ];
-
-  propagatedBuildInputs = with pythonPackages; [
-
-    # core
-    twisted
-    jinja2
-    zope_interface
-    future
-    sqlalchemy
-    sqlalchemy_migrate
-    future
-    dateutil
-    txaio
-    autobahn
-
-    # tls
-    pyopenssl
-    service-identity
-    idna
-    pkgs.treq
-
-    # docs
-    sphinx
-    sphinxcontrib-blockdiag
-    sphinxcontrib-spelling
-    pyenchant
-    docutils
-    ramlfications
-    sphinx-jinja
-
-  ] ++ plugins;
-
-  preInstall = ''
-    # writes out a file that can't be read properly
-    sed -i.bak -e '69,84d' buildbot/test/unit/test_www_config.py
-
-    # re-hardcode path to tail
-    sed -i.bak 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py
-  '';
-
-  postFixup = ''
-    mv -v $out/bin/buildbot $out/bin/.wrapped-buildbot
-    echo "#!/bin/sh" > $out/bin/buildbot
-    echo "export PYTHONPATH=$PYTHONPATH" >> $out/bin/buildbot
-    echo "exec $out/bin/.wrapped-buildbot \"\$@\"" >> $out/bin/buildbot
-    chmod -c 555 $out/bin/buildbot
-  '';
-
-  meta = with stdenv.lib; {
-    homepage = http://buildbot.net/;
-    description = "Continuous integration system that automates the build/test cycle";
-    maintainers = with maintainers; [ nand0p ryansydnor ];
-    platforms = platforms.all;
-    license = licenses.gpl2;
-  };
-})
diff --git a/krebs/5pkgs/simple/buildbot/worker.nix b/krebs/5pkgs/simple/buildbot/worker.nix
deleted file mode 100644
index 34e526858..000000000
--- a/krebs/5pkgs/simple/buildbot/worker.nix
+++ /dev/null
@@ -1,24 +0,0 @@
-{ pkgs, stdenv, fetchurl, pythonPackages }:
-pythonPackages.buildPythonApplication (rec {
-  name = "${pname}-${version}";
-  pname = "buildbot-worker";
-  version = "0.9.4";
-
-  doCheck = false;
-  src = fetchurl {
-    url = "mirror://pypi/b/${pname}/${name}.tar.gz";
-    sha256 = "0rdrr8x7sn2nxl51p6h9ad42s3c28lb6sys84zrg0d7fm4zhv7hj";
-  };
-
-  buildInputs = with pythonPackages; [ setuptoolsTrial mock ];
-  propagatedBuildInputs = with pythonPackages; [ twisted future pkgs.treq ];
-
-  meta = with stdenv.lib; {
-    homepage = http://buildbot.net/;
-    description = "Buildbot Worker Daemon";
-    maintainers = with maintainers; [ nand0p ryansydnor ];
-    platforms = platforms.all;
-    license = licenses.gpl2;
-  };
-})
-