From 8580a5a53fd8eb356c0e55eabe9f568e985dbfa9 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 23 Jul 2017 21:54:11 +0200 Subject: Revert "buildbot: 0.8.4 -> 0.9.0rc2" This reverts commit 6c092cd20ca256b1cefa92aa779380cdb71c1313. --- krebs/3modules/buildbot/buildbot-worker.patch | 11 --- krebs/3modules/buildbot/buildbot.patch | 11 --- krebs/3modules/buildbot/master.nix | 94 ++++++++++------------ krebs/3modules/buildbot/slave.nix | 107 +++++++++++++++++--------- 4 files changed, 114 insertions(+), 109 deletions(-) delete mode 100644 krebs/3modules/buildbot/buildbot-worker.patch delete mode 100644 krebs/3modules/buildbot/buildbot.patch (limited to 'krebs/3modules') diff --git a/krebs/3modules/buildbot/buildbot-worker.patch b/krebs/3modules/buildbot/buildbot-worker.patch deleted file mode 100644 index df6f7ed37..000000000 --- a/krebs/3modules/buildbot/buildbot-worker.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- ./buildbot_worker/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100 -+++ ./buildbot_worker/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100 -@@ -76,7 +76,7 @@ - if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'): - tailBin = "/usr/xpg4/bin/tail" - else: -- tailBin = "/usr/bin/tail" -+ tailBin = "tail" - self.p = reactor.spawnProcess(self.pp, tailBin, - ("tail", "-f", "-n", "0", self.logfile), - env=os.environ, diff --git a/krebs/3modules/buildbot/buildbot.patch b/krebs/3modules/buildbot/buildbot.patch deleted file mode 100644 index 3a5794d82..000000000 --- a/krebs/3modules/buildbot/buildbot.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- ./buildbot/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100 -+++ ./buildbot/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100 -@@ -76,7 +76,7 @@ - if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'): - tailBin = "/usr/xpg4/bin/tail" - else: -- tailBin = "/usr/bin/tail" -+ tailBin = "tail" - self.p = reactor.spawnProcess(self.pp, tailBin, - ("tail", "-f", "-n", "0", self.logfile), - env=os.environ, diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index d75e6c880..6c7af6da5 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -3,10 +3,14 @@ with import ; let - buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-full (old:{ - patches = [ ./buildbot.patch ]; - propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ]; - }); + # https://github.com/NixOS/nixpkgs/issues/14026 + nixpkgs-fix = import (pkgs.fetchgit { + url = https://github.com/nixos/nixpkgs; + rev = "e026b5c243ea39810826e68362718f5d703fb5d0"; + sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8"; + }) {}; + + buildbot = nixpkgs-fix.buildbot; buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' # -*- python -*- from buildbot.plugins import * @@ -14,11 +18,11 @@ let import json c = BuildmasterConfig = {} - c['workers'] = [] - workers = json.loads('${builtins.toJSON cfg.workers}') - workernames = [ s for s in workers ] - for k,v in workers.items(): - c['workers'].append(worker.Worker(k, v)) + c['slaves'] = [] + slaves = json.loads('${builtins.toJSON cfg.slaves}') + slavenames = [ s for s in slaves ] + for k,v in slaves.items(): + c['slaves'].append(buildslave.BuildSlave(k, v)) # TODO: configure protocols? c['protocols'] = {'pb': {'port': 9989}} @@ -59,45 +63,32 @@ let ####### Status - c['services'] = [] + c['status'] = st = [] # If you want to configure this url, override with extraConfig c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/" ${optionalString (cfg.web.enable) '' - from buildbot.plugins import util - - #authz_cfg=authz.Authz( - # auth=auth.BasicAuth([ ]), - # # TODO: configure harder - # gracefulShutdown = False, - # forceBuild = 'auth', - # forceAllBuilds = 'auth', - # pingBuilder = False, - # stopBuild = 'auth', - # stopAllBuilds = 'auth', - # cancelPendingBuild = 'auth' - #) - c['www'] = dict( - port = ${toString cfg.web.port}, - plugins = { 'waterfall_view':{}, 'console_view':{} } - ) - c['www']['auth'] = util.UserPasswordAuth({"${cfg.web.username}":"${cfg.web.password}"}) - c['www']['authz'] = util.Authz( - allowRules = [ - util.StopBuildEndpointMatcher(role="admins"), - util.ForceBuildEndpointMatcher(role="admins"), - util.RebuildBuildEndpointMatcher(role="admins") - ], - roleMatchers = [ - util.RolesFromEmails(admins=["${cfg.web.username}"]) - ] + from buildbot.status import html + from buildbot.status.web import authz, auth + authz_cfg=authz.Authz( + auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]), + # TODO: configure harder + gracefulShutdown = False, + forceBuild = 'auth', + forceAllBuilds = 'auth', + pingBuilder = False, + stopBuild = 'auth', + stopAllBuilds = 'auth', + cancelPendingBuild = 'auth' ) + # TODO: configure krebs.nginx + st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg)) ''} ${optionalString (cfg.irc.enable) '' - from buildbot.plugins import reporters - irc = reporters.IRC("${cfg.irc.server}", "${cfg.irc.nick}", + from buildbot.status import words + irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}", channels=${builtins.toJSON cfg.irc.channels}, notify_events={ 'success': 1, @@ -106,7 +97,7 @@ let 'successToFailure': 1, 'failureToSuccess': 1, }${optionalString cfg.irc.allowForce ",allowForce=True"}) - c['services'].append(irc) + c['status'].append(irc) ''} ${ concatStringsSep "\n" @@ -159,12 +150,12 @@ let ''; }; - workers = mkOption { + slaves = mkOption { default = {}; type = types.attrsOf types.str; description = '' - Attrset of workernames with their passwords - workername = workerpassword + Attrset of slavenames with their passwords + slavename = slavepassword ''; }; @@ -292,12 +283,8 @@ let options = { enable = mkEnableOption "Buildbot Master IRC Status"; channels = mkOption { - default = [ { channel = "nix-buildbot-meetup";} ]; - example = literalExample ''[ - {channel = "nix-buildbot-meetup";} - {channel = "nix-buildbot-lol"; "password" = "lol";} - ]''; - type = with types; listOf (attrsOf str); + default = [ "nix-buildbot-meetup" ]; + type = with types; listOf str; description = '' irc channels the bot should connect to ''; @@ -346,7 +333,7 @@ let }; users.extraGroups.buildbotMaster = { - gid = genid "buildbotMaster"; + gid = 672626386; }; systemd.services.buildbotMaster = { @@ -363,6 +350,8 @@ let secretsdir = shell.escape (toString ); in { PermissionsStartOnly = true; + Type = "forking"; + PIDFile = "${workdir}/twistd.pid"; # TODO: maybe also prepare buildbot.tac? ExecStartPre = pkgs.writeDash "buildbot-master-init" '' set -efux @@ -386,8 +375,9 @@ let chmod 700 -R ${workdir} chown buildbotMaster:buildbotMaster -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildbot start --nodaemon ${workdir}"; - # ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; + ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; + ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; + ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; PrivateTmp = "true"; User = "buildbotMaster"; Restart = "always"; diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 95b547081..932923ae5 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -2,21 +2,59 @@ with import ; let + nixpkgs-fix = import (pkgs.fetchgit { + url = https://github.com/nixos/nixpkgs; + rev = "e026b5c243ea39810826e68362718f5d703fb5d0"; + sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8"; + }) {}; + + buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" '' + import os + + from buildslave.bot import BuildSlave + from twisted.application import service + + basedir = '${cfg.workDir}' + rotateLength = 10000000 + maxRotatedFiles = 10 + + application = service.Application('buildslave') + + from twisted.python.logfile import LogFile + from twisted.python.log import ILogObserver, FileLogObserver + logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, + maxRotatedFiles=maxRotatedFiles) + application.setComponent(ILogObserver, FileLogObserver(logfile).emit) + + buildmaster_host = '${cfg.masterhost}' + # TODO: masterport? + port = 9989 + slavename = '${cfg.username}' + passwd = '${cfg.password}' + keepalive = 600 + usepty = 0 + umask = None + maxdelay = 300 + allow_shutdown = None + + ${cfg.extraConfig} + + s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, + keepalive, usepty, umask=umask, maxdelay=maxdelay, + allow_shutdown=allow_shutdown) + s.setServiceParent(application) + ''; default-packages = [ pkgs.git pkgs.bash ]; - buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-worker (old:{ - patches = [ ./buildbot-worker.patch ]; - propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ]; - }); - cfg = config.krebs.buildbot.worker; + cfg = config.krebs.buildbot.slave; api = { - enable = mkEnableOption "Buildbot worker"; + enable = mkEnableOption "Buildbot Slave"; workDir = mkOption { - default = "/var/lib/buildbot/worker"; + default = "/var/lib/buildbot/slave"; type = types.str; description = '' - Path to build bot worker directory. + Path to build bot slave directory. Will be created on startup. ''; }; @@ -32,30 +70,30 @@ let username = mkOption { type = types.str; description = '' - workername used to authenticate with master + slavename used to authenticate with master ''; }; password = mkOption { type = types.str; description = '' - worker password used to authenticate with master + slave password used to authenticate with master ''; }; contact = mkOption { - default = "nix worker "; + default = "nix slave "; type = types.str; description = '' - contact to be announced by buildworker + contact to be announced by buildslave ''; }; description = mkOption { - default = "Nix Generated Buildworker"; + default = "Nix Generated BuildSlave"; type = types.str; description = '' - description for hostto be announced by buildworker + description for hostto be announced by buildslave ''; }; @@ -63,7 +101,7 @@ let default = [ pkgs.git ]; type = with types; listOf package; description = '' - packages which should be in path for buildworker + packages which should be in path for buildslave ''; }; @@ -74,7 +112,7 @@ let }; type = types.attrsOf types.str; description = '' - extra environment variables to be provided to the buildworker service + extra environment variables to be provided to the buildslave service if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here. ''; }; @@ -87,26 +125,26 @@ let keepalive = 600 ''; description = '' - extra config evaluated before calling Buildworker init in .tac file + extra config evaluated before calling BuildSlave init in .tac file ''; }; }; imp = { - users.extraUsers.buildbotworker = { - uid = genid "buildbotworker"; - description = "Buildbot worker"; + users.extraUsers.buildbotSlave = { + uid = genid "buildbotSlave"; + description = "Buildbot Slave"; home = cfg.workDir; createHome = false; }; - users.extraGroups.buildbotworker = { - gid = genid "buildbotworker"; + users.extraGroups.buildbotSlave = { + gid = 1408105834; }; - systemd.services."buildbotworker-${cfg.username}-${cfg.masterhost}" = { - description = "Buildbot worker for ${cfg.username}@${cfg.masterhost}"; + systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = { + description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = default-packages ++ cfg.packages; @@ -120,28 +158,27 @@ let workdir = shell.escape cfg.workDir; contact = shell.escape cfg.contact; description = shell.escape cfg.description; - masterhost = shell.escape cfg.masterhost; - username = shell.escape cfg.username; - password = shell.escape cfg.password; + buildbot = nixpkgs-fix.buildbot-slave; + # TODO:make this in { PermissionsStartOnly = true; Type = "forking"; PIDFile = "${workdir}/twistd.pid"; - ExecStartPre = pkgs.writeDash "buildbot-slave-init" '' + # TODO: maybe also prepare buildbot.tac? + ExecStartPre = pkgs.writeDash "buildbot-master-init" '' set -efux mkdir -p ${workdir}/info - # TODO: cleanup .tac file? - ${buildbot}/bin/buildbot-worker create-worker ${workdir} ${masterhost} ${username} ${password} + cp ${buildbot-slave-init} ${workdir}/buildbot.tac echo ${contact} > ${workdir}/info/admin echo ${description} > ${workdir}/info/host - chown buildbotworker:buildbotworker -R ${workdir} + chown buildbotSlave:buildbotSlave -R ${workdir} chmod 700 -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildbot-worker start ${workdir}"; - ExecStop = "${buildbot}/bin/buildbot-worker stop ${workdir}"; + ExecStart = "${buildbot}/bin/buildslave start ${workdir}"; + ExecStop = "${buildbot}/bin/buildslave stop ${workdir}"; PrivateTmp = "true"; - User = "buildbotworker"; + User = "buildbotSlave"; Restart = "always"; RestartSec = "10"; }; @@ -149,6 +186,6 @@ let }; in { - options.krebs.buildbot.worker = api; + options.krebs.buildbot.slave = api; config = lib.mkIf cfg.enable imp; } -- cgit v1.2.3