From dbb25f7288be2c9d2afe796d63d1a070e353daca Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 8 Nov 2016 16:48:58 +0100 Subject: k 5 Reaktor: harden sed-plugin --- krebs/5pkgs/Reaktor/plugins.nix | 2 +- krebs/5pkgs/Reaktor/scripts/sed-plugin.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) (limited to 'krebs') diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index a483db32c..242373ced 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -59,7 +59,7 @@ rec { }; sed-plugin = buildSimpleReaktorPlugin "sed-plugin" { - path = [ pkgs.gnused pkgs.python3 ]; + path = [ pkgs.gnused pkgs.proot pkgs.python3 ]; # only support s///gi the plugin needs to see every msg # TODO: this will eat up the last regex, fix Reaktor to support fallthru append_rule = true; diff --git a/krebs/5pkgs/Reaktor/scripts/sed-plugin.py b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py index 8103c9585..6039aeb43 100644 --- a/krebs/5pkgs/Reaktor/scripts/sed-plugin.py +++ b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py @@ -34,9 +34,22 @@ if m: flagstr = '' last = d.get(usr,None) if last: - #print(re.sub(fn,tn,last,count=count,flags=flags)) from subprocess import Popen,PIPE - p = Popen(['sed','s/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE ) + import shutil + from os.path import realpath + # sed only needs stdin/stdout, we protect state_dir with this + # input to read/write arbitrary files: + # s/.\/\/; w /tmp/i (props to waldi) + # conclusion: sed is untrusted and we handle it like this + p = Popen(['proot', + # '-v','1', + '-w','/', # cwd is root + '-b','/nix/store', # mount important folders + '-b','/usr', + '-b','/bin', + '-r','/var/empty', # chroot to /var/empty + realpath(shutil.which('sed')), + 's/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE ) so,se = p.communicate(bytes("{}\n".format(last),"UTF-8")) if p.returncode: print("something went wrong when trying to process your regex: {}".format(se.decode())) -- cgit v1.2.3 From 6c092cd20ca256b1cefa92aa779380cdb71c1313 Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 11 Nov 2016 01:34:18 +0100 Subject: buildbot: 0.8.4 -> 0.9.0rc2 --- krebs/3modules/buildbot/buildbot-worker.patch | 11 +++ krebs/3modules/buildbot/buildbot.patch | 11 +++ krebs/3modules/buildbot/master.nix | 95 +++++++++++++----------- krebs/3modules/buildbot/slave.nix | 101 +++++++++----------------- 4 files changed, 110 insertions(+), 108 deletions(-) create mode 100644 krebs/3modules/buildbot/buildbot-worker.patch create mode 100644 krebs/3modules/buildbot/buildbot.patch (limited to 'krebs') diff --git a/krebs/3modules/buildbot/buildbot-worker.patch b/krebs/3modules/buildbot/buildbot-worker.patch new file mode 100644 index 000000000..df6f7ed37 --- /dev/null +++ b/krebs/3modules/buildbot/buildbot-worker.patch @@ -0,0 +1,11 @@ +--- ./buildbot_worker/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100 ++++ ./buildbot_worker/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100 +@@ -76,7 +76,7 @@ + if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'): + tailBin = "/usr/xpg4/bin/tail" + else: +- tailBin = "/usr/bin/tail" ++ tailBin = "tail" + self.p = reactor.spawnProcess(self.pp, tailBin, + ("tail", "-f", "-n", "0", self.logfile), + env=os.environ, diff --git a/krebs/3modules/buildbot/buildbot.patch b/krebs/3modules/buildbot/buildbot.patch new file mode 100644 index 000000000..3a5794d82 --- /dev/null +++ b/krebs/3modules/buildbot/buildbot.patch @@ -0,0 +1,11 @@ +--- ./buildbot/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100 ++++ ./buildbot/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100 +@@ -76,7 +76,7 @@ + if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'): + tailBin = "/usr/xpg4/bin/tail" + else: +- tailBin = "/usr/bin/tail" ++ tailBin = "tail" + self.p = reactor.spawnProcess(self.pp, tailBin, + ("tail", "-f", "-n", "0", self.logfile), + env=os.environ, diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 9e144ee0e..b31661572 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -3,14 +3,10 @@ with import ; let - # https://github.com/NixOS/nixpkgs/issues/14026 - nixpkgs-fix = import (pkgs.fetchgit { - url = https://github.com/nixos/nixpkgs; - rev = "e026b5c243ea39810826e68362718f5d703fb5d0"; - sha256 = "87e0724910a6df0371f883f99a8cf42e366fb4119f676f6f74ffb404beca2632"; - }) {}; - - buildbot = nixpkgs-fix.buildbot; + buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-full (old:{ + patches = [ ./buildbot.patch ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ]; + }); buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' # -*- python -*- from buildbot.plugins import * @@ -18,11 +14,11 @@ let import json c = BuildmasterConfig = {} - c['slaves'] = [] - slaves = json.loads('${builtins.toJSON cfg.slaves}') - slavenames = [ s for s in slaves ] - for k,v in slaves.items(): - c['slaves'].append(buildslave.BuildSlave(k, v)) + c['workers'] = [] + workers = json.loads('${builtins.toJSON cfg.workers}') + workernames = [ s for s in workers ] + for k,v in workers.items(): + c['workers'].append(worker.Worker(k, v)) # TODO: configure protocols? c['protocols'] = {'pb': {'port': 9989}} @@ -63,32 +59,46 @@ let ####### Status - c['status'] = st = [] + c['services'] = [] # If you want to configure this url, override with extraConfig c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/" ${optionalString (cfg.web.enable) '' - from buildbot.status import html - from buildbot.status.web import authz, auth - authz_cfg=authz.Authz( - auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]), - # TODO: configure harder - gracefulShutdown = False, - forceBuild = 'auth', - forceAllBuilds = 'auth', - pingBuilder = False, - stopBuild = 'auth', - stopAllBuilds = 'auth', - cancelPendingBuild = 'auth' - ) + from buildbot.plugins import util + + #authz_cfg=authz.Authz( + # auth=auth.BasicAuth([ ]), + # # TODO: configure harder + # gracefulShutdown = False, + # forceBuild = 'auth', + # forceAllBuilds = 'auth', + # pingBuilder = False, + # stopBuild = 'auth', + # stopAllBuilds = 'auth', + # cancelPendingBuild = 'auth' + #) # TODO: configure krebs.nginx - st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg)) + c['www'] = dict( + port = ${toString cfg.web.port}, + plugins = { 'waterfall_view':{}, 'console_view':{} } + ) + c['www']['auth'] = util.UserPasswordAuth({"${cfg.web.username}":"${cfg.web.password}"}) + c['www']['authz'] = util.Authz( + allowRules = [ + util.StopBuildEndpointMatcher(role="admins"), + util.ForceBuildEndpointMatcher(role="admins"), + util.RebuildBuildEndpointMatcher(role="admins") + ], + roleMatchers = [ + util.RolesFromEmails(admins=["${cfg.web.username}"]) + ] + ) ''} ${optionalString (cfg.irc.enable) '' - from buildbot.status import words - irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}", + from buildbot.plugins import reporters + irc = reporters.IRC("${cfg.irc.server}", "${cfg.irc.nick}", channels=${builtins.toJSON cfg.irc.channels}, notify_events={ 'success': 1, @@ -97,7 +107,7 @@ let 'successToFailure': 1, 'failureToSuccess': 1, }${optionalString cfg.irc.allowForce ",allowForce=True"}) - c['status'].append(irc) + c['services'].append(irc) ''} ${ concatStringsSep "\n" @@ -150,12 +160,12 @@ let ''; }; - slaves = mkOption { + workers = mkOption { default = {}; type = types.attrsOf types.str; description = '' - Attrset of slavenames with their passwords - slavename = slavepassword + Attrset of workernames with their passwords + workername = workerpassword ''; }; @@ -283,8 +293,12 @@ let options = { enable = mkEnableOption "Buildbot Master IRC Status"; channels = mkOption { - default = [ "nix-buildbot-meetup" ]; - type = with types; listOf str; + default = [ { channel = "nix-buildbot-meetup";} ]; + example = literalExample ''[ + {channel = "nix-buildbot-meetup";} + {channel = "nix-buildbot-lol"; "password" = "lol";} + ]''; + type = with types; listOf (attrsOf str); description = '' irc channels the bot should connect to ''; @@ -333,7 +347,7 @@ let }; users.extraGroups.buildbotMaster = { - gid = 672626386; + gid = genid "buildbotMaster"; }; systemd.services.buildbotMaster = { @@ -350,8 +364,6 @@ let secretsdir = shell.escape (toString ); in { PermissionsStartOnly = true; - Type = "forking"; - PIDFile = "${workdir}/twistd.pid"; # TODO: maybe also prepare buildbot.tac? ExecStartPre = pkgs.writeDash "buildbot-master-init" '' set -efux @@ -375,9 +387,8 @@ let chmod 700 -R ${workdir} chown buildbotMaster:buildbotMaster -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; - ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; - ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; + ExecStart = "${buildbot}/bin/buildbot start --nodaemon ${workdir}"; + # ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; PrivateTmp = "true"; User = "buildbotMaster"; Restart = "always"; diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 650594a6c..95b547081 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -2,53 +2,21 @@ with import ; let - buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" '' - import os - - from buildslave.bot import BuildSlave - from twisted.application import service - - basedir = '${cfg.workDir}' - rotateLength = 10000000 - maxRotatedFiles = 10 - - application = service.Application('buildslave') - - from twisted.python.logfile import LogFile - from twisted.python.log import ILogObserver, FileLogObserver - logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, - maxRotatedFiles=maxRotatedFiles) - application.setComponent(ILogObserver, FileLogObserver(logfile).emit) - - buildmaster_host = '${cfg.masterhost}' - # TODO: masterport? - port = 9989 - slavename = '${cfg.username}' - passwd = '${cfg.password}' - keepalive = 600 - usepty = 0 - umask = None - maxdelay = 300 - allow_shutdown = None - - ${cfg.extraConfig} - - s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, - keepalive, usepty, umask=umask, maxdelay=maxdelay, - allow_shutdown=allow_shutdown) - s.setServiceParent(application) - ''; default-packages = [ pkgs.git pkgs.bash ]; - cfg = config.krebs.buildbot.slave; + buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-worker (old:{ + patches = [ ./buildbot-worker.patch ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ]; + }); + cfg = config.krebs.buildbot.worker; api = { - enable = mkEnableOption "Buildbot Slave"; + enable = mkEnableOption "Buildbot worker"; workDir = mkOption { - default = "/var/lib/buildbot/slave"; + default = "/var/lib/buildbot/worker"; type = types.str; description = '' - Path to build bot slave directory. + Path to build bot worker directory. Will be created on startup. ''; }; @@ -64,30 +32,30 @@ let username = mkOption { type = types.str; description = '' - slavename used to authenticate with master + workername used to authenticate with master ''; }; password = mkOption { type = types.str; description = '' - slave password used to authenticate with master + worker password used to authenticate with master ''; }; contact = mkOption { - default = "nix slave "; + default = "nix worker "; type = types.str; description = '' - contact to be announced by buildslave + contact to be announced by buildworker ''; }; description = mkOption { - default = "Nix Generated BuildSlave"; + default = "Nix Generated Buildworker"; type = types.str; description = '' - description for hostto be announced by buildslave + description for hostto be announced by buildworker ''; }; @@ -95,7 +63,7 @@ let default = [ pkgs.git ]; type = with types; listOf package; description = '' - packages which should be in path for buildslave + packages which should be in path for buildworker ''; }; @@ -106,7 +74,7 @@ let }; type = types.attrsOf types.str; description = '' - extra environment variables to be provided to the buildslave service + extra environment variables to be provided to the buildworker service if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here. ''; }; @@ -119,26 +87,26 @@ let keepalive = 600 ''; description = '' - extra config evaluated before calling BuildSlave init in .tac file + extra config evaluated before calling Buildworker init in .tac file ''; }; }; imp = { - users.extraUsers.buildbotSlave = { - uid = genid "buildbotSlave"; - description = "Buildbot Slave"; + users.extraUsers.buildbotworker = { + uid = genid "buildbotworker"; + description = "Buildbot worker"; home = cfg.workDir; createHome = false; }; - users.extraGroups.buildbotSlave = { - gid = 1408105834; + users.extraGroups.buildbotworker = { + gid = genid "buildbotworker"; }; - systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = { - description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}"; + systemd.services."buildbotworker-${cfg.username}-${cfg.masterhost}" = { + description = "Buildbot worker for ${cfg.username}@${cfg.masterhost}"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = default-packages ++ cfg.packages; @@ -152,27 +120,28 @@ let workdir = shell.escape cfg.workDir; contact = shell.escape cfg.contact; description = shell.escape cfg.description; - buildbot = pkgs.buildbot-slave; - # TODO:make this + masterhost = shell.escape cfg.masterhost; + username = shell.escape cfg.username; + password = shell.escape cfg.password; in { PermissionsStartOnly = true; Type = "forking"; PIDFile = "${workdir}/twistd.pid"; - # TODO: maybe also prepare buildbot.tac? - ExecStartPre = pkgs.writeDash "buildbot-master-init" '' + ExecStartPre = pkgs.writeDash "buildbot-slave-init" '' set -efux mkdir -p ${workdir}/info - cp ${buildbot-slave-init} ${workdir}/buildbot.tac + # TODO: cleanup .tac file? + ${buildbot}/bin/buildbot-worker create-worker ${workdir} ${masterhost} ${username} ${password} echo ${contact} > ${workdir}/info/admin echo ${description} > ${workdir}/info/host - chown buildbotSlave:buildbotSlave -R ${workdir} + chown buildbotworker:buildbotworker -R ${workdir} chmod 700 -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildslave start ${workdir}"; - ExecStop = "${buildbot}/bin/buildslave stop ${workdir}"; + ExecStart = "${buildbot}/bin/buildbot-worker start ${workdir}"; + ExecStop = "${buildbot}/bin/buildbot-worker stop ${workdir}"; PrivateTmp = "true"; - User = "buildbotSlave"; + User = "buildbotworker"; Restart = "always"; RestartSec = "10"; }; @@ -180,6 +149,6 @@ let }; in { - options.krebs.buildbot.slave = api; + options.krebs.buildbot.worker = api; config = lib.mkIf cfg.enable imp; } -- cgit v1.2.3 From 0155fc32e1b79bc25fdfc81ee8460438b70ba128 Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 15 Nov 2016 23:26:55 +0100 Subject: k 5: override buildbot --- krebs/5pkgs/buildbot/default.nix | 81 ++++++++++++++++++++++++++++++++++++++++ krebs/5pkgs/buildbot/worker.nix | 24 ++++++++++++ krebs/5pkgs/default.nix | 6 +++ krebs/5pkgs/treq/default.nix | 17 +++++++++ 4 files changed, 128 insertions(+) create mode 100644 krebs/5pkgs/buildbot/default.nix create mode 100644 krebs/5pkgs/buildbot/worker.nix create mode 100644 krebs/5pkgs/treq/default.nix (limited to 'krebs') diff --git a/krebs/5pkgs/buildbot/default.nix b/krebs/5pkgs/buildbot/default.nix new file mode 100644 index 000000000..a0e6bb6a5 --- /dev/null +++ b/krebs/5pkgs/buildbot/default.nix @@ -0,0 +1,81 @@ +{ pkgs, stdenv, pythonPackages, fetchurl, coreutils, plugins ? [] }: + +pythonPackages.buildPythonApplication (rec { + name = "${pname}-${version}"; + pname = "buildbot"; + version = "0.9.1"; + src = fetchurl { + url = "mirror://pypi/b/${pname}/${name}.tar.gz"; + sha256 = "1kk4dlkk4rznwid9xykq2lbzksvkcr4r5kmz9hgh5hswdzv8bwx9"; + }; + doCheck = false; + buildInputs = with pythonPackages; [ + lz4 + txrequests + pyjade + boto3 + moto + txgithub + mock + setuptoolsTrial + isort + pylint + astroid + pyflakes + ]; + + propagatedBuildInputs = with pythonPackages; [ + + # core + twisted + jinja2 + zope_interface + future + sqlalchemy + sqlalchemy_migrate + future + dateutil + txaio + autobahn + + # tls + pyopenssl + service-identity + idna + pkgs.treq + + # docs + sphinx + sphinxcontrib-blockdiag + sphinxcontrib-spelling + pyenchant + docutils + ramlfications + sphinx-jinja + + ] ++ plugins; + + preInstall = '' + # writes out a file that can't be read properly + sed -i.bak -e '69,84d' buildbot/test/unit/test_www_config.py + + # re-hardcode path to tail + sed -i.bak 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py + ''; + + postFixup = '' + mv -v $out/bin/buildbot $out/bin/.wrapped-buildbot + echo "#!/bin/sh" > $out/bin/buildbot + echo "export PYTHONPATH=$PYTHONPATH" >> $out/bin/buildbot + echo "exec $out/bin/.wrapped-buildbot \"\$@\"" >> $out/bin/buildbot + chmod -c 555 $out/bin/buildbot + ''; + + meta = with stdenv.lib; { + homepage = http://buildbot.net/; + description = "Continuous integration system that automates the build/test cycle"; + maintainers = with maintainers; [ nand0p ryansydnor ]; + platforms = platforms.all; + license = licenses.gpl2; + }; +}) diff --git a/krebs/5pkgs/buildbot/worker.nix b/krebs/5pkgs/buildbot/worker.nix new file mode 100644 index 000000000..c100de5d2 --- /dev/null +++ b/krebs/5pkgs/buildbot/worker.nix @@ -0,0 +1,24 @@ +{ pkgs, stdenv, fetchurl, pythonPackages }: +pythonPackages.buildPythonApplication (rec { + name = "${pname}-${version}"; + pname = "buildbot-worker"; + version = "0.9.1"; + + doCheck = false; + src = fetchurl { + url = "mirror://pypi/b/${pname}/${name}.tar.gz"; + sha256 = "00p9l1qz6mx12npjwsycp8f9a8f2har15ig79pfsg8z7a7yw93hx"; + }; + + buildInputs = with pythonPackages; [ setuptoolsTrial mock ]; + propagatedBuildInputs = with pythonPackages; [ twisted future pkgs.treq ]; + + meta = with stdenv.lib; { + homepage = http://buildbot.net/; + description = "Buildbot Worker Daemon"; + maintainers = with maintainers; [ nand0p ryansydnor ]; + platforms = platforms.all; + license = licenses.gpl2; + }; +}) + diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix index 876f8b9a4..c1ec0f333 100644 --- a/krebs/5pkgs/default.nix +++ b/krebs/5pkgs/default.nix @@ -38,6 +38,12 @@ with import ; ReaktorPlugins = callPackage ./Reaktor/plugins.nix {}; + buildbot = callPackage ./buildbot {}; + buildbot-full = callPackage ./buildbot { + plugins = with pkgs.buildbot-plugins; [ www console-view waterfall-view ]; + }; + buildbot-worker = callPackage ./buildbot/worker.nix {}; + # XXX symlinkJoin changed arguments somewhere around nixpkgs d541e0d symlinkJoin = { name, paths, ... }@args: let x = pkgs.symlinkJoin args; diff --git a/krebs/5pkgs/treq/default.nix b/krebs/5pkgs/treq/default.nix new file mode 100644 index 000000000..20387b9cb --- /dev/null +++ b/krebs/5pkgs/treq/default.nix @@ -0,0 +1,17 @@ +{ stdenv, fetchurl, pythonPackages }: + +pythonPackages.buildPythonPackage rec { + name = "${pname}-${version}"; + pname = "treq"; + version = "15.1.0"; + src = fetchurl { + url = "mirror://pypi/t/${pname}/${name}.tar.gz"; + sha256= "425a47d5d52a993d51211028fb6ade252e5fbea094e878bb4b644096a7322de8"; + }; + propagatedBuildInputs = with pythonPackages; [ + twisted + pyopenssl + requests2 + service-identity + ]; +} -- cgit v1.2.3