diff options
-rw-r--r-- | krebs/3modules/buildbot/buildbot-worker.patch | 11 | ||||
-rw-r--r-- | krebs/3modules/buildbot/buildbot.patch | 11 | ||||
-rw-r--r-- | krebs/3modules/buildbot/master.nix | 95 | ||||
-rw-r--r-- | krebs/3modules/buildbot/slave.nix | 101 | ||||
-rw-r--r-- | krebs/5pkgs/Reaktor/plugins.nix | 2 | ||||
-rw-r--r-- | krebs/5pkgs/Reaktor/scripts/sed-plugin.py | 17 | ||||
-rw-r--r-- | krebs/5pkgs/buildbot/default.nix | 81 | ||||
-rw-r--r-- | krebs/5pkgs/buildbot/worker.nix | 24 | ||||
-rw-r--r-- | krebs/5pkgs/default.nix | 6 | ||||
-rw-r--r-- | krebs/5pkgs/treq/default.nix | 17 | ||||
-rw-r--r-- | lass/2configs/buildbot-standalone.nix | 2 | ||||
-rw-r--r-- | makefu/1systems/vbob.nix | 8 | ||||
-rw-r--r-- | makefu/2configs/graphite-standalone.nix | 2 | ||||
-rw-r--r-- | makefu/2configs/save-diskspace.nix | 8 | ||||
-rw-r--r-- | makefu/5pkgs/debmirror/default.nix | 41 | ||||
-rw-r--r-- | makefu/5pkgs/default.nix | 4 | ||||
-rw-r--r-- | shared/1systems/test-all-krebs-modules.nix | 2 | ||||
-rw-r--r-- | shared/2configs/shared-buildbot.nix | 36 |
18 files changed, 323 insertions, 145 deletions
diff --git a/krebs/3modules/buildbot/buildbot-worker.patch b/krebs/3modules/buildbot/buildbot-worker.patch new file mode 100644 index 000000000..df6f7ed37 --- /dev/null +++ b/krebs/3modules/buildbot/buildbot-worker.patch @@ -0,0 +1,11 @@ +--- ./buildbot_worker/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100 ++++ ./buildbot_worker/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100 +@@ -76,7 +76,7 @@ + if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'): + tailBin = "/usr/xpg4/bin/tail" + else: +- tailBin = "/usr/bin/tail" ++ tailBin = "tail" + self.p = reactor.spawnProcess(self.pp, tailBin, + ("tail", "-f", "-n", "0", self.logfile), + env=os.environ, diff --git a/krebs/3modules/buildbot/buildbot.patch b/krebs/3modules/buildbot/buildbot.patch new file mode 100644 index 000000000..3a5794d82 --- /dev/null +++ b/krebs/3modules/buildbot/buildbot.patch @@ -0,0 +1,11 @@ +--- ./buildbot/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100 ++++ ./buildbot/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100 +@@ -76,7 +76,7 @@ + if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'): + tailBin = "/usr/xpg4/bin/tail" + else: +- tailBin = "/usr/bin/tail" ++ tailBin = "tail" + self.p = reactor.spawnProcess(self.pp, tailBin, + ("tail", "-f", "-n", "0", self.logfile), + env=os.environ, diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 9e144ee0e..b31661572 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -3,14 +3,10 @@ with import <stockholm/lib>; let - # https://github.com/NixOS/nixpkgs/issues/14026 - nixpkgs-fix = import (pkgs.fetchgit { - url = https://github.com/nixos/nixpkgs; - rev = "e026b5c243ea39810826e68362718f5d703fb5d0"; - sha256 = "87e0724910a6df0371f883f99a8cf42e366fb4119f676f6f74ffb404beca2632"; - }) {}; - - buildbot = nixpkgs-fix.buildbot; + buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-full (old:{ + patches = [ ./buildbot.patch ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ]; + }); buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' # -*- python -*- from buildbot.plugins import * @@ -18,11 +14,11 @@ let import json c = BuildmasterConfig = {} - c['slaves'] = [] - slaves = json.loads('${builtins.toJSON cfg.slaves}') - slavenames = [ s for s in slaves ] - for k,v in slaves.items(): - c['slaves'].append(buildslave.BuildSlave(k, v)) + c['workers'] = [] + workers = json.loads('${builtins.toJSON cfg.workers}') + workernames = [ s for s in workers ] + for k,v in workers.items(): + c['workers'].append(worker.Worker(k, v)) # TODO: configure protocols? c['protocols'] = {'pb': {'port': 9989}} @@ -63,32 +59,46 @@ let ####### Status - c['status'] = st = [] + c['services'] = [] # If you want to configure this url, override with extraConfig c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/" ${optionalString (cfg.web.enable) '' - from buildbot.status import html - from buildbot.status.web import authz, auth - authz_cfg=authz.Authz( - auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]), - # TODO: configure harder - gracefulShutdown = False, - forceBuild = 'auth', - forceAllBuilds = 'auth', - pingBuilder = False, - stopBuild = 'auth', - stopAllBuilds = 'auth', - cancelPendingBuild = 'auth' - ) + from buildbot.plugins import util + + #authz_cfg=authz.Authz( + # auth=auth.BasicAuth([ ]), + # # TODO: configure harder + # gracefulShutdown = False, + # forceBuild = 'auth', + # forceAllBuilds = 'auth', + # pingBuilder = False, + # stopBuild = 'auth', + # stopAllBuilds = 'auth', + # cancelPendingBuild = 'auth' + #) # TODO: configure krebs.nginx - st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg)) + c['www'] = dict( + port = ${toString cfg.web.port}, + plugins = { 'waterfall_view':{}, 'console_view':{} } + ) + c['www']['auth'] = util.UserPasswordAuth({"${cfg.web.username}":"${cfg.web.password}"}) + c['www']['authz'] = util.Authz( + allowRules = [ + util.StopBuildEndpointMatcher(role="admins"), + util.ForceBuildEndpointMatcher(role="admins"), + util.RebuildBuildEndpointMatcher(role="admins") + ], + roleMatchers = [ + util.RolesFromEmails(admins=["${cfg.web.username}"]) + ] + ) ''} ${optionalString (cfg.irc.enable) '' - from buildbot.status import words - irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}", + from buildbot.plugins import reporters + irc = reporters.IRC("${cfg.irc.server}", "${cfg.irc.nick}", channels=${builtins.toJSON cfg.irc.channels}, notify_events={ 'success': 1, @@ -97,7 +107,7 @@ let 'successToFailure': 1, 'failureToSuccess': 1, }${optionalString cfg.irc.allowForce ",allowForce=True"}) - c['status'].append(irc) + c['services'].append(irc) ''} ${ concatStringsSep "\n" @@ -150,12 +160,12 @@ let ''; }; - slaves = mkOption { + workers = mkOption { default = {}; type = types.attrsOf types.str; description = '' - Attrset of slavenames with their passwords - slavename = slavepassword + Attrset of workernames with their passwords + workername = workerpassword ''; }; @@ -283,8 +293,12 @@ let options = { enable = mkEnableOption "Buildbot Master IRC Status"; channels = mkOption { - default = [ "nix-buildbot-meetup" ]; - type = with types; listOf str; + default = [ { channel = "nix-buildbot-meetup";} ]; + example = literalExample ''[ + {channel = "nix-buildbot-meetup";} + {channel = "nix-buildbot-lol"; "password" = "lol";} + ]''; + type = with types; listOf (attrsOf str); description = '' irc channels the bot should connect to ''; @@ -333,7 +347,7 @@ let }; users.extraGroups.buildbotMaster = { - gid = 672626386; + gid = genid "buildbotMaster"; }; systemd.services.buildbotMaster = { @@ -350,8 +364,6 @@ let secretsdir = shell.escape (toString <secrets>); in { PermissionsStartOnly = true; - Type = "forking"; - PIDFile = "${workdir}/twistd.pid"; # TODO: maybe also prepare buildbot.tac? ExecStartPre = pkgs.writeDash "buildbot-master-init" '' set -efux @@ -375,9 +387,8 @@ let chmod 700 -R ${workdir} chown buildbotMaster:buildbotMaster -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; - ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; - ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; + ExecStart = "${buildbot}/bin/buildbot start --nodaemon ${workdir}"; + # ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; PrivateTmp = "true"; User = "buildbotMaster"; Restart = "always"; diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 650594a6c..95b547081 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -2,53 +2,21 @@ with import <stockholm/lib>; let - buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" '' - import os - - from buildslave.bot import BuildSlave - from twisted.application import service - - basedir = '${cfg.workDir}' - rotateLength = 10000000 - maxRotatedFiles = 10 - - application = service.Application('buildslave') - - from twisted.python.logfile import LogFile - from twisted.python.log import ILogObserver, FileLogObserver - logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, - maxRotatedFiles=maxRotatedFiles) - application.setComponent(ILogObserver, FileLogObserver(logfile).emit) - - buildmaster_host = '${cfg.masterhost}' - # TODO: masterport? - port = 9989 - slavename = '${cfg.username}' - passwd = '${cfg.password}' - keepalive = 600 - usepty = 0 - umask = None - maxdelay = 300 - allow_shutdown = None - - ${cfg.extraConfig} - - s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, - keepalive, usepty, umask=umask, maxdelay=maxdelay, - allow_shutdown=allow_shutdown) - s.setServiceParent(application) - ''; default-packages = [ pkgs.git pkgs.bash ]; - cfg = config.krebs.buildbot.slave; + buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-worker (old:{ + patches = [ ./buildbot-worker.patch ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ]; + }); + cfg = config.krebs.buildbot.worker; api = { - enable = mkEnableOption "Buildbot Slave"; + enable = mkEnableOption "Buildbot worker"; workDir = mkOption { - default = "/var/lib/buildbot/slave"; + default = "/var/lib/buildbot/worker"; type = types.str; description = '' - Path to build bot slave directory. + Path to build bot worker directory. Will be created on startup. ''; }; @@ -64,30 +32,30 @@ let username = mkOption { type = types.str; description = '' - slavename used to authenticate with master + workername used to authenticate with master ''; }; password = mkOption { type = types.str; description = '' - slave password used to authenticate with master + worker password used to authenticate with master ''; }; contact = mkOption { - default = "nix slave <buildslave@${config.networking.hostName}>"; + default = "nix worker <buildworker@${config.networking.hostName}>"; type = types.str; description = '' - contact to be announced by buildslave + contact to be announced by buildworker ''; }; description = mkOption { - default = "Nix Generated BuildSlave"; + default = "Nix Generated Buildworker"; type = types.str; description = '' - description for hostto be announced by buildslave + description for hostto be announced by buildworker ''; }; @@ -95,7 +63,7 @@ let default = [ pkgs.git ]; type = with types; listOf package; description = '' - packages which should be in path for buildslave + packages which should be in path for buildworker ''; }; @@ -106,7 +74,7 @@ let }; type = types.attrsOf types.str; description = '' - extra environment variables to be provided to the buildslave service + extra environment variables to be provided to the buildworker service if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here. ''; }; @@ -119,26 +87,26 @@ let keepalive = 600 ''; description = '' - extra config evaluated before calling BuildSlave init in .tac file + extra config evaluated before calling Buildworker init in .tac file ''; }; }; imp = { - users.extraUsers.buildbotSlave = { - uid = genid "buildbotSlave"; - description = "Buildbot Slave"; + users.extraUsers.buildbotworker = { + uid = genid "buildbotworker"; + description = "Buildbot worker"; home = cfg.workDir; createHome = false; }; - users.extraGroups.buildbotSlave = { - gid = 1408105834; + users.extraGroups.buildbotworker = { + gid = genid "buildbotworker"; }; - systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = { - description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}"; + systemd.services."buildbotworker-${cfg.username}-${cfg.masterhost}" = { + description = "Buildbot worker for ${cfg.username}@${cfg.masterhost}"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = default-packages ++ cfg.packages; @@ -152,27 +120,28 @@ let workdir = shell.escape cfg.workDir; contact = shell.escape cfg.contact; description = shell.escape cfg.description; - buildbot = pkgs.buildbot-slave; - # TODO:make this + masterhost = shell.escape cfg.masterhost; + username = shell.escape cfg.username; + password = shell.escape cfg.password; in { PermissionsStartOnly = true; Type = "forking"; PIDFile = "${workdir}/twistd.pid"; - # TODO: maybe also prepare buildbot.tac? - ExecStartPre = pkgs.writeDash "buildbot-master-init" '' + ExecStartPre = pkgs.writeDash "buildbot-slave-init" '' set -efux mkdir -p ${workdir}/info - cp ${buildbot-slave-init} ${workdir}/buildbot.tac + # TODO: cleanup .tac file? + ${buildbot}/bin/buildbot-worker create-worker ${workdir} ${masterhost} ${username} ${password} echo ${contact} > ${workdir}/info/admin echo ${description} > ${workdir}/info/host - chown buildbotSlave:buildbotSlave -R ${workdir} + chown buildbotworker:buildbotworker -R ${workdir} chmod 700 -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildslave start ${workdir}"; - ExecStop = "${buildbot}/bin/buildslave stop ${workdir}"; + ExecStart = "${buildbot}/bin/buildbot-worker start ${workdir}"; + ExecStop = "${buildbot}/bin/buildbot-worker stop ${workdir}"; PrivateTmp = "true"; - User = "buildbotSlave"; + User = "buildbotworker"; Restart = "always"; RestartSec = "10"; }; @@ -180,6 +149,6 @@ let }; in { - options.krebs.buildbot.slave = api; + options.krebs.buildbot.worker = api; config = lib.mkIf cfg.enable imp; } diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix index a483db32c..242373ced 100644 --- a/krebs/5pkgs/Reaktor/plugins.nix +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -59,7 +59,7 @@ rec { }; sed-plugin = buildSimpleReaktorPlugin "sed-plugin" { - path = [ pkgs.gnused pkgs.python3 ]; + path = [ pkgs.gnused pkgs.proot pkgs.python3 ]; # only support s///gi the plugin needs to see every msg # TODO: this will eat up the last regex, fix Reaktor to support fallthru append_rule = true; diff --git a/krebs/5pkgs/Reaktor/scripts/sed-plugin.py b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py index 8103c9585..6039aeb43 100644 --- a/krebs/5pkgs/Reaktor/scripts/sed-plugin.py +++ b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py @@ -34,9 +34,22 @@ if m: flagstr = '' last = d.get(usr,None) if last: - #print(re.sub(fn,tn,last,count=count,flags=flags)) from subprocess import Popen,PIPE - p = Popen(['sed','s/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE ) + import shutil + from os.path import realpath + # sed only needs stdin/stdout, we protect state_dir with this + # input to read/write arbitrary files: + # s/.\/\/; w /tmp/i (props to waldi) + # conclusion: sed is untrusted and we handle it like this + p = Popen(['proot', + # '-v','1', + '-w','/', # cwd is root + '-b','/nix/store', # mount important folders + '-b','/usr', + '-b','/bin', + '-r','/var/empty', # chroot to /var/empty + realpath(shutil.which('sed')), + 's/{}/{}/{}'.format(f,t,flagstr)],stdin=PIPE,stdout=PIPE ) so,se = p.communicate(bytes("{}\n".format(last),"UTF-8")) if p.returncode: print("something went wrong when trying to process your regex: {}".format(se.decode())) diff --git a/krebs/5pkgs/buildbot/default.nix b/krebs/5pkgs/buildbot/default.nix new file mode 100644 index 000000000..a0e6bb6a5 --- /dev/null +++ b/krebs/5pkgs/buildbot/default.nix @@ -0,0 +1,81 @@ +{ pkgs, stdenv, pythonPackages, fetchurl, coreutils, plugins ? [] }: + +pythonPackages.buildPythonApplication (rec { + name = "${pname}-${version}"; + pname = "buildbot"; + version = "0.9.1"; + src = fetchurl { + url = "mirror://pypi/b/${pname}/${name}.tar.gz"; + sha256 = "1kk4dlkk4rznwid9xykq2lbzksvkcr4r5kmz9hgh5hswdzv8bwx9"; + }; + doCheck = false; + buildInputs = with pythonPackages; [ + lz4 + txrequests + pyjade + boto3 + moto + txgithub + mock + setuptoolsTrial + isort + pylint + astroid + pyflakes + ]; + + propagatedBuildInputs = with pythonPackages; [ + + # core + twisted + jinja2 + zope_interface + future + sqlalchemy + sqlalchemy_migrate + future + dateutil + txaio + autobahn + + # tls + pyopenssl + service-identity + idna + pkgs.treq + + # docs + sphinx + sphinxcontrib-blockdiag + sphinxcontrib-spelling + pyenchant + docutils + ramlfications + sphinx-jinja + + ] ++ plugins; + + preInstall = '' + # writes out a file that can't be read properly + sed -i.bak -e '69,84d' buildbot/test/unit/test_www_config.py + + # re-hardcode path to tail + sed -i.bak 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py + ''; + + postFixup = '' + mv -v $out/bin/buildbot $out/bin/.wrapped-buildbot + echo "#!/bin/sh" > $out/bin/buildbot + echo "export PYTHONPATH=$PYTHONPATH" >> $out/bin/buildbot + echo "exec $out/bin/.wrapped-buildbot \"\$@\"" >> $out/bin/buildbot + chmod -c 555 $out/bin/buildbot + ''; + + meta = with stdenv.lib; { + homepage = http://buildbot.net/; + description = "Continuous integration system that automates the build/test cycle"; + maintainers = with maintainers; [ nand0p ryansydnor ]; + platforms = platforms.all; + license = licenses.gpl2; + }; +}) diff --git a/krebs/5pkgs/buildbot/worker.nix b/krebs/5pkgs/buildbot/worker.nix new file mode 100644 index 000000000..c100de5d2 --- /dev/null +++ b/krebs/5pkgs/buildbot/worker.nix @@ -0,0 +1,24 @@ +{ pkgs, stdenv, fetchurl, pythonPackages }: +pythonPackages.buildPythonApplication (rec { + name = "${pname}-${version}"; + pname = "buildbot-worker"; + version = "0.9.1"; + + doCheck = false; + src = fetchurl { + url = "mirror://pypi/b/${pname}/${name}.tar.gz"; + sha256 = "00p9l1qz6mx12npjwsycp8f9a8f2har15ig79pfsg8z7a7yw93hx"; + }; + + buildInputs = with pythonPackages; [ setuptoolsTrial mock ]; + propagatedBuildInputs = with pythonPackages; [ twisted future pkgs.treq ]; + + meta = with stdenv.lib; { + homepage = http://buildbot.net/; + description = "Buildbot Worker Daemon"; + maintainers = with maintainers; [ nand0p ryansydnor ]; + platforms = platforms.all; + license = licenses.gpl2; + }; +}) + diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix index 876f8b9a4..c1ec0f333 100644 --- a/krebs/5pkgs/default.nix +++ b/krebs/5pkgs/default.nix @@ -38,6 +38,12 @@ with import <stockholm/lib>; ReaktorPlugins = callPackage ./Reaktor/plugins.nix {}; + buildbot = callPackage ./buildbot {}; + buildbot-full = callPackage ./buildbot { + plugins = with pkgs.buildbot-plugins; [ www console-view waterfall-view ]; + }; + buildbot-worker = callPackage ./buildbot/worker.nix {}; + # XXX symlinkJoin changed arguments somewhere around nixpkgs d541e0d symlinkJoin = { name, paths, ... }@args: let x = pkgs.symlinkJoin args; diff --git a/krebs/5pkgs/treq/default.nix b/krebs/5pkgs/treq/default.nix new file mode 100644 index 000000000..20387b9cb --- /dev/null +++ b/krebs/5pkgs/treq/default.nix @@ -0,0 +1,17 @@ +{ stdenv, fetchurl, pythonPackages }: + +pythonPackages.buildPythonPackage rec { + name = "${pname}-${version}"; + pname = "treq"; + version = "15.1.0"; + src = fetchurl { + url = "mirror://pypi/t/${pname}/${name}.tar.gz"; + sha256= "425a47d5d52a993d51211028fb6ade252e5fbea094e878bb4b644096a7322de8"; + }; + propagatedBuildInputs = with pythonPackages; [ + twisted + pyopenssl + requests2 + service-identity + ]; +} diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix index 4397bf786..cf3354fd7 100644 --- a/lass/2configs/buildbot-standalone.nix +++ b/lass/2configs/buildbot-standalone.nix @@ -162,7 +162,7 @@ in { enable = true; nick = "buildbot-lass"; server = "ni.r"; - channels = [ "retiolum" ]; + channels = [ { channels = "retiolum"; } ]; allowForce = true; }; }; diff --git a/makefu/1systems/vbob.nix b/makefu/1systems/vbob.nix index 56d1b01ba..7421125e4 100644 --- a/makefu/1systems/vbob.nix +++ b/makefu/1systems/vbob.nix @@ -8,12 +8,12 @@ (toString <nixpkgs/nixos/modules/virtualisation/virtualbox-image.nix>) (toString <nixpkgs/nixos/modules/virtualisation/virtualbox-guest.nix>) ../2configs/main-laptop.nix #< base-gui - # <secrets/extra-hosts.nix> # environment ../2configs/tinc/retiolum.nix ]; + networking.extraHosts = import (toString <secrets/extra-hosts.nix>); # workaround for https://github.com/NixOS/nixpkgs/issues/16641 services.xserver.videoDrivers = lib.mkOverride 45 [ "virtualbox" "modesetting" ]; @@ -41,8 +41,10 @@ get logstash # docker - devpi-web - devpi-client + #devpi-web + #devpi-client + debmirror + ansible ]; # virtualisation.docker.enable = true; diff --git a/makefu/2configs/graphite-standalone.nix b/makefu/2configs/graphite-standalone.nix index 15ae6b68f..51c4c9561 100644 --- a/makefu/2configs/graphite-standalone.nix +++ b/makefu/2configs/graphite-standalone.nix @@ -9,7 +9,7 @@ with import <stockholm/lib>; services.graphite = { web = { enable = true; - host = "0.0.0.0"; + listenAddress = "0.0.0.0"; }; carbon = { enableCache = true; diff --git a/makefu/2configs/save-diskspace.nix b/makefu/2configs/save-diskspace.nix index cc2b29cac..4fd569768 100644 --- a/makefu/2configs/save-diskspace.nix +++ b/makefu/2configs/save-diskspace.nix @@ -1,9 +1,11 @@ _: # TODO: do not check out nixpkgs master but fetch revision from github { - services.nixosManual.enable = false; - programs.man.enable = false; - services.journald.extraConfig = "SystemMaxUse=50M"; + environment.noXlibs = true; nix.gc.automatic = true; nix.gc.dates = "03:10"; + programs.info.enable = false; + programs.man.enable = false; + services.journald.extraConfig = "SystemMaxUse=50M"; + services.nixosManual.enable = false; } diff --git a/makefu/5pkgs/debmirror/default.nix b/makefu/5pkgs/debmirror/default.nix new file mode 100644 index 000000000..67b34b6ee --- /dev/null +++ b/makefu/5pkgs/debmirror/default.nix @@ -0,0 +1,41 @@ +{ stdenv, pkgs, fetchgit }: + +pkgs.perlPackages.buildPerlPackage rec { + name = "debmirror-${version}"; + version = "2.25"; + + enableParallelBuilding = true; + + src = fetchgit { + url = "https://anonscm.debian.org/git/collab-maint/debmirror.git"; + rev = "c77e5caa15a4ab6497db5d819614387e647ccf4e"; + sha256 = "1zp8ff9ajw22b4wradnw1hnfcpbyx5ibqzqgk6kp79nsj1dzmm0d"; + }; + preConfigure = '' + touch Makefile.PL + ''; + + outputs = [ "out" ]; + + buildPhase = '' + make + ''; + + doCheck = false; + + installPhase = '' + mkdir -p $out/bin $out/share/man/man1/ + cp debmirror mirror-size $out/bin + cp debmirror.1 $out/share/man/man1/ + ''; + propagatedBuildInputs = (with pkgs.perlPackages; [ LockFileSimple LWP]) ++ + (with pkgs; [ rsync patch ed gzip diffutils findutils gnupg1 xz ]); + + meta = { + description = "mirror apt repos"; + homepage = https://tracker.debian.org/pkg/debmirror; + license = stdenv.lib.licenses.gpl2; + platforms = stdenv.lib.platforms.linux; + maintainers = with stdenv.lib.maintainers; [ makefu ]; + }; +} diff --git a/makefu/5pkgs/default.nix b/makefu/5pkgs/default.nix index 0d375a510..8994b3856 100644 --- a/makefu/5pkgs/default.nix +++ b/makefu/5pkgs/default.nix @@ -10,7 +10,9 @@ in alsa-hdsploader = callPackage ./alsa-tools { alsaToolTarget="hdsploader";}; awesomecfg = callPackage ./awesomecfg {}; bintray-upload = callPackage ./bintray-upload {}; - inherit (callPackage ./devpi {}) devpi-web devpi-server; + debmirror = callPackage ./debmirror {}; + inherit (callPackage ./devpi {}) devpi-web devpi-server devpi-client; + elchhub = callPackage ./elchhub {}; f3 = callPackage ./f3 {}; farpd = callPackage ./farpd {}; git-xlsx-textconv = callPackage ./git-xlsx-textconv {}; diff --git a/shared/1systems/test-all-krebs-modules.nix b/shared/1systems/test-all-krebs-modules.nix index 60d5f3252..0bfcff685 100644 --- a/shared/1systems/test-all-krebs-modules.nix +++ b/shared/1systems/test-all-krebs-modules.nix @@ -16,7 +16,7 @@ in { # FIXME fast-tests / instantiate-test-all-modules fails at wolfbot # http://wolf:8010/builders/fast-tests/builds/442 #buildbot.master.enable = true; - buildbot.slave = { + buildbot.worker = { enable = true; username = "lol"; password = "wut"; diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix index 7aed6272c..cf08882a9 100644 --- a/shared/2configs/shared-buildbot.nix +++ b/shared/2configs/shared-buildbot.nix @@ -11,14 +11,13 @@ # /nix/store should be cleaned up automatically as well nix.gc.automatic = true; nix.gc.dates = "05:23"; - networking.firewall.allowedTCPPorts = [ 8010 9989 ]; krebs.buildbot.master = let stockholm-mirror-url = http://cgit.wolf/stockholm-mirror ; in { secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; - slaves = { - testslave = "krebspass"; + workers = { + testworker = "krebspass"; }; change_source.stockholm = '' stockholm_repo = '${stockholm-mirror-url}' @@ -40,9 +39,7 @@ ''; fast-tests-scheduler = '' # test everything real quick - sched.append(schedulers.SingleBranchScheduler( - ## all branches - change_filter=util.ChangeFilter(branch_re=".*"), + sched.append(schedulers.AnyBranchScheduler( treeStableTimer=10, name="fast-all-branches", builderNames=["fast-tests"])) @@ -109,7 +106,7 @@ system={}".format(i)]) bu.append(util.BuilderConfig(name="fast-tests", - slavenames=slavenames, + workernames=workernames, factory=f)) ''; @@ -119,36 +116,27 @@ f = util.BuildFactory() f.addStep(grab_repo) - for i in [ "test-all-krebs-modules", "wolf" ]: - addShell(f,name="build-{}".format(i),env=env, - command=nixshell + \ - ["mkdir -p /tmp/testbuild/$LOGNAME && touch /tmp/testbuild/$LOGNAME/.populate; \ - make \ - test \ - target=$LOGNAME@${config.krebs.build.host.name}/tmp/testbuild/$LOGNAME \ - method=build \ - system={}".format(i)]) bu.append(util.BuilderConfig(name="build-local", - slavenames=slavenames, + workernames=workernames, factory=f)) ''; # slow-tests = '' # s = util.BuildFactory() # s.addStep(grab_repo) # -# # slave needs 2 files: +# # worker needs 2 files: # # * cac.json # # * retiolum -# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json")) -# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv")) +# s.addStep(steps.FileDownload(mastersrc="${config.k |