k 3 buildbot.master: refactor

see buildbot-standalone.nix in shared/2configs for the current buildbot config

2 files changed, 317 insertions, 131 deletions

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 6bf3fda2c..7078000fe 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -7,134 +7,81 @@ let
     # -*- python -*-
     from buildbot.plugins import *
     import re
-
+    import json
     c = BuildmasterConfig = {}
 
     c['slaves'] = []
-    # TODO: template potential buildslaves
-    # TODO: set password?
-    slavenames= [ 'testslave' ]
-    for i in slavenames:
-      c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
+    slaves = json.loads('${builtins.toJSON cfg.slaves}')
+    slavenames = [ s for s in slaves ]
+    for k,v in slaves.items():
+      c['slaves'].append(buildslave.BuildSlave(k, v))
 
+    # TODO: configure protocols?
     c['protocols'] = {'pb': {'port': 9989}}
 
     ####### Build Inputs
-    stockholm_repo = 'http://cgit.gum/stockholm'
-    c['change_source'] = []
-    c['change_source'].append(changes.GitPoller(
-            stockholm_repo,
-            workdir='stockholm-poller', branch='master',
-            project='stockholm',
-            pollinterval=120))
+    c['change_source'] = cs = []
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Change_Source: Begin of ${n}
+        ${v}
+        #### Change_Source: End of ${n}
+      '') cfg.change_source )}
 
     ####### Build Scheduler
-    # TODO: configure scheduler
-    c['schedulers'] = []
-
-    # test the master real quick
-    fast = schedulers.SingleBranchScheduler(
-                                change_filter=util.ChangeFilter(branch="master"),
-                                name="fast-master-test",
-                                builderNames=["fast-tests"])
-
-    force = schedulers.ForceScheduler(
-                                name="force",
-                                builderNames=["full-tests"])
-
-    # files everyone depends on or are part of the share branch
-    def shared_files(change):
-      r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
-      for file in change.files:
-        if r.match(file):
-          return True
-      return False
-
-    full = schedulers.SingleBranchScheduler(
-                                change_filter=util.ChangeFilter(branch="master"),
-                                fileIsImportant=shared_files,
-                                name="full-master-test",
-                                builderNames=["full-tests"])
-    c['schedulers'] = [ fast, force, full ]
-    ###### The actual build
-    # couple of fast steps:
-    f = util.BuildFactory()
-    # some slow steps
-    s = util.BuildFactory()
-    ## fetch repo
-    grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
-    f.addStep(grab_repo)
-    s.addStep(grab_repo)
-
-    # the dependencies which are used by the test script
-    deps = [ "gnumake", "jq","nix",
-             "(import <stockholm> {}).pkgs.test.infest-cac-centos7",
-             "rsync" ]
-    # TODO: --pure , prepare ENV in nix-shell command:
-    #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
-    nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
-    env = {"LOGNAME": "shared",
-           "NIX_REMOTE": "daemon"}
-    def addShell(f,**kwargs):
-      f.addStep(steps.ShellCommand(**kwargs))
-
-    addShell(f,name="centos7-eval",env=env,
-             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
-
-    addShell(f,name="wolf-eval",env=env,
-             command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
-
-    addShell(f,name="eval-cross-check",env=env,
-             command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
-
-    c['builders'] = []
-    c['builders'].append(
-        util.BuilderConfig(name="fast-tests",
-          slavenames=slavenames,
-          factory=f))
-
-    # slave needs 2 files:
-    # * cac.json
-    # * retiolum
-    for file in ["cac.json", "retiolum.rsa_key.priv"]:
-      s.addStep(steps.FileDownload(mastersrc="${cfg.workDir}/{}".format(file),
-                              slavedest=file))
-
-    addShell(s, name="infest-cac-centos7",env=env,
-                sigtermTime=60, # SIGTERM 1 minute before SIGKILL
-                timeout=5400,   # 1.5h timeout
-                command=nixshell + ["infest-cac-centos7"])
-
-    c['builders'].append(
-        util.BuilderConfig(name="full-tests",
-          slavenames=slavenames,
-          factory=s))
-
-    ####### Status of Builds
-    c['status'] = []
-
-    from buildbot.status import html
-    from buildbot.status.web import authz, auth
-    # TODO: configure if http is wanted
-    authz_cfg=authz.Authz(
-        # TODO: configure user/pw
-        auth=auth.BasicAuth([("krebs","bob")]),
-        gracefulShutdown = False,
-        forceBuild = 'auth',
-        forceAllBuilds = 'auth',
-        pingBuilder = False,
-        stopBuild = 'auth',
-        stopAllBuilds = False,
-        cancelPendingBuild = False,
-    )
-    # TODO: configure nginx
-    c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
-
-    from buildbot.status import words
+    c['schedulers'] = sched = []
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Schedulers: Begin of ${n}
+        ${v}
+        #### Schedulers: End of ${n}
+      '') cfg.scheduler )}
+
+    ###### Builder
+    c['builders'] = bu = []
+    
+    # Builder Pre: Begin
+    ${cfg.builder_pre}
+    # Builder Pre: End
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Builder: Begin of ${n}
+        ${v}
+        #### Builder: End of ${n}
+      '') cfg.builder )}
+
+
+    ####### Status
+    c['status'] = st = []
+
+    # If you want to configure this url, override with extraConfig
+    c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
+
+    ${optionalString (cfg.web.enable) ''
+      from buildbot.status import html
+      from buildbot.status.web import authz, auth
+      authz_cfg=authz.Authz(
+          auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
+          # TODO: configure harder
+          gracefulShutdown = False,
+          forceBuild = 'auth',
+          forceAllBuilds = 'auth',
+          pingBuilder = False,
+          stopBuild = 'auth',
+          stopAllBuilds = 'auth',
+          cancelPendingBuild = 'auth'
+      )
+      # TODO: configure krebs.nginx
+      st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
+      ''}
+
     ${optionalString (cfg.irc.enable) ''
-      irc = words.IRC("${cfg.irc.server}", "krebsbuild",
-                      # TODO: multiple channels
-                      channels=["${cfg.irc.channel}"],
+      from buildbot.status import words
+      irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
+                      channels=${builtins.toJSON cfg.irc.channels},
                       notify_events={
                         'success': 1,
                         'failure': 1,
@@ -145,15 +92,20 @@ let
       c['status'].append(irc)
       ''}
 
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Status: Begin of ${n}
+        ${v}
+        #### Status: End of ${n}
+      '') cfg.status )}
+
     ####### PROJECT IDENTITY
-    c['title'] = "Stockholm"
+    c['title'] = "${cfg.title}"
     c['titleURL'] = "http://krebsco.de"
 
-    #c['buildbotURL'] = "http://buildbot.krebsco.de/"
-    # TODO: configure url
-    c['buildbotURL'] = "http://vbob:8010/"
 
     ####### DB URL
+    # TODO: configure
     c['db'] = {
         'db_url' : "sqlite:///state.sqlite",
     }
@@ -164,6 +116,13 @@ let
 
   api = {
     enable = mkEnableOption "Buildbot Master";
+    title = mkOption {
+      default = "Buildbot CI";
+      type = types.str;
+      description = ''
+        Title of the Buildbot Installation
+      '';
+    };
     workDir = mkOption {
       default = "/var/lib/buildbot/master";
       type = types.str;
@@ -172,16 +131,144 @@ let
         Will be created on startup.
       '';
     };
+
+    slaves = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      description = ''
+        Attrset of slavenames with their passwords
+        slavename = slavepassword
+      '';
+    };
+
+    change_source = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        stockholm = ''
+          cs.append(changes.GitPoller(
+                  'http://cgit.gum/stockholm',
+                  workdir='stockholm-poller', branch='master',
+                  project='stockholm',
+                  pollinterval=120))
+        '';
+      };
+      description = ''
+        Attrset of all the change_sources which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>cs</literal>
+      '';
+    };
+
+    scheduler = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        force-scheduler = ''
+          sched.append(schedulers.ForceScheduler(
+                                      name="force",
+                                      builderNames=["full-tests"]))
+        '';
+      };
+      description = ''
+        Attrset of all the schedulers which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>sched</literal>
+      '';
+    };
+
+    builder_pre = mkOption {
+      default = "";
+      type = types.lines;
+      example = ''
+        grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+      '';
+      description = ''
+        some code before the builders are being assembled.
+        can be used to define functions used by multiple builders
+      '';
+    };
+
+    builder = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        fast-test = ''
+        '';
+      };
+      description = ''
+        Attrset of all the builder which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>bu</literal>
+      '';
+    };
+
+    status = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      description = ''
+        Attrset of all the extra status which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>st</literal>
+
+        Right now IRC and Web status can be configured by setting
+        <literal>buildbot.master.irc.enable</literal> and
+        <literal>buildbot.master.web.enable</literal>
+      '';
+    };
+
+    # Configurable Stati
+    web = mkOption {
+      default = {};
+      type = types.submodule ({ config2, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master Web Status";
+          username = mkOption {
+            default = "krebs";
+            type = types.str;
+            description = ''
+              username for web authentication
+            '';
+          };
+          hostname = mkOption {
+            default = config.networking.hostName;
+            type = types.str;
+            description = ''
+              web interface Hostname
+            '';
+          };
+          password = mkOption {
+            default = "bob";
+            type = types.str;
+            description = ''
+              password for web authentication
+            '';
+          };
+          port = mkOption {
+            default = 8010;
+            type = types.int;
+            description = ''
+              port for buildbot web status
+            '';
+          };
+        };
+      });
+    };
+
     irc = mkOption {
       default = {};
       type = types.submodule ({ config, ... }: {
         options = {
           enable = mkEnableOption "Buildbot Master IRC Status";
-          channel = mkOption {
-            default = "nix-buildbot-meetup";
-            type = types.str;
+          channels = mkOption {
+            default = [ "nix-buildbot-meetup" ];
+            type = with types; listOf str;
             description = ''
-              irc channel the bot should connect to
+              irc channels the bot should connect to
             '';
           };
           allowForce = mkOption {
@@ -235,6 +322,7 @@ let
       description = "Buildbot Master";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
+      # TODO: add extra dependencies to master like svn and cvs
       path = [ pkgs.git ];
       environment = {
         SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
@@ -242,7 +330,6 @@ let
       serviceConfig = let
         workdir="${lib.shell.escape cfg.workDir}";
         secretsdir="${lib.shell.escape (toString <secrets>)}";
-        # TODO: check if git is the only dep
       in {
         PermissionsStartOnly = true;
         Type = "forking";
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index adf44cada..baab059c9 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -8,16 +8,115 @@ in {
   };
   networking.firewall.allowedTCPPorts = [ 8010 ];
   krebs.buildbot.master = {
+    slaves = {
+      testslave =  "krebspass";
+      testslave2 = "krebspass";
+    };
+    change_source.stockholm = ''
+  stockholm_repo = 'http://cgit.gum/stockholm'
+  cs.append(changes.GitPoller(
+          stockholm_repo,
+          workdir='stockholm-poller', branch='master',
+          project='stockholm',
+          pollinterval=120))
+    '';
+    scheduler = {
+        force-scheduler = ''
+  sched.append(schedulers.ForceScheduler(
+                              name="force",
+                              builderNames=["full-tests"]))
+        '';
+        fast-tests-scheduler = ''
+  # test the master real quick
+  sched.append(schedulers.SingleBranchScheduler(
+                              change_filter=util.ChangeFilter(branch="master"),
+                              name="fast-master-test",
+                              builderNames=["fast-tests"]))
+        '';
+        full-master-scheduler = ''
+  # files everyone depends on or are part of the share branch
+  def shared_files(change):
+    r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
+    for file in change.files:
+      if r.match(file):
+        return True
+    return False
+
+  sched.append(schedulers.SingleBranchScheduler(
+                              change_filter=util.ChangeFilter(branch="master"),
+                              fileIsImportant=shared_files,
+                              name="full-master-test",
+                              builderNames=["full-tests"]))
+        '';
+    };
+    builder_pre = ''
+  # prepare grab_repo step for stockholm
+  stockholm_repo = "http://cgit.gum.retiolum/stockholm"
+  grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+
+  env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"}
+
+  # prepare nix-shell
+  # the dependencies which are used by the test script
+  deps = [ "gnumake", "jq","nix","rsync",
+            "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
+  # TODO: --pure , prepare ENV in nix-shell command:
+  #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
+  nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
+
+  # prepare addShell function
+  def addShell(factory,**kwargs):
+    factory.addStep(steps.ShellCommand(**kwargs))
+    '';
+    builder = {
+      fast-tests = ''
+  f = util.BuildFactory()
+  f.addStep(grab_repo)
+  addShell(f,name="centos7-eval",env=env,
+            command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
+
+  addShell(f,name="wolf-eval",env=env,
+            command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
+
+  addShell(f,name="eval-cross-check",env=env,
+            command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
+
+  bu.append(util.BuilderConfig(name="fast-tests",
+        slavenames=slavenames,
+        factory=f))
+      '';
+      slow-tests = ''
+  s = util.BuildFactory()
+  s.addStep(grab_repo)
+
+  # slave needs 2 files:
+  # * cac.json
+  # * retiolum
+  for file in ["cac.json", "retiolum.rsa_key.priv"]:
+    s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/{}".format(file),
+                            slavedest=file))
+
+  addShell(s, name="infest-cac-centos7",env=env,
+              sigtermTime=60,           # SIGTERM 1 minute before SIGKILL
+              timeout=5400,             # 1.5h timeout
+              command=nixshell + ["infest-cac-centos7"])
+
+  bu.append(util.BuilderConfig(name="full-tests",
+        slavenames=slavenames,
+        factory=s))
+      '';
+    };
     enable = true;
+    web = {
+      enable = true;
+    };
     irc = {
       enable = true;
+      nick = "shared-buildbot";
       server = "cd.retiolum";
-      channel = "retiolum";
+      channels = [ "retiolum" ];
       allowForce = true;
     };
-    extraConfig = ''
-      c['buildbotURL']  = "http://${config.krebs.build.host.name}:8010/"
-    '';
   };
 
   krebs.buildbot.slave = {