ci: build steps dynamically, cleanupprism/staging/ci

6 files changed, 179 insertions, 216 deletions

diff --git a/krebs/1systems/hotdog/config.nix b/krebs/1systems/hotdog/config.nix
index 662e094d1..dc13b03a8 100644
--- a/krebs/1systems/hotdog/config.nix
+++ b/krebs/1systems/hotdog/config.nix
@@ -9,7 +9,7 @@
     <stockholm/krebs>
     <stockholm/krebs/2configs>
 
-    <stockholm/krebs/2configs/buildbot-all.nix>
+    <stockholm/krebs/2configs/buildbot-stockholm.nix>
     <stockholm/krebs/2configs/gitlab-runner-shackspace.nix>
     <stockholm/krebs/2configs/binary-cache/nixos.nix>
     <stockholm/krebs/2configs/ircd.nix>
@@ -20,6 +20,5 @@
 
   boot.isContainer = true;
   networking.useDHCP = false;
-  krebs.ci.stockholmSrc = "http://cgit.prism.r/stockholm";
   environment.variables.NIX_REMOTE = "daemon";
 }
diff --git a/krebs/2configs/buildbot-all.nix b/krebs/2configs/buildbot-all.nix
deleted file mode 100644
index d85cde175..000000000
--- a/krebs/2configs/buildbot-all.nix
+++ /dev/null
@@ -1,9 +0,0 @@
-with import <stockholm/lib>;
-{ lib, config, pkgs, ... }:
-{
-  networking.firewall.allowedTCPPorts = [ 80 8010 9989 ];
-  krebs.ci.enable = true;
-  krebs.ci.treeStableTimer = 1;
-  krebs.ci.hosts = filter (getAttr "ci") (attrValues config.krebs.hosts);
-}
-
diff --git a/krebs/2configs/buildbot-krebs.nix b/krebs/2configs/buildbot-krebs.nix
deleted file mode 100644
index a09b3b98b..000000000
--- a/krebs/2configs/buildbot-krebs.nix
+++ /dev/null
@@ -1,12 +0,0 @@
-with import <stockholm/lib>;
-{ lib, config, pkgs, ... }:
-{
-  imports = [
-    <stockholm/krebs/2configs/repo-sync.nix>
-  ];
-
-  networking.firewall.allowedTCPPorts = [ 80 8010 9989 ];
-  krebs.ci.enable = true;
-  krebs.ci.treeStableTimer = 120;
-  krebs.ci.hosts = [ config.krebs.build.host ];
-}
diff --git a/krebs/2configs/buildbot-stockholm.nix b/krebs/2configs/buildbot-stockholm.nix
new file mode 100644
index 000000000..04b1c999f
--- /dev/null
+++ b/krebs/2configs/buildbot-stockholm.nix
@@ -0,0 +1,178 @@
+{ config, pkgs, ... }: with import <stockholm/lib>;
+
+let
+
+  hostname = config.networking.hostName;
+
+in
+{
+  networking.firewall.allowedTCPPorts = [ 80 ];
+  services.nginx = {
+    enable = true;
+    virtualHosts.build = {
+      serverAliases = [ "build.${hostname}.r" ];
+      locations."/".extraConfig = ''
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port};
+      '';
+    };
+  };
+
+  krebs.buildbot.master = {
+    slaves = {
+      testslave = "lasspass";
+    };
+    change_source.stockholm = ''
+      stockholm_repo = 'http://cgit.prism.r/stockholm'
+      cs.append(
+          changes.GitPoller(
+              stockholm_repo,
+              workdir='stockholm-poller', branches=True,
+              project='stockholm',
+              pollinterval=10
+          )
+      )
+    '';
+    scheduler = {
+      auto-scheduler = ''
+        sched.append(
+              schedulers.SingleBranchScheduler(
+                  change_filter=util.ChangeFilter(branch_re=".*"),
+                  treeStableTimer=60,
+                  name="build-all-branches",
+                  builderNames=[
+                      "hosts",
+                  ]
+              )
+        )
+      '';
+      force-scheduler = ''
+        sched.append(
+            schedulers.ForceScheduler(
+                  name="hosts",
+                  builderNames=[
+                      "hosts",
+                  ]
+            )
+        )
+      '';
+    };
+    builder_pre = ''
+      # prepare grab_repo step for stockholm
+      grab_repo = steps.Git(
+          repourl=stockholm_repo,
+          mode='full',
+      )
+    '';
+    builder = {
+      hosts = ''
+        from buildbot import interfaces
+        from buildbot.steps.shell import ShellCommand
+
+        class StepToStartMoreSteps(ShellCommand):
+            def __init__(self, **kwargs):
+                ShellCommand.__init__(self, **kwargs)
+
+            def addBuildSteps(self, steps_factories):
+                for sf in steps_factories:
+                    step = interfaces.IBuildStepFactory(sf).buildStep()
+                    step.setBuild(self.build)
+                    step.setBuildSlave(self.build.slavebuilder.slave)
+                    step_status = self.build.build_status.addStepWithName(step.name)
+                    step.setStepStatus(step_status)
+                    self.build.steps.append(step)
+
+            def start(self):
+                props = self.build.getProperties()
+                hosts = json.loads(props.getProperty('hosts_json'))
+                for host in hosts:
+                    user = hosts[host]['owner']
+
+                    self.addBuildSteps([steps.ShellCommand(
+                        name=str(host),
+                        env={
+                          "NIX_PATH": "secrets=/var/src/stockholm/null:stockholm=./:/var/src",
+                          "NIX_REMOTE": "daemon",
+                          "dummy_secrets": "true",
+                        },
+                        command=[
+                          "nix-shell", "-I", "stockholm=.", "--run", " ".join(["test",
+                            "--user={}".format(user),
+                            "--system={}".format(host),
+                            "--force-populate",
+                            "--target=$LOGNAME@${config.krebs.build.host.name}$HOME/{}".format(user),
+                          ])
+                        ],
+                        timeout=90001,
+                        workdir='build', # TODO figure out why we need this?
+                    )])
+
+                ShellCommand.start(self)
+
+
+        f = util.BuildFactory()
+        f.addStep(grab_repo)
+
+        f.addStep(steps.SetPropertyFromCommand(
+            env={
+              "NIX_PATH": "secrets=/var/src/stockholm/null:stockholm=./:/var/src",
+              "NIX_REMOTE": "daemon",
+            },
+            name="get_hosts",
+            command=["nix-instantiate", "--json", "--strict", "--eval", "-E", """
+                with import <nixpkgs> {};
+                let
+                  eval-config = cfg:
+                    import <nixpkgs/nixos/lib/eval-config.nix> {
+                      modules = [
+                        (import cfg)
+                      ];
+                    }
+                  ;
+
+                  system = eval-config ./krebs/1systems/hotdog/config.nix; # TODO put a better config here
+
+                  ci-systems = lib.filterAttrs (_: v: v.ci) system.config.krebs.hosts;
+
+                  filtered-attrs = lib.mapAttrs ( n: v: {
+                    owner = v.owner.name;
+                  }) ci-systems;
+
+                in filtered-attrs
+            """],
+            property="hosts_json"
+        ))
+        f.addStep(StepToStartMoreSteps(command=["echo"])) # TODO remove dummy command from here
+
+        bu.append(
+            util.BuilderConfig(
+                name="hosts",
+                slavenames=slavenames,
+                factory=f
+            )
+        )
+      '';
+    };
+    enable = true;
+    web.enable = true;
+    irc = {
+      enable = true;
+      nick = "build|${hostname}";
+      server = "irc.r";
+      channels = [ "noise" "xxx" ];
+      allowForce = true;
+    };
+    extraConfig = ''
+      c['buildbotURL'] = "http://build.${hostname}.r/"
+    '';
+  };
+
+  krebs.buildbot.slave = {
+    enable = true;
+    masterhost = "localhost";
+    username = "testslave";
+    password = "lasspass";
+    packages = with pkgs; [ gnumake jq nix populate ];
+  };
+}
diff --git a/krebs/3modules/ci.nix b/krebs/3modules/ci.nix
deleted file mode 100644
index e97aa16eb..000000000
--- a/krebs/3modules/ci.nix
+++ /dev/null
@@ -1,192 +0,0 @@
-{ config, pkgs, ... }:
-with import <stockholm/lib>;
-let
-  cfg = config.krebs.ci;
-
-  hostname = config.networking.hostName;
-in
-{
-  options.krebs.ci = {
-    enable = mkEnableOption "krebs continous integration";
-    stockholmSrc = mkOption {
-      type = types.str;
-      default = "http://cgit.${hostname}.r/stockholm";
-    };
-    treeStableTimer = mkOption {
-      type = types.int;
-      default = 10;
-      description = "how long to wait until we test changes (in minutes)";
-    };
-    hosts = mkOption {
-      type = types.listOf types.host;
-      default = [];
-      description = ''
-        List of hosts that should be build
-      '';
-    };
-    tests = mkOption {
-      type = types.listOf types.str;
-      default = [];
-      description = ''
-        List of tests that should be build
-      '';
-    };
-  };
-
-  config = mkIf cfg.enable {
-    services.nginx = {
-      enable = true;
-      virtualHosts.build = {
-        serverAliases = [ "build.${hostname}.r" ];
-        locations."/".extraConfig = ''
-          proxy_set_header Upgrade $http_upgrade;
-          proxy_set_header Connection "upgrade";
-          proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port};
-        '';
-      };
-    };
-
-    krebs.buildbot.master = {
-      slaves = {
-        testslave = "lasspass";
-      };
-      change_source.stockholm = ''
-        stockholm_repo = '${cfg.stockholmSrc}'
-        cs.append(
-            changes.GitPoller(
-                stockholm_repo,
-                workdir='stockholm-poller', branches=True,
-                project='stockholm',
-                pollinterval=10
-            )
-        )
-      '';
-      scheduler = {
-        build-scheduler = ''
-          sched.append(
-                schedulers.SingleBranchScheduler(
-                    change_filter=util.ChangeFilter(branch_re=".*"),
-                    treeStableTimer=${toString cfg.treeStableTimer}*60,
-                    name="build-all-branches",
-                    builderNames=[
-                        ${optionalString (cfg.hosts != []) ''"hosts",''}
-                        ${optionalString (cfg.tests != []) ''"tests",''}
-                    ]
-                )
-          )
-        '';
-        force-scheduler = ''
-          sched.append(
-              schedulers.ForceScheduler(
-                    name="force",
-                    builderNames=[
-                        ${optionalString (cfg.hosts != []) ''"hosts",''}
-                        ${optionalString (cfg.tests != []) ''"tests",''}
-                    ]
-              )
-          )
-        '';
-      };
-      builder_pre = ''
-        # prepare grab_repo step for stockholm
-        grab_repo = steps.Git(
-            repourl=stockholm_repo,
-            mode='full'
-        )
-
-        # prepare addShell function
-        def addShell(factory,**kwargs):
-          factory.addStep(steps.ShellCommand(**kwargs))
-      '';
-      builder = {
-        hosts = mkIf (cfg.hosts != []) ''
-          f = util.BuildFactory()
-          f.addStep(grab_repo)
-
-          def build_host(user, host):
-              addShell(f,
-                  name="{}".format(host),
-                  env={
-                    "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
-                    "NIX_REMOTE": "daemon",
-                    "dummy_secrets": "true",
-                  },
-                  command=[
-                    "nix-shell", "-I", "stockholm=.", "--run", " ".join(["test",
-                      "--user={}".format(user),
-                      "--system={}".format(host),
-                      "--force-populate",
-                      "--target=$LOGNAME@${config.krebs.build.host.name}$HOME/{}".format(user),
-                    ])
-                  ],
-                  timeout=90001
-              )
-
-          ${concatMapStringsSep "\n" (host:
-             "build_host(\"${host.owner.name}\", \"${host.name}\")"
-          ) cfg.hosts}
-
-          bu.append(
-              util.BuilderConfig(
-                  name="hosts",
-                  slavenames=slavenames,
-                  factory=f
-              )
-          )
-        '';
-        tests = mkIf (cfg.tests != []) ''
-          f = util.BuildFactory()
-          f.addStep(grab_repo)
-
-          def run_test(test):
-              addShell(f,
-                  name="{}".format(test),
-                  env={
-                    "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
-                    "NIX_REMOTE": "daemon",
-                    "dummy_secrets": "true",
-                  },
-                  command=[
-                    "nix-build", "-I", "stockholm=.", "krebs/0tests",
-                    "-A", "{}".format(test)
-                  ],
-                  timeout=90001
-              )
-
-          ${concatMapStringsSep "\n" (test:
-             "run_test(\"${test}\")"
-          ) cfg.tests}
-
-          bu.append(
-              util.BuilderConfig(
-                  name="tests",
-                  slavenames=slavenames,
-                  factory=f
-              )
-          )
-        '';
-      };
-      enable = true;
-      web.enable = true;
-      irc = {
-        enable = true;
-        nick = "build|${hostname}";
-        server = "irc.r";
-        channels = [ "xxx" "noise" ];
-        allowForce = true;
-      };
-      extraConfig = ''
-        c['buildbotURL'] = "http://build.${hostname}.r/"
-      '';
-    };
-
-    krebs.buildbot.slave = {
-      enable = true;
-      masterhost = "localhost";
-      username = "testslave";
-      password = "lasspass";
-      packages = with pkgs; [ gnumake jq nix populate ];
-    };
-
-  };
-}
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 6fe01e466..afc96e9ee 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -13,7 +13,6 @@ let
       ./buildbot/master.nix
       ./buildbot/slave.nix
       ./build.nix
-      ./ci.nix
       ./current.nix
       ./exim.nix
       ./exim-retiolum.nix