ci: move to extra folder, report failed scripts

2 files changed, 355 insertions, 0 deletions

diff --git a/krebs/3modules/ci/default.nix b/krebs/3modules/ci/default.nix
new file mode 100644
index 000000000..0f85b27c0
--- /dev/null
+++ b/krebs/3modules/ci/default.nix
@@ -0,0 +1,210 @@
+{ config, lib, pkgs, ... }:
+
+with import <stockholm/lib>;
+
+let
+  cfg = config.krebs.ci;
+
+  out = {
+    options.krebs.ci = api;
+    config = lib.mkIf cfg.enable imp;
+  };
+
+  api = {
+    enable = mkEnableOption "Enable krebs ci service";
+    repos = mkOption {
+      type = types.attrsOf (types.submodule ({ config, ...}: {
+        options = {
+          urls = mkOption {
+            type = types.listOf types.str;
+            default = [ "git@localhost:${config._module.args.name}" ];
+          };
+        };
+      }));
+    };
+  };
+
+  hostname = config.networking.hostName;
+  getJobs = pkgs.writeDash "get_jobs" ''
+    set -efu
+    ${pkgs.nix}/bin/nix-build --no-out-link --quiet --show-trace -Q ./ci.nix >&2
+    json="$(${pkgs.nix}/bin/nix-instantiate --quiet -Q --eval --strict --json ./ci.nix)"
+    echo "$json" | ${pkgs.jq}/bin/jq -r 'to_entries[] | [.key, .value] | @tsv' \
+      | while read -r host builder; do
+        gcroot=${shell.escape profileRoot}/$host-builder
+        ${pkgs.nix}/bin/nix-env -p "$gcroot" --set "$builder"
+      done
+    echo "$json"
+  '';
+
+  profileRoot = "/nix/var/nix/profiles/ci";
+
+  bcfg = config.services.buildbot-master;
+
+  imp = {
+    services.buildbot-master = {
+      workers = [ "worker.Worker('testworker', 'pass')" ];
+
+      changeSource = mapAttrsToList (name: repo:
+        concatMapStringsSep "," (url: ''
+          changes.GitPoller(
+              "${url}",
+              workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
+              project='${name}',
+              pollinterval=30
+          )
+        '') repo.urls
+      ) cfg.repos;
+
+      schedulers = mapAttrsToList (name: repo: ''
+        schedulers.SingleBranchScheduler(
+            change_filter=util.ChangeFilter(
+                branch_re=".*",
+                project='${name}',
+            ),
+            treeStableTimer=60,
+            name="${name}-all-branches",
+            builderNames=[
+                "${name}",
+            ]
+        ),
+        schedulers.ForceScheduler(
+            name="${name}",
+            builderNames=[
+                "${name}",
+            ]
+        )
+      '') cfg.repos;
+
+      builders = [];
+
+      extraConfig = ''
+        # https://docs.buildbot.net/latest/manual/configuration/buildfactories.html
+        from buildbot.plugins import util, steps
+        from buildbot.process import buildstep, logobserver
+        from twisted.internet import defer
+        import json
+        import sys
+
+        class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep):
+            def __init__(self, **kwargs):
+                kwargs = self.setupShellMixin(kwargs)
+                super().__init__(**kwargs)
+                self.observer = logobserver.BufferLogObserver()
+                self.addLogObserver('stdio', self.observer)
+
+            def extract_stages(self, stdout):
+                stages = json.loads(stdout)
+                return stages
+
+            @defer.inlineCallbacks
+            def run(self):
+                # run nix-instanstiate to generate the dict of stages
+                cmd = yield self.makeRemoteShellCommand()
+                yield self.runCommand(cmd)
+
+                # if the command passes extract the list of stages
+                result = cmd.results()
+                if result == util.SUCCESS:
+                    # create a ShellCommand for each stage and add them to the build
+                    stages = self.extract_stages(self.observer.getStdout())
+                    self.build.addStepsAfterCurrentStep([
+                        steps.ShellCommand(
+                          name=stage,
+                          env=dict(
+                            build_name = stage,
+                            build_script = stages[stage],
+                          ),
+                          command="${pkgs.writeDash "build.sh" ''
+                            set -xefu
+                            profile=${shell.escape profileRoot}/$build_name
+                            result=$("$build_script")
+                            if [ -n "$result" ]; then
+                              ${pkgs.nix}/bin/nix-env -p "$profile" --set "$result"
+                            fi
+                          ''}",
+                        ) for stage in stages
+                    ])
+
+                return result
+
+
+        ${concatStringsSep "\n" (mapAttrsToList (name: repo: ''
+          factory_${name} = util.BuildFactory()
+          factory_${name}.addStep(steps.Git(
+              repourl=util.Property('repository', '${head repo.urls}'),
+              method='clobber',
+              mode='full',
+              submodules=True,
+          ))
+
+          factory_${name}.addStep(GenerateStagesCommand(
+              env={
+                  "NIX_REMOTE": "daemon",
+                  "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
+              },
+              name="Generate build stages",
+              command=[
+                  "${getJobs}"
+              ],
+              haltOnFailure=True,
+          ))
+
+          c['builders'].append(
+              util.BuilderConfig(
+                  name="${name}",
+                  workernames=['testworker'],
+                  factory=factory_${name}
+              )
+          )
+        '') cfg.repos)}
+
+        # fancy irc notification by Mic92 https://github.com/Mic92/dotfiles/tree/master/nixos/eve/modules/buildbot
+        sys.path.append("${./modules}")
+        from irc_notify import NotifyFailedBuilds
+        c['services'].append(
+            NotifyFailedBuilds("irc://buildbot|test@irc.r:6667/#xxx")
+        )
+
+      '';
+
+      enable = true;
+      reporters = [
+        ''
+          reporters.IRC(
+            host = "irc.r",
+            nick = "buildbot|${hostname}",
+            notify_events = [ 'started', 'finished', 'failure', 'success', 'exception', 'problem' ],
+            channels = [{"channel": "#xxx"}],
+            showBlameList = True,
+            authz={'force': True},
+          )
+        ''
+      ];
+
+      buildbotUrl = "http://build.${hostname}.r/";
+    };
+
+    services.buildbot-worker = {
+      enable = true;
+      workerUser = "testworker";
+      workerPass = "pass";
+      packages = with pkgs; [ git gnutar gzip jq nix populate ];
+    };
+
+    system.activationScripts.buildbots-nix-profile = ''
+      ${pkgs.coreutils}/bin/mkdir -p ${shell.escape profileRoot}
+      ${pkgs.coreutils}/bin/chmod 0770 ${shell.escape profileRoot}
+      ${pkgs.coreutils}/bin/chgrp buildbots ${shell.escape profileRoot}
+    '';
+
+    users = {
+      groups.buildbots.gid = genid "buildbots";
+      users = {
+        buildbot.extraGroups = [ "buildbots" ];
+        bbworker.extraGroups = [ "buildbots" ];
+      };
+    };
+  };
+
+in out
diff --git a/krebs/3modules/ci/modules/irc_notify.py b/krebs/3modules/ci/modules/irc_notify.py
new file mode 100644
index 000000000..4b7969aaf
--- /dev/null
+++ b/krebs/3modules/ci/modules/irc_notify.py
@@ -0,0 +1,145 @@
+from typing import Optional, Generator, Any
+import socket
+import ssl
+import threading
+import re
+from urllib.parse import urlparse
+import base64
+
+from buildbot.reporters.base import ReporterBase
+from buildbot.reporters.generators.build import BuildStatusGenerator
+from buildbot.reporters.message import MessageFormatter
+from twisted.internet import defer
+
+DEBUG = False
+
+
+def _irc_send(
+    server: str,
+    nick: str,
+    channel: str,
+    sasl_password: Optional[str] = None,
+    server_password: Optional[str] = None,
+    tls: bool = True,
+    port: int = 6697,
+    messages: list[str] = [],
+) -> None:
+    if not messages:
+        return
+
+    # don't give a shit about legacy ip
+    sock = socket.socket(family=socket.AF_INET6)
+    if tls:
+        sock = ssl.wrap_socket(
+            sock, cert_reqs=ssl.CERT_NONE, ssl_version=ssl.PROTOCOL_TLSv1_2
+        )
+
+    def _send(command: str) -> int:
+        if DEBUG:
+            print(command)
+        return sock.send((f"{command}\r\n").encode())
+
+    def _pong(ping: str):
+        if ping.startswith("PING"):
+            sock.send(ping.replace("PING", "PONG").encode("ascii"))
+
+    recv_file = sock.makefile(mode="r")
+
+    print(f"connect {server}:{port}")
+    sock.connect((server, port))
+    if server_password:
+        _send(f"PASS {server_password}")
+    _send(f"USER {nick} 0 * :{nick}")
+    _send(f"NICK {nick}")
+    for line in recv_file.readline():
+        if re.match(r"^:[^ ]* (MODE|221|376|422) ", line):
+            break
+        else:
+            _pong(line)
+
+    if sasl_password:
+        _send("CAP REQ :sasl")
+        _send("AUTHENTICATE PLAIN")
+        auth = base64.encodebytes(f"{nick}\0{nick}\0{sasl_password}".encode("ascii"))
+        _send(f"AUTHENTICATE {auth.decode('ascii')}")
+        _send("CAP END")
+    _send(f"JOIN :{channel}")
+
+    for m in messages:
+        _send(f"PRIVMSG {channel} :{m}")
+
+    _send("INFO")
+    for line in recv_file:
+        if DEBUG:
+            print(line, end="")
+        # Assume INFO reply means we are done
+        if "End of /INFO" in line:
+            break
+        else:
+            _pong(line)
+
+    sock.send(b"QUIT")
+    print("disconnect")
+    sock.close()
+
+
+def irc_send(
+    url: str, notifications: list[str], password: Optional[str] = None
+) -> None:
+    parsed = urlparse(f"{url}")
+    username = parsed.username or "prometheus"
+    server = parsed.hostname or "chat.freenode.net"
+    if parsed.fragment != "":
+        channel = f"#{parsed.fragment}"
+    else:
+        channel = "#krebs-announce"
+    port = parsed.port or 6697
+    if not password:
+        password = parsed.password
+    if len(notifications) == 0:
+        return
+    # put this in a thread to not block buildbot
+    t = threading.Thread(
+        target=_irc_send,
+        kwargs=dict(
+            server=server,
+            nick=username,
+            sasl_password=password,
+            channel=channel,
+            port=port,
+            messages=notifications,
+            tls=parsed.scheme == "irc+tls",
+        ),
+    )
+    t.start()
+
+
+subject_template = """\
+{{ '☠' if result_names[results] == 'failure' else '☺' if result_names[results] == 'success' else '☝' }} \
+{{ build['properties'].get('project', ['whole buildset'])[0] if is_buildset else buildername }} \
+- \
+{{ build['state_string'] }} \
+{{ '(%s)' % (build['properties']['branch'][0] if (build['properties']['branch'] and build['properties']['branch'][0]) else build['properties'].get('got_revision', ['(unknown revision)'])[0]) }} \
+({{ build_url }})
+"""  # # noqa pylint: disable=line-too-long
+
+
+class NotifyFailedBuilds(ReporterBase):
+    def _generators(self) -> list[BuildStatusGenerator]:
+        formatter = MessageFormatter(template_type="plain", subject=subject_template)
+        return [BuildStatusGenerator(message_formatter=formatter)]
+
+    def checkConfig(self, url: str):
+        super().checkConfig(generators=self._generators())
+
+    @defer.inlineCallbacks
+    def reconfigService(self, url: str) -> Generator[Any, object, Any]:
+        self.url = url
+        yield super().reconfigService(generators=self._generators())
+
+    def sendMessage(self, reports: list):
+        msgs = []
+        for r in reports:
+            if r["builds"][0]["state_string"] != "build successful":
+                msgs.append(r["subject"])
+        irc_send(self.url, notifications=msgs)