summaryrefslogtreecommitdiffstats
path: root/krebs/3modules/ci
diff options
context:
space:
mode:
authorlassulus <lassulus@lassul.us>2022-10-23 18:30:56 +0200
committerlassulus <lassulus@lassul.us>2022-10-23 18:30:56 +0200
commitf13fed245b94e5e2669ab348475cc0169a01b7bf (patch)
tree85960202a5646c220b590830fe139d8bcb535796 /krebs/3modules/ci
parent411ebc426cf98debcde3515491a2c0afdda51b48 (diff)
ci: move to extra folder, report failed scripts
Diffstat (limited to 'krebs/3modules/ci')
-rw-r--r--krebs/3modules/ci/default.nix210
-rw-r--r--krebs/3modules/ci/modules/irc_notify.py145
2 files changed, 355 insertions, 0 deletions
diff --git a/krebs/3modules/ci/default.nix b/krebs/3modules/ci/default.nix
new file mode 100644
index 000000000..0f85b27c0
--- /dev/null
+++ b/krebs/3modules/ci/default.nix
@@ -0,0 +1,210 @@
+{ config, lib, pkgs, ... }:
+
+with import <stockholm/lib>;
+
+let
+ cfg = config.krebs.ci;
+
+ out = {
+ options.krebs.ci = api;
+ config = lib.mkIf cfg.enable imp;
+ };
+
+ api = {
+ enable = mkEnableOption "Enable krebs ci service";
+ repos = mkOption {
+ type = types.attrsOf (types.submodule ({ config, ...}: {
+ options = {
+ urls = mkOption {
+ type = types.listOf types.str;
+ default = [ "git@localhost:${config._module.args.name}" ];
+ };
+ };
+ }));
+ };
+ };
+
+ hostname = config.networking.hostName;
+ getJobs = pkgs.writeDash "get_jobs" ''
+ set -efu
+ ${pkgs.nix}/bin/nix-build --no-out-link --quiet --show-trace -Q ./ci.nix >&2
+ json="$(${pkgs.nix}/bin/nix-instantiate --quiet -Q --eval --strict --json ./ci.nix)"
+ echo "$json" | ${pkgs.jq}/bin/jq -r 'to_entries[] | [.key, .value] | @tsv' \
+ | while read -r host builder; do
+ gcroot=${shell.escape profileRoot}/$host-builder
+ ${pkgs.nix}/bin/nix-env -p "$gcroot" --set "$builder"
+ done
+ echo "$json"
+ '';
+
+ profileRoot = "/nix/var/nix/profiles/ci";
+
+ bcfg = config.services.buildbot-master;
+
+ imp = {
+ services.buildbot-master = {
+ workers = [ "worker.Worker('testworker', 'pass')" ];
+
+ changeSource = mapAttrsToList (name: repo:
+ concatMapStringsSep "," (url: ''
+ changes.GitPoller(
+ "${url}",
+ workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
+ project='${name}',
+ pollinterval=30
+ )
+ '') repo.urls
+ ) cfg.repos;
+
+ schedulers = mapAttrsToList (name: repo: ''
+ schedulers.SingleBranchScheduler(
+ change_filter=util.ChangeFilter(
+ branch_re=".*",
+ project='${name}',
+ ),
+ treeStableTimer=60,
+ name="${name}-all-branches",
+ builderNames=[
+ "${name}",
+ ]
+ ),
+ schedulers.ForceScheduler(
+ name="${name}",
+ builderNames=[
+ "${name}",
+ ]
+ )
+ '') cfg.repos;
+
+ builders = [];
+
+ extraConfig = ''
+ # https://docs.buildbot.net/latest/manual/configuration/buildfactories.html
+ from buildbot.plugins import util, steps
+ from buildbot.process import buildstep, logobserver
+ from twisted.internet import defer
+ import json
+ import sys
+
+ class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep):
+ def __init__(self, **kwargs):
+ kwargs = self.setupShellMixin(kwargs)
+ super().__init__(**kwargs)
+ self.observer = logobserver.BufferLogObserver()
+ self.addLogObserver('stdio', self.observer)
+
+ def extract_stages(self, stdout):
+ stages = json.loads(stdout)
+ return stages
+
+ @defer.inlineCallbacks
+ def run(self):
+ # run nix-instanstiate to generate the dict of stages
+ cmd = yield self.makeRemoteShellCommand()
+ yield self.runCommand(cmd)
+
+ # if the command passes extract the list of stages
+ result = cmd.results()
+ if result == util.SUCCESS:
+ # create a ShellCommand for each stage and add them to the build
+ stages = self.extract_stages(self.observer.getStdout())
+ self.build.addStepsAfterCurrentStep([
+ steps.ShellCommand(
+ name=stage,
+ env=dict(
+ build_name = stage,
+ build_script = stages[stage],
+ ),
+ command="${pkgs.writeDash "build.sh" ''
+ set -xefu
+ profile=${shell.escape profileRoot}/$build_name
+ result=$("$build_script")
+ if [ -n "$result" ]; then
+ ${pkgs.nix}/bin/nix-env -p "$profile" --set "$result"
+ fi
+ ''}",
+ ) for stage in stages
+ ])
+
+ return result
+
+
+ ${concatStringsSep "\n" (mapAttrsToList (name: repo: ''
+ factory_${name} = util.BuildFactory()
+ factory_${name}.addStep(steps.Git(
+ repourl=util.Property('repository', '${head repo.urls}'),
+ method='clobber',
+ mode='full',
+ submodules=True,
+ ))
+
+ factory_${name}.addStep(GenerateStagesCommand(
+ env={
+ "NIX_REMOTE": "daemon",
+ "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
+ },
+ name="Generate build stages",
+ command=[
+ "${getJobs}"
+ ],
+ haltOnFailure=True,
+ ))
+
+ c['builders'].append(
+ util.BuilderConfig(
+ name="${name}",
+ workernames=['testworker'],
+ factory=factory_${name}
+ )
+ )
+ '') cfg.repos)}
+
+ # fancy irc notification by Mic92 https://github.com/Mic92/dotfiles/tree/master/nixos/eve/modules/buildbot
+ sys.path.append("${./modules}")
+ from irc_notify import NotifyFailedBuilds
+ c['services'].append(
+ NotifyFailedBuilds("irc://buildbot|test@irc.r:6667/#xxx")
+ )
+
+ '';
+
+ enable = true;
+ reporters = [
+ ''
+ reporters.IRC(
+ host = "irc.r",
+ nick = "buildbot|${hostname}",
+ notify_events = [ 'started', 'finished', 'failure', 'success', 'exception', 'problem' ],
+ channels = [{"channel": "#xxx"}],
+ showBlameList = True,
+ authz={'force': True},
+ )
+ ''
+ ];
+
+ buildbotUrl = "http://build.${hostname}.r/";
+ };
+
+ services.buildbot-worker = {
+ enable = true;
+ workerUser = "testworker";
+ workerPass = "pass";
+ packages = with pkgs; [ git gnutar gzip jq nix populate ];
+ };
+
+ system.activationScripts.buildbots-nix-profile = ''
+ ${pkgs.coreutils}/bin/mkdir -p ${shell.escape profileRoot}
+ ${pkgs.coreutils}/bin/chmod 0770 ${shell.escape profileRoot}
+ ${pkgs.coreutils}/bin/chgrp buildbots ${shell.escape profileRoot}
+ '';
+
+ users = {
+ groups.buildbots.gid = genid "buildbots";
+ users = {
+ buildbot.extraGroups = [ "buildbots" ];
+ bbworker.extraGroups = [ "buildbots" ];
+ };
+ };
+ };
+
+in out
diff --git a/krebs/3modules/ci/modules/irc_notify.py b/krebs/3modules/ci/modules/irc_notify.py
new file mode 100644
index 000000000..4b7969aaf
--- /dev/null
+++ b/krebs/3modules/ci/modules/irc_notify.py
@@ -0,0 +1,145 @@
+from typing import Optional, Generator, Any
+import socket
+import ssl
+import threading
+import re
+from urllib.parse import urlparse
+import base64
+
+from buildbot.reporters.base import ReporterBase
+from buildbot.reporters.generators.build import BuildStatusGenerator
+from buildbot.reporters.message import MessageFormatter
+from twisted.internet import defer
+
+DEBUG = False
+
+
+def _irc_send(
+ server: str,
+ nick: str,
+ channel: str,
+ sasl_password: Optional[str] = None,
+ server_password: Optional[str] = None,
+ tls: bool = True,
+ port: int = 6697,
+ messages: list[str] = [],
+) -> None:
+ if not messages:
+ return
+
+ # don't give a shit about legacy ip
+ sock = socket.socket(family=socket.AF_INET6)
+ if tls:
+ sock = ssl.wrap_socket(
+ sock, cert_reqs=ssl.CERT_NONE, ssl_version=ssl.PROTOCOL_TLSv1_2
+ )
+
+ def _send(command: str) -> int:
+ if DEBUG:
+ print(command)
+ return sock.send((f"{command}\r\n").encode())
+
+ def _pong(ping: str):
+ if ping.startswith("PING"):
+ sock.send(ping.replace("PING", "PONG").encode("ascii"))
+
+ recv_file = sock.makefile(mode="r")
+
+ print(f"connect {server}:{port}")
+ sock.connect((server, port))
+ if server_password:
+ _send(f"PASS {server_password}")
+ _send(f"USER {nick} 0 * :{nick}")
+ _send(f"NICK {nick}")
+ for line in recv_file.readline():
+ if re.match(r"^:[^ ]* (MODE|221|376|422) ", line):
+ break
+ else:
+ _pong(line)
+
+ if sasl_password:
+ _send("CAP REQ :sasl")
+ _send("AUTHENTICATE PLAIN")
+ auth = base64.encodebytes(f"{nick}\0{nick}\0{sasl_password}".encode("ascii"))
+ _send(f"AUTHENTICATE {auth.decode('ascii')}")
+ _send("CAP END")
+ _send(f"JOIN :{channel}")
+
+ for m in messages:
+ _send(f"PRIVMSG {channel} :{m}")
+
+ _send("INFO")
+ for line in recv_file:
+ if DEBUG:
+ print(line, end="")
+ # Assume INFO reply means we are done
+ if "End of /INFO" in line:
+ break
+ else:
+ _pong(line)
+
+ sock.send(b"QUIT")
+ print("disconnect")
+ sock.close()
+
+
+def irc_send(
+ url: str, notifications: list[str], password: Optional[str] = None
+) -> None:
+ parsed = urlparse(f"{url}")
+ username = parsed.username or "prometheus"
+ server = parsed.hostname or "chat.freenode.net"
+ if parsed.fragment != "":
+ channel = f"#{parsed.fragment}"
+ else:
+ channel = "#krebs-announce"
+ port = parsed.port or 6697
+ if not password:
+ password = parsed.password
+ if len(notifications) == 0:
+ return
+ # put this in a thread to not block buildbot
+ t = threading.Thread(
+ target=_irc_send,
+ kwargs=dict(
+ server=server,
+ nick=username,
+ sasl_password=password,
+ channel=channel,
+ port=port,
+ messages=notifications,
+ tls=parsed.scheme == "irc+tls",
+ ),
+ )
+ t.start()
+
+
+subject_template = """\
+{{ '☠' if result_names[results] == 'failure' else '☺' if result_names[results] == 'success' else '☝' }} \
+{{ build['properties'].get('project', ['whole buildset'])[0] if is_buildset else buildername }} \
+- \
+{{ build['state_string'] }} \
+{{ '(%s)' % (build['properties']['branch'][0] if (build['properties']['branch'] and build['properties']['branch'][0]) else build['properties'].get('got_revision', ['(unknown revision)'])[0]) }} \
+({{ build_url }})
+""" # # noqa pylint: disable=line-too-long
+
+
+class NotifyFailedBuilds(ReporterBase):
+ def _generators(self) -> list[BuildStatusGenerator]:
+ formatter = MessageFormatter(template_type="plain", subject=subject_template)
+ return [BuildStatusGenerator(message_formatter=formatter)]
+
+ def checkConfig(self, url: str):
+ super().checkConfig(generators=self._generators())
+
+ @defer.inlineCallbacks
+ def reconfigService(self, url: str) -> Generator[Any, object, Any]:
+ self.url = url
+ yield super().reconfigService(generators=self._generators())
+
+ def sendMessage(self, reports: list):
+ msgs = []
+ for r in reports:
+ if r["builds"][0]["state_string"] != "build successful":
+ msgs.append(r["subject"])
+ irc_send(self.url, notifications=msgs)