summaryrefslogtreecommitdiffstats
path: root/krebs/3modules/ci/default.nix
diff options
context:
space:
mode:
authorlassulus <lassulus@lassul.us>2022-10-23 18:30:56 +0200
committerlassulus <lassulus@lassul.us>2022-10-23 18:30:56 +0200
commitf13fed245b94e5e2669ab348475cc0169a01b7bf (patch)
tree85960202a5646c220b590830fe139d8bcb535796 /krebs/3modules/ci/default.nix
parent411ebc426cf98debcde3515491a2c0afdda51b48 (diff)
ci: move to extra folder, report failed scripts
Diffstat (limited to 'krebs/3modules/ci/default.nix')
-rw-r--r--krebs/3modules/ci/default.nix210
1 files changed, 210 insertions, 0 deletions
diff --git a/krebs/3modules/ci/default.nix b/krebs/3modules/ci/default.nix
new file mode 100644
index 000000000..0f85b27c0
--- /dev/null
+++ b/krebs/3modules/ci/default.nix
@@ -0,0 +1,210 @@
+{ config, lib, pkgs, ... }:
+
+with import <stockholm/lib>;
+
+let
+ cfg = config.krebs.ci;
+
+ out = {
+ options.krebs.ci = api;
+ config = lib.mkIf cfg.enable imp;
+ };
+
+ api = {
+ enable = mkEnableOption "Enable krebs ci service";
+ repos = mkOption {
+ type = types.attrsOf (types.submodule ({ config, ...}: {
+ options = {
+ urls = mkOption {
+ type = types.listOf types.str;
+ default = [ "git@localhost:${config._module.args.name}" ];
+ };
+ };
+ }));
+ };
+ };
+
+ hostname = config.networking.hostName;
+ getJobs = pkgs.writeDash "get_jobs" ''
+ set -efu
+ ${pkgs.nix}/bin/nix-build --no-out-link --quiet --show-trace -Q ./ci.nix >&2
+ json="$(${pkgs.nix}/bin/nix-instantiate --quiet -Q --eval --strict --json ./ci.nix)"
+ echo "$json" | ${pkgs.jq}/bin/jq -r 'to_entries[] | [.key, .value] | @tsv' \
+ | while read -r host builder; do
+ gcroot=${shell.escape profileRoot}/$host-builder
+ ${pkgs.nix}/bin/nix-env -p "$gcroot" --set "$builder"
+ done
+ echo "$json"
+ '';
+
+ profileRoot = "/nix/var/nix/profiles/ci";
+
+ bcfg = config.services.buildbot-master;
+
+ imp = {
+ services.buildbot-master = {
+ workers = [ "worker.Worker('testworker', 'pass')" ];
+
+ changeSource = mapAttrsToList (name: repo:
+ concatMapStringsSep "," (url: ''
+ changes.GitPoller(
+ "${url}",
+ workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
+ project='${name}',
+ pollinterval=30
+ )
+ '') repo.urls
+ ) cfg.repos;
+
+ schedulers = mapAttrsToList (name: repo: ''
+ schedulers.SingleBranchScheduler(
+ change_filter=util.ChangeFilter(
+ branch_re=".*",
+ project='${name}',
+ ),
+ treeStableTimer=60,
+ name="${name}-all-branches",
+ builderNames=[
+ "${name}",
+ ]
+ ),
+ schedulers.ForceScheduler(
+ name="${name}",
+ builderNames=[
+ "${name}",
+ ]
+ )
+ '') cfg.repos;
+
+ builders = [];
+
+ extraConfig = ''
+ # https://docs.buildbot.net/latest/manual/configuration/buildfactories.html
+ from buildbot.plugins import util, steps
+ from buildbot.process import buildstep, logobserver
+ from twisted.internet import defer
+ import json
+ import sys
+
+ class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep):
+ def __init__(self, **kwargs):
+ kwargs = self.setupShellMixin(kwargs)
+ super().__init__(**kwargs)
+ self.observer = logobserver.BufferLogObserver()
+ self.addLogObserver('stdio', self.observer)
+
+ def extract_stages(self, stdout):
+ stages = json.loads(stdout)
+ return stages
+
+ @defer.inlineCallbacks
+ def run(self):
+ # run nix-instanstiate to generate the dict of stages
+ cmd = yield self.makeRemoteShellCommand()
+ yield self.runCommand(cmd)
+
+ # if the command passes extract the list of stages
+ result = cmd.results()
+ if result == util.SUCCESS:
+ # create a ShellCommand for each stage and add them to the build
+ stages = self.extract_stages(self.observer.getStdout())
+ self.build.addStepsAfterCurrentStep([
+ steps.ShellCommand(
+ name=stage,
+ env=dict(
+ build_name = stage,
+ build_script = stages[stage],
+ ),
+ command="${pkgs.writeDash "build.sh" ''
+ set -xefu
+ profile=${shell.escape profileRoot}/$build_name
+ result=$("$build_script")
+ if [ -n "$result" ]; then
+ ${pkgs.nix}/bin/nix-env -p "$profile" --set "$result"
+ fi
+ ''}",
+ ) for stage in stages
+ ])
+
+ return result
+
+
+ ${concatStringsSep "\n" (mapAttrsToList (name: repo: ''
+ factory_${name} = util.BuildFactory()
+ factory_${name}.addStep(steps.Git(
+ repourl=util.Property('repository', '${head repo.urls}'),
+ method='clobber',
+ mode='full',
+ submodules=True,
+ ))
+
+ factory_${name}.addStep(GenerateStagesCommand(
+ env={
+ "NIX_REMOTE": "daemon",
+ "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src",
+ },
+ name="Generate build stages",
+ command=[
+ "${getJobs}"
+ ],
+ haltOnFailure=True,
+ ))
+
+ c['builders'].append(
+ util.BuilderConfig(
+ name="${name}",
+ workernames=['testworker'],
+ factory=factory_${name}
+ )
+ )
+ '') cfg.repos)}
+
+ # fancy irc notification by Mic92 https://github.com/Mic92/dotfiles/tree/master/nixos/eve/modules/buildbot
+ sys.path.append("${./modules}")
+ from irc_notify import NotifyFailedBuilds
+ c['services'].append(
+ NotifyFailedBuilds("irc://buildbot|test@irc.r:6667/#xxx")
+ )
+
+ '';
+
+ enable = true;
+ reporters = [
+ ''
+ reporters.IRC(
+ host = "irc.r",
+ nick = "buildbot|${hostname}",
+ notify_events = [ 'started', 'finished', 'failure', 'success', 'exception', 'problem' ],
+ channels = [{"channel": "#xxx"}],
+ showBlameList = True,
+ authz={'force': True},
+ )
+ ''
+ ];
+
+ buildbotUrl = "http://build.${hostname}.r/";
+ };
+
+ services.buildbot-worker = {
+ enable = true;
+ workerUser = "testworker";
+ workerPass = "pass";
+ packages = with pkgs; [ git gnutar gzip jq nix populate ];
+ };
+
+ system.activationScripts.buildbots-nix-profile = ''
+ ${pkgs.coreutils}/bin/mkdir -p ${shell.escape profileRoot}
+ ${pkgs.coreutils}/bin/chmod 0770 ${shell.escape profileRoot}
+ ${pkgs.coreutils}/bin/chgrp buildbots ${shell.escape profileRoot}
+ '';
+
+ users = {
+ groups.buildbots.gid = genid "buildbots";
+ users = {
+ buildbot.extraGroups = [ "buildbots" ];
+ bbworker.extraGroups = [ "buildbots" ];
+ };
+ };
+ };
+
+in out