summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authortv <tv@krebsco.de>2026-11-19 22:29:29 +0000
committertv <tv@krebsco.de>2026-11-20 01:24:09 +0100
commit0bc6887673afafdda35fb3887b14a65d3968790e (patch)
treece3689e2ddfc0e607a70efc70e4e4523e5aa4945
parent58ee2e3a72d46847ee63baf768864a14ce0716fc (diff)
hrm.ci: init
Build pushed branches of git repositories. Each instance in hrm.ci.instances serves one repository: its hook queues each pushed branch, and hrm-ci-NAME.service runs the instance's build script for each job its jobs script lists, keeping request.json, build.json and a log per job for every run in /var/lib/hrm-ci/NAME/runs. Dependencies are branches of other repositories, unpacked into the build's $HOME. Their hooks queue rebuilds of all branches, and each request records the dependency revisions it gets built against. Pushed trees are untrusted. Each instance runs as its own user, with a sandboxed service that can write only to its state directory, can't see /home, and limits each script run with timeout(1). Job names are restricted to [A-Za-z0-9_][A-Za-z0-9._-]*.
-rw-r--r--modules/ci.nix479
1 files changed, 479 insertions, 0 deletions
diff --git a/modules/ci.nix b/modules/ci.nix
new file mode 100644
index 0000000..034b50e
--- /dev/null
+++ b/modules/ci.nix
@@ -0,0 +1,479 @@
+{ config, lib, pkgs, ... }:
+
+# hrm.ci builds pushed branches of git repositories.
+#
+# Each instance (hrm.ci.instances.NAME) serves one repository and has its own
+# user and group, hrm-ci-NAME. Its post-receive hook
+# (hrm.ci.instances.NAME.hook) queues each pushed branch as a request: a
+# directory with request.json and tarballs of the pushed tree and of the
+# current revision of each dependency. A dependency is a branch of another
+# repository; its post-receive hook (hrm.ci.instances.NAME.dependencies.PATH.hook)
+# queues requests for all branches of the instance's repository, as does
+# hrm.ci.instances.NAME.rebuild when run by hand.
+#
+# A path unit starts hrm-ci-NAME.service, which takes requests from the queue
+# one at a time, unpacks the pushed tree to $HOME/srcDir and each dependency
+# to $HOME/PATH, creates the symlinks given by home, and runs the build script
+# for each job listed by the jobs script. $HOME is a scratch directory that
+# exists only during a run. Job names must match
+# [A-Za-z0-9_][A-Za-z0-9._-]*, and each script run is limited by timeout.
+#
+# The pushed tree is untrusted: evaluating it can read whatever hrm-ci-NAME
+# can read. The service can write only to stateDir/NAME, and the targets
+# given by home are outside /home, which the service can't see.
+#
+# Each run has its own directory, stateDir/NAME/runs/RUN, containing
+# request.json what to build: ref, oldrev, newrev, pusher, queued,
+# trigger (push, rebuild or dependency PATH), and per
+# dependency: repo, ref, rev
+# build.json request.json plus run number, timestamps, overall status,
+# and per job: status, exit code, out paths, log file name
+# JOB.log the build script's stderr
+# jobs.log the jobs script's stderr and invalid job names, if any
+# stateDir/NAME/runs/latest points to the most recent run. stateDir/NAME is
+# readable only by hrm-ci-NAME and its group.
+
+let
+ cfg = config.hrm.ci;
+ nixosConfig = config;
+
+ instances = lib.attrValues cfg.instances;
+
+ instanceModule = { config, name, ... }: let
+ instance = config;
+ instanceDir = "${cfg.stateDir}/${name}";
+ queue = "${instanceDir}/queue";
+
+ # [{ path, repo, ref }]; the tarball of element I is dependency-I.tar
+ dependencies = pkgs.writeText "hrm-ci-${name}-dependencies.json"
+ (builtins.toJSON (map
+ (dependency: { inherit (dependency) path repo ref; })
+ (lib.attrValues config.dependencies)));
+
+ dependencyModule = { config, name, ... }: {
+ options = {
+ path = lib.mkOption {
+ type = lib.types.str;
+ default = name;
+ readOnly = true;
+ };
+
+ repo = lib.mkOption {
+ type = lib.types.str;
+ example = "/var/lib/git/krops";
+ description = "Path to the dependency's bare repository.";
+ };
+
+ ref = lib.mkOption {
+ type = lib.types.str;
+ default = "refs/heads/master";
+ description = "The dependency's branch to build against.";
+ };
+
+ hook = lib.mkOption {
+ type = lib.types.package;
+ readOnly = true;
+ default = pkgs.writers.writeDash "hrm-ci-${instance.name}-dependency-hook" /* sh */ ''
+ set -efu
+ while read -r oldrev newrev ref; do
+ case $newrev in *[!0]*) ;; *) continue ;; esac
+ if [ "$ref" = ${lib.escapeShellArg config.ref} ]; then
+ exec ${instance.rebuild} ${lib.escapeShellArg "dependency ${name}"}
+ fi
+ done
+ '';
+ description = ''
+ post-receive hook for the dependency's repository that queues
+ requests for all branches of the instance's repository when ref
+ gets pushed. It must run as one of the instance's submitters.
+ '';
+ };
+ };
+ };
+ in {
+ options = {
+ name = lib.mkOption {
+ type = lib.types.strMatching "[A-Za-z0-9_][A-Za-z0-9._-]*";
+ default = name;
+ readOnly = true;
+ };
+
+ dir = lib.mkOption {
+ type = lib.types.str;
+ default = instanceDir;
+ internal = true;
+ readOnly = true;
+ };
+
+ build = lib.mkOption {
+ type = lib.types.lines;
+ example = ''
+ nix-build krops.nix --no-out-link --argstr name "$1" -A build
+ '';
+ description = ''
+ Shell script that builds the job named by $1. It runs in the
+ pushed tree. Its stdout lists the out paths, one per line.
+ '';
+ };
+
+ dependencies = lib.mkOption {
+ type = lib.types.attrsOf (lib.types.submodule dependencyModule);
+ default = {};
+ example = {
+ "sync/src/krops".repo = "/var/lib/git/krops";
+ };
+ description = ''
+ Branches of other repositories to unpack into the build's $HOME, as
+ paths relative to $HOME.
+ '';
+ };
+
+ home = lib.mkOption {
+ type = lib.types.attrsOf lib.types.str;
+ default = {};
+ example = {
+ "sync/src/krops" = "/var/lib/hrm-ci/src/krops";
+ };
+ description = ''
+ Symlinks to create in the build's $HOME, as paths relative to
+ $HOME mapped to their targets. Targets must be outside /home and
+ readable by hrm-ci-NAME, e.g. through its group.
+ '';
+ };
+
+ hook = lib.mkOption {
+ type = lib.types.package;
+ readOnly = true;
+ default = pkgs.writers.writeDash "hrm-ci-${name}-post-receive" /* sh */ ''
+ set -efu
+ PATH=${lib.makeBinPath [ pkgs.coreutils pkgs.git pkgs.jq ]}
+ tmp=
+ trap '[ -z "$tmp" ] || rm -rf "$tmp"' EXIT
+ while read -r oldrev newrev ref; do
+ case $ref in refs/heads/*) ;; *) continue ;; esac
+ case $newrev in *[!0]*) ;; *) continue ;; esac
+ id=$(date -u +%Y%m%dT%H%M%S.%NZ)-$newrev
+ tmp=${queue}/.$id
+ mkdir -m 2770 "$tmp"
+ git --git-dir=${lib.escapeShellArg config.repo} archive --format=tar "$newrev" \
+ > "$tmp"/source.tar
+ revs=
+ i=0
+ while [ "$i" -lt "$(jq length ${dependencies})" ]; do
+ dependency_repo=$(jq -r --argjson i "$i" '.[$i].repo' ${dependencies})
+ dependency_ref=$(jq -r --argjson i "$i" '.[$i].ref' ${dependencies})
+ dependency_rev=$(
+ git --git-dir="$dependency_repo" rev-parse --verify "$dependency_ref^{commit}"
+ )
+ git --git-dir="$dependency_repo" archive --format=tar "$dependency_rev" \
+ > "$tmp"/dependency-"$i".tar
+ revs="$revs $dependency_rev"
+ i=$((i + 1))
+ done
+ jq -n \
+ --arg id "$id" \
+ --arg ref "$ref" \
+ --arg oldrev "$oldrev" \
+ --arg newrev "$newrev" \
+ --arg pusher "''${GIT_SSH_USER-}" \
+ --arg queued "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+ --arg trigger "''${HRM_CI_TRIGGER-push}" \
+ --slurpfile dependencies ${dependencies} \
+ '{
+ $id, $ref, $oldrev, $newrev, $pusher, $queued, $trigger,
+ dependencies: ($dependencies[0] | to_entries | map(.key as $i | {
+ key: .value.path,
+ value: (.value | del(.path) | .rev = $ARGS.positional[$i])
+ }) | from_entries)
+ }' \
+ --args $revs \
+ > "$tmp"/request.json
+ mv "$tmp" ${queue}/"$id"
+ tmp=
+ done
+ '';
+ description = ''
+ post-receive hook that queues each pushed branch. It reads
+ `oldrev newrev ref` lines from stdin and must run as one of
+ submitters.
+ '';
+ };
+
+ jobs = lib.mkOption {
+ type = lib.types.lines;
+ example = ''
+ echo default
+ '';
+ description = ''
+ Shell script that lists the names of the jobs to build, one per
+ line. It runs in the pushed tree.
+ '';
+ };
+
+ nixPath = lib.mkOption {
+ type = lib.types.listOf lib.types.str;
+ default = nixosConfig.nix.nixPath;
+ defaultText = lib.literalExpression "config.nix.nixPath";
+ description = "NIX_PATH of the jobs and build scripts.";
+ };
+
+ rebuild = lib.mkOption {
+ type = lib.types.package;
+ readOnly = true;
+ default = pkgs.writers.writeDash "hrm-ci-${name}-rebuild" /* sh */ ''
+ set -efu
+ ${pkgs.git}/bin/git --git-dir=${lib.escapeShellArg config.repo} for-each-ref \
+ --format='%(objectname) %(objectname) %(refname)' refs/heads |
+ HRM_CI_TRIGGER=''${1-rebuild} ${config.hook}
+ '';
+ description = ''
+ Script that queues requests for all branches of repo. It must run
+ as one of submitters.
+ '';
+ };
+
+ repo = lib.mkOption {
+ type = lib.types.str;
+ example = "/var/lib/git/hrm";
+ description = "Path to the instance's bare repository.";
+ };
+
+ submitters = lib.mkOption {
+ type = lib.types.listOf lib.types.str;
+ default = [];
+ example = [ "git" ];
+ description = ''
+ Users that may queue builds, i.e. run hook. They become members
+ of the group hrm-ci-NAME.
+ '';
+ };
+
+ timeout = lib.mkOption {
+ type = lib.types.str;
+ default = "4h";
+ description = ''
+ Time limit for each run of the jobs or build script, as accepted
+ by timeout(1).
+ '';
+ };
+
+ srcDir = lib.mkOption {
+ type = lib.types.str;
+ default = "sync/src/${name}";
+ description = "Where to unpack the pushed tree, relative to $HOME.";
+ };
+
+ runner = lib.mkOption {
+ type = lib.types.package;
+ internal = true;
+ readOnly = true;
+ default = let
+ jobs = pkgs.writers.writeDash "hrm-ci-${name}-jobs" config.jobs;
+ build = pkgs.writers.writeDash "hrm-ci-${name}-build" config.build;
+ in pkgs.writers.writeDash "hrm-ci-${name}" /* sh */ ''
+ set -efu
+ cd ${instanceDir}
+ mkdir -p runs
+ export HOME=${instanceDir}/work/home
+ src=$HOME/${lib.escapeShellArg config.srcDir}
+
+ now() {
+ date -u +%Y-%m-%dT%H:%M:%SZ
+ }
+
+ limit() {
+ timeout --kill-after=1m ${lib.escapeShellArg config.timeout} "$@"
+ }
+
+ # update FILTER [JQ-ARGS...]
+ update() {
+ filter=$1
+ shift
+ jq "$@" "$filter" "$dir"/build.json > "$dir"/build.json.tmp
+ mv "$dir"/build.json.tmp "$dir"/build.json
+ }
+
+ while
+ request=$(find queue -mindepth 1 -maxdepth 1 -name '[!.]*' | sort | head -n 1)
+ [ -n "$request" ]
+ do
+ run=$(($(cat last-run 2>/dev/null || echo 0) + 1))
+ dir=${instanceDir}/runs/$run
+
+ # Take the request out of the queue before anything can fail, so
+ # a broken request can't make the path unit retrigger forever.
+ mkdir "$dir"
+ find "$request" -mindepth 1 -maxdepth 1 -exec mv -t "$dir" {} +
+ rmdir "$request"
+ echo "$run" > last-run
+ ln -sfn "$run" runs/latest
+
+ jq \
+ --argjson run "$run" \
+ --arg started "$(now)" \
+ '. + { run: $run, status: "running", started: $started, jobs: {} }' \
+ "$dir"/request.json \
+ > "$dir"/build.json
+
+ rm -rf work
+ mkdir -p "$src"
+ tar -x -C "$src" -f "$dir"/source.tar
+ rm "$dir"/source.tar
+ i=0
+ while [ "$i" -lt "$(jq length ${dependencies})" ]; do
+ path=$HOME/$(jq -r --argjson i "$i" '.[$i].path' ${dependencies})
+ mkdir -p "$path"
+ tar -x -C "$path" -f "$dir"/dependency-"$i".tar
+ rm "$dir"/dependency-"$i".tar
+ i=$((i + 1))
+ done
+ ${lib.concatStringsSep "\n" (lib.mapAttrsToList
+ (path: target: /* sh */ ''
+ mkdir -p "$(dirname "$HOME"/${lib.escapeShellArg path})"
+ ln -s ${lib.escapeShellArg target} "$HOME"/${lib.escapeShellArg path}
+ '')
+ config.home)}
+
+ if ! jobs=$(cd "$src" && limit ${jobs} 2> "$dir"/jobs.log); then
+ update '.status = "error" | .finished = $finished' \
+ --arg finished "$(now)"
+ continue
+ fi
+ if invalid=$(
+ echo "$jobs" | grep -vx -e "" -e '[A-Za-z0-9_][A-Za-z0-9._-]*'
+ ); then
+ echo "$invalid" | sed 's/^/invalid job name: /' >> "$dir"/jobs.log
+ update '.status = "error" | .finished = $finished' \
+ --arg finished "$(now)"
+ continue
+ fi
+ rm "$dir"/jobs.log
+
+ status=success
+ for job in $jobs; do
+ started=$(now)
+ if out=$(cd "$src" && limit ${build} "$job" 2> "$dir/$job".log); then
+ code=0
+ else
+ code=$?
+ status=failure
+ fi
+ update '.jobs[$job] = {
+ status: (if $code == 0 then "success" else "failure" end),
+ exitCode: $code,
+ outPaths: ($out | split("\n") | map(select(. != ""))),
+ log: ($job + ".log"),
+ started: $started,
+ finished: $finished
+ }' \
+ --arg job "$job" \
+ --argjson code "$code" \
+ --arg out "$out" \
+ --arg started "$started" \
+ --arg finished "$(now)"
+ done
+
+ update '.status = $status | .finished = $finished' \
+ --arg status "$status" \
+ --arg finished "$(now)"
+ done
+ rm -rf work
+ '';
+ };
+ };
+ };
+in
+{
+ options.hrm.ci = {
+ instances = lib.mkOption {
+ type = lib.types.attrsOf (lib.types.submodule instanceModule);
+ default = {};
+ };
+
+ stateDir = lib.mkOption {
+ type = lib.types.str;
+ default = "/var/lib/hrm-ci";
+ };
+ };
+
+ config = lib.mkIf (instances != []) {
+ users.users = lib.listToAttrs (map (instance: {
+ name = "hrm-ci-${instance.name}";
+ value = {
+ isSystemUser = true;
+ group = "hrm-ci-${instance.name}";
+ };
+ }) instances);
+
+ users.groups = lib.listToAttrs (map (instance: {
+ name = "hrm-ci-${instance.name}";
+ value.members = instance.submitters;
+ }) instances);
+
+ systemd.tmpfiles.rules = [
+ "d ${cfg.stateDir} 0755 root root -"
+ ] ++ lib.concatMap (instance: let
+ id = "hrm-ci-${instance.name}";
+ in [
+ "d ${instance.dir} 0750 ${id} ${id} -"
+ "d ${instance.dir}/queue 2770 ${id} ${id} -"
+ ]) instances;
+
+ systemd.paths = lib.listToAttrs (map (instance: {
+ name = "hrm-ci-${instance.name}";
+ value = {
+ wantedBy = [ "paths.target" ];
+ pathConfig.PathExistsGlob = "${instance.dir}/queue/*";
+ };
+ }) instances);
+
+ systemd.services = lib.listToAttrs (map (instance: {
+ name = "hrm-ci-${instance.name}";
+ value = {
+ environment.NIX_PATH = lib.concatStringsSep ":" instance.nixPath;
+ path = [
+ config.nix.package
+ pkgs.coreutils
+ pkgs.findutils
+ pkgs.git
+ pkgs.gnugrep
+ pkgs.gnused
+ pkgs.gnutar
+ pkgs.jq
+ ];
+ serviceConfig = {
+ Type = "oneshot";
+ TimeoutStartSec = "infinity";
+ User = "hrm-ci-${instance.name}";
+ Group = "hrm-ci-${instance.name}";
+ ExecStart = instance.runner;
+ UMask = "0027";
+
+ CapabilityBoundingSet = "";
+ LockPersonality = true;
+ MemoryDenyWriteExecute = true;
+ NoNewPrivileges = true;
+ PrivateDevices = true;
+ PrivateTmp = true;
+ ProtectClock = true;
+ ProtectControlGroups = true;
+ ProtectHome = true;
+ ProtectHostname = true;
+ ProtectKernelLogs = true;
+ ProtectKernelModules = true;
+ ProtectKernelTunables = true;
+ ProtectProc = "invisible";
+ ProtectSystem = "strict";
+ ReadWritePaths = [ instance.dir ];
+ RemoveIPC = true;
+ RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
+ RestrictNamespaces = true;
+ RestrictRealtime = true;
+ RestrictSUIDSGID = true;
+ SystemCallArchitectures = "native";
+ SystemCallFilter = [ "@system-service" "~@privileged" ];
+ };
+ };
+ }) instances);
+ };
+}