4 files changed, 127 insertions, 0 deletions
diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix
index 97082f56a..d22dd3810 100644
--- a/krebs/3modules/backup.nix
+++ b/krebs/3modules/backup.nix
@@ -60,6 +60,12 @@ let
   };
 
   imp = {
+    krebs.on-failure.plans =
+      listToAttrs (map (plan: nameValuePair "backup.${plan.name}" {
+      }) (filter (plan: build-host-is "pull" "dst" plan ||
+                        build-host-is "push" "src" plan)
+                 enabled-plans));
+
     systemd.services =
       listToAttrs (map (plan: nameValuePair "backup.${plan.name}" {
         # TODO if there is plan.user, then use its privkey
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index bdd9049cb..77fb3d61c 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -23,6 +23,7 @@ let
       ./lib.nix
       ./nginx.nix
       ./nixpkgs.nix
+      ./on-failure.nix
       ./os-release.nix
       ./per-user.nix
       ./Reaktor.nix
diff --git a/krebs/3modules/on-failure.nix b/krebs/3modules/on-failure.nix
new file mode 100644
index 000000000..13d561b8d
--- /dev/null
+++ b/krebs/3modules/on-failure.nix
@@ -0,0 +1,91 @@
+{ config, lib, pkgs, ... }: with config.krebs.lib; let
+  out = {
+    options.krebs.on-failure = api;
+    config = lib.mkIf cfg.enable imp;
+  };
+
+  cfg = config.krebs.on-failure;
+
+  api = {
+    enable = mkEnableOption "krebs.on-failure" // {
+      default = cfg.plans != {};
+    };
+    plans = mkOption {
+      default = {};
+      type = let
+        inherit (config) krebs;
+      in types.attrsOf (types.submodule ({ config, ... }: {
+        options = {
+          enable = mkEnableOption "krebs.on-failure.${config.name}" // {
+            default = true;
+          };
+          journalctl = {
+            lines = mkOption {
+              type = types.int;
+              default = 100;
+            };
+            output = mkOption {
+              type = types.enum [
+                "cat"
+                "export"
+                "json"
+                "json-pretty"
+                "json-sse"
+                "short"
+                "short-iso"
+                "short-precise"
+                "verbose"
+              ];
+              default = "short-iso";
+            };
+          };
+          mailto = mkOption {
+            type = types.str;
+            default = krebs.build.user.mail;
+            description = "Mail address to send journal extract to.";
+          };
+          subject = mkOption {
+            type = types.str;
+            default = "[${krebs.build.host.name}] ${config.name} has failed";
+          };
+          name = mkOption {
+            type = types.str;
+            default = config._module.args.name;
+            description = "Name of the to-be-monitored service.";
+          };
+        };
+      }));
+    };
+    sendmail = mkOption {
+      type = types.str;
+      default = "/var/setuid-wrappers/sendmail";
+    };
+  };
+
+  imp = {
+    systemd.services = foldl (a: b: a // b) {} (map to-services enabled-plans);
+  };
+
+  enabled-plans = filter (getAttr "enable") (attrValues cfg.plans);
+
+  to-services = plan: {
+    "${plan.name}".unitConfig.OnFailure = "on-failure.${plan.name}.service";
+    "on-failure.${plan.name}".serviceConfig = rec {
+      ExecStart = start plan;
+      SyslogIdentifier = ExecStart.name;
+      Type = "oneshot";
+    };
+  };
+
+  start = plan: pkgs.writeDash "on-failure.${plan.name}" ''
+    { echo Subject: ${shell.escape plan.subject}
+      echo To: ${shell.escape plan.mailto}
+      echo
+      ${pkgs.systemd}/bin/journalctl \
+          --lines=${toString plan.journalctl.lines} \
+          --output=${plan.journalctl.output} \
+          --unit=${shell.escape plan.name}.service
+    } | ${shell.escape cfg.sendmail} -t
+  '';
+
+in out
diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix
index a0237d361..262f508c3 100644
--- a/krebs/3modules/tv/default.nix
+++ b/krebs/3modules/tv/default.nix
@@ -357,6 +357,35 @@ with config.krebs.lib;
     };
     tv = {
       mail = "tv@nomic.retiolum";
+      pgp.pubkeys.default = ''
+        -----BEGIN PGP PUBLIC KEY BLOCK-----
+        mQINBFbJ/B0BEADZx8l5gRurzhEHcc3PbBepdZqDJQZ2cGHixi8VEk9iN25qJO5y
+        HB0q5sQRsh7oNCbzKp6qRhaG9kXmEda+Uu+qbHWxE32QcT76+W8npH73qthaFwC/
+        5RA8KcSE8/XFxVBnVb14PNVHyAVxPHawawbhsOeaiZcHrq5IF6sVzcsc2KN87sIE
+        SthR4E01LBK4AFeFuKxga9OKFQV5WJNrihu+6H4wZwUfMpbE552N1rggxT4CouqZ
+        RocSg+el/aPRj3Jk9jDe/JFv4HU7KfioOD+NO8xLAkyw3aLsu/bv9nfUvcvTGeRp
+        z31UOjpNYpT3PS0+lNCUKQKUadAmhwU95V/0GdhadgxCFcS65qNO7ZZYDJqMIT2y
+        YH1d9MaVPDQD9W2v0ITCJcrks9p47o+C8zzDlcVr2VEGrTSngRDkWVNYjKwd3L8w
+        HuaTarqOprLzeZ6yblcLVOrW8tGTmxum0jB4Fn3enpTyJNzCfp6c0CoYp/ZziQ82
+        2jgLWuqKv3EKhX9aCUUgbeDFhnsM3GzdT5qYupX7UyWTLfiUlAEUQUgtyM7yBUNN
+        PsD5OeYeRQ/xFzUO30kglbjXOOUQpm7kyX38OJA01JdOOhXNI7BTvkFZsJzBLoVM
+        AdK3LvF4Rjau3HzYqL1Cr0ai1Y9jZVXP3vimcvUcI9bTRg9pMfD8LekiQQARAQAB
+        tAl0diA8dHZAcj6JAj0EEwEIACcFAlbJ/B0CGwMFCQHhM4AFCwkIBwIGFQgJCgsC
+        BBYCAwECHgECF4AACgkQJdgKWiyu47Xwow//ZS6Y1UcTDxHa066AQxL5UWL86Jj4
+        pIw3k630384VrUlStP+OcwOSwa4igvyIUPrOhVLynkijNsutg6KAVi8BrtSZ8ZcP
+        58gnyCPCQG4Ir0cSanp/GxMxfHKdEMyfMOopTLusLBa55VPr7sYrNi7WY20aojjJ
+        05bviSrFv0+u9dEJGmCChLDv+IhHJDe4zXHbmwspGDMwlhy/E/clSZG7a1yoJjLf
+        DpqRVn8KmICqMX0lvBP6fsS51pSD0n82kCpedLZmnwYEHCp+Bkx/Cla7aS33N1+n
+        5CUAR6HQvPT91LsLK/h/BKZ+SHAg4j7hANSfMFO+/0A5pby3JBo6Fck0LvrEMyog
+        6oGedzszZztO1eSJ5h0UQlowD4g0Y7wlWrR8znvdO1gBxQpGIjZXKqGRcuIPNZpu
+        lgqIXw/pX6b0CWh2GsbHGE0FfIkBkgW2A2akA8cGEiKqOdp/kP4o7VGCLI5iZXZA
+        ZY405gOo3ePTTRJ3zxF7YFRzjMhTlc6KtLiA9/Wps67lrOU0w/O8Dd+zYxmZoani
+        lnXaqOj32/UCW76fZ+ovUzKP2lav5wf3tpJeekjV5Zs5dNpAYmrK6EuW7LvUg5lm
+        7i5yz8yuD/xU6R3o1FycogDU6H0JtdFDYTJI9gd5EzNe3UNUEzBJF1yqQFwiW6xY
+        3yFvks3C6e58YNE=
+        =Sqyp
+        -----END PGP PUBLIC KEY BLOCK-----
+      '';
       pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDFR//RnCvEZAt0F6ExDsatKZ/DDdifanuSL360mqOhaFieKI34RoOwfQT9T+Ga52Vh5V2La6esvlph686EdgzeKLvDoxEwFM9ZYFBcMrNzu4bMTlgE7YUYw5JiORyXNfznBGnme6qpuvx9ibYhUyiZo99kM8ys5YrUHrP2JXQJMezDFZHxT4GFMOuSdh/1daGoKKD6hYL/jEHX8CI4E3BSmKK6ygYr1fVX0K0Tv77lIi5mLXucjR7CytWYWYnhM6DC3Hxpv2zRkPgf3k0x/Y1hrw3V/r0Me5h90pd2C8pFaWA2ZoUT/fmyVqvx1tZPYToU/O2dMItY0zgx2kR0yD+6g7Aahz3R+KlXkV8k5c8bbTbfGnZWDR1ZlbLRM9Yt5vosfwapUD90MmVkpmR3wUkO2sUKi80QfC7b4KvSDXQ+MImbGxMaU5Bnsq1PqLN95q+uat3nlAVBAELkcx51FlE9CaIS65y4J7FEDg8BE5JeuCNshh62VSYRXVSFt8bk3f/TFGgzC8OIo14BhVmiRQQ503Z1sROyf5xLX2a/EJavMm1i2Bs2TH6ROKY9z5Pz8hT5US0r381V8oG7TZyLF9HTtoy3wCYsgWA5EmLanjAsVU2YEeAA0rxzdtYP8Y2okFiJ6u+M4HQZ3Wg3peSodyp3vxdYce2vk4EKeqEFuuS82850DYb7Et7fmp+wQQUT8Q/bMO0DreWjHoMM5lE4LJ4ME6AxksmMiFtfo/4Fe2q9D+LAqZ+ANOcv9M+8Rn6ngiYmuRNd0l/a02q1PEvO6vTfXgcl4f7Z1IULHPEaDNZHCJS1K5RXYFqYQ6OHsTmOm7hnwaRAS97+VFMo1i5uvTx9nYaAcY7yzq3Ckfb67dMBKApGOpJpkvPgfrP7bgBO5rOZXM1opXqVPb09nljAhhAhyCTh1e/8+mJrBo0cLQ/LupQzVxGDgm3awSMPxsZAN45PSWz76zzxdDa1MMo51do+VJHfs7Wl0NcXAQrniOBYL9Wqt0qNkn1gY5smkkISGeQ/vxNap4MmzeZE7b5fpOy+2fpcRVQLpc4nooQzJvSVTFz+25lgZ6iHf45K87gQFMIAri1Pf/EDDpL87az+bRWvWi+BA2kMe1kf+Ay1LyMz8r+g51H0ma0bNFh6+fbWMfUiD9JCepIObclnUJ4NlWfcgHxTf17d/4tl6z4DTcLpCCk8Da77JouSHgvtcRbRlFV1OfhWZLXUsrlfpaQTiItv6TGIr3k7+7b66o3Qw/GQVs5GmYifaIZIz8n8my4XjkaMBd0SZfBzzvFjHMq6YUP9+SbjvReqofuoO+5tW1wTYZXitFFBfwuHlXm6w77K5QDBW6olT7pat41/F5eGxLcz tv@wu";
       uid = 1337; # TODO use default
     };