3 files changed, 98 insertions, 0 deletions
diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix
index 97082f56a..d22dd3810 100644
--- a/krebs/3modules/backup.nix
+++ b/krebs/3modules/backup.nix
@@ -60,6 +60,12 @@ let
   };
 
   imp = {
+    krebs.on-failure.plans =
+      listToAttrs (map (plan: nameValuePair "backup.${plan.name}" {
+      }) (filter (plan: build-host-is "pull" "dst" plan ||
+                        build-host-is "push" "src" plan)
+                 enabled-plans));
+
     systemd.services =
       listToAttrs (map (plan: nameValuePair "backup.${plan.name}" {
         # TODO if there is plan.user, then use its privkey
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index bdd9049cb..77fb3d61c 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -23,6 +23,7 @@ let
       ./lib.nix
       ./nginx.nix
       ./nixpkgs.nix
+      ./on-failure.nix
       ./os-release.nix
       ./per-user.nix
       ./Reaktor.nix
diff --git a/krebs/3modules/on-failure.nix b/krebs/3modules/on-failure.nix
new file mode 100644
index 000000000..13d561b8d
--- /dev/null
+++ b/krebs/3modules/on-failure.nix
@@ -0,0 +1,91 @@
+{ config, lib, pkgs, ... }: with config.krebs.lib; let
+  out = {
+    options.krebs.on-failure = api;
+    config = lib.mkIf cfg.enable imp;
+  };
+
+  cfg = config.krebs.on-failure;
+
+  api = {
+    enable = mkEnableOption "krebs.on-failure" // {
+      default = cfg.plans != {};
+    };
+    plans = mkOption {
+      default = {};
+      type = let
+        inherit (config) krebs;
+      in types.attrsOf (types.submodule ({ config, ... }: {
+        options = {
+          enable = mkEnableOption "krebs.on-failure.${config.name}" // {
+            default = true;
+          };
+          journalctl = {
+            lines = mkOption {
+              type = types.int;
+              default = 100;
+            };
+            output = mkOption {
+              type = types.enum [
+                "cat"
+                "export"
+                "json"
+                "json-pretty"
+                "json-sse"
+                "short"
+                "short-iso"
+                "short-precise"
+                "verbose"
+              ];
+              default = "short-iso";
+            };
+          };
+          mailto = mkOption {
+            type = types.str;
+            default = krebs.build.user.mail;
+            description = "Mail address to send journal extract to.";
+          };
+          subject = mkOption {
+            type = types.str;
+            default = "[${krebs.build.host.name}] ${config.name} has failed";
+          };
+          name = mkOption {
+            type = types.str;
+            default = config._module.args.name;
+            description = "Name of the to-be-monitored service.";
+          };
+        };
+      }));
+    };
+    sendmail = mkOption {
+      type = types.str;
+      default = "/var/setuid-wrappers/sendmail";
+    };
+  };
+
+  imp = {
+    systemd.services = foldl (a: b: a // b) {} (map to-services enabled-plans);
+  };
+
+  enabled-plans = filter (getAttr "enable") (attrValues cfg.plans);
+
+  to-services = plan: {
+    "${plan.name}".unitConfig.OnFailure = "on-failure.${plan.name}.service";
+    "on-failure.${plan.name}".serviceConfig = rec {
+      ExecStart = start plan;
+      SyslogIdentifier = ExecStart.name;
+      Type = "oneshot";
+    };
+  };
+
+  start = plan: pkgs.writeDash "on-failure.${plan.name}" ''
+    { echo Subject: ${shell.escape plan.subject}
+      echo To: ${shell.escape plan.mailto}
+      echo
+      ${pkgs.systemd}/bin/journalctl \
+          --lines=${toString plan.journalctl.lines} \
+          --output=${plan.journalctl.output} \
+          --unit=${shell.escape plan.name}.service
+    } | ${shell.escape cfg.sendmail} -t
+  '';
+
+in out