merge cd/master

43 files changed, 511 insertions, 426 deletions

diff --git a/Makefile b/Makefile
index 7252ca70e..a74e3a877 100644
--- a/Makefile
+++ b/Makefile
@@ -9,6 +9,7 @@ export STOCKHOLM_VERSION ?= $(shell \
 	printf '%s' "$$date.$$version"; \
 )
 
+system ?= $(HOSTNAME)
 $(if $(system),,$(error unbound variable: system))
 
 nixos-config ?= $(stockholm)/$(LOGNAME)/1systems/$(system).nix
@@ -54,15 +55,17 @@ evaluate = \
 		--show-trace \
 		-I nixos-config=$(nixos-config) \
 		-I stockholm=$(stockholm) \
-		-E '{ eval, f }: f eval' \
-		--arg eval 'import ./.' \
-		--arg f "eval@{ config, ... }: $(1)"
+		-E "let eval = import <stockholm>; in with eval; $(1)"
 
 execute = \
 	result=$$($(call evaluate,config.krebs.build.$(1))) && \
 	script=$$(echo "$$result" | jq -r .) && \
 	echo "$$script" | PS5=% sh
 
+ifeq ($(MAKECMDGOALS),)
+$(error No goals specified)
+endif
+
 # usage: make deploy system=foo [target_host=bar]
 deploy: ssh ?= ssh
 deploy:
@@ -73,7 +76,7 @@ deploy:
 
 # usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name
 eval eval.:;@$(call evaluate,$${expr-eval})
-eval.%:;@$(call evaluate,$*)
+eval.%:;@$(call evaluate,$@)
 
 # usage: make install system=foo [target_host=bar]
 install: ssh ?= ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix
index d22dd3810..71b22d8cb 100644
--- a/krebs/3modules/backup.nix
+++ b/krebs/3modules/backup.nix
@@ -103,103 +103,91 @@ let
     plan.method == method &&
     config.krebs.build.host.name == plan.${side}.host.name;
 
-  start = plan: pkgs.writeDash "backup.${plan.name}" ''
+  start = plan: let
+    login-name = "root";
+    identity = local.host.ssh.privkey.path;
+    ssh = "ssh -i ${shell.escape identity}";
+    local = getAttr plan.method {
+      push = plan.src // { rsync = src-rsync; };
+      pull = plan.dst // { rsync = dst-rsync; };
+    };
+    remote = getAttr plan.method {
+      push = plan.dst // { rsync = dst-rsync; };
+      pull = plan.src // { rsync = src-rsync; };
+    };
+    src-rsync = "rsync";
+    dst-rsync = concatStringsSep " && " [
+      "stat ${shell.escape plan.dst.path} >/dev/null"
+      "mkdir -m 0700 -p ${shell.escape plan.dst.path}/current"
+      "flock -n ${shell.escape plan.dst.path} rsync"
+    ];
+  in pkgs.writeScript "backup.${plan.name}" ''
+    #! ${pkgs.bash}/bin/bash
     set -efu
+    start_date=$(date +%s)
+    ssh_target=${shell.escape login-name}@$(${fastest-address remote.host})
     ${getAttr plan.method {
       push = ''
-        identity=${shell.escape plan.src.host.ssh.privkey.path}
-        src_path=${shell.escape plan.src.path}
-        src=$src_path
-        dst_user=root
-        dst_host=$(${fastest-address plan.dst.host})
-        dst_port=$(${network-ssh-port plan.dst.host "$dst_host"})
-        dst_path=${shell.escape plan.dst.path}
-        dst=$dst_user@$dst_host:$dst_path
-        echo "update snapshot: current; $src -> $dst" >&2
-        dst_shell() {
-          exec ssh -F /dev/null \
-              -i "$identity" \
-              ''${dst_port:+-p $dst_port} \
-              "$dst_user@$dst_host" \
-              -T "$with_dst_path_lock_script"
-        }
-        rsh="ssh -F /dev/null -i $identity ''${dst_port:+-p $dst_port}"
-        local_rsync() {
-          rsync "$@"
-        }
-        remote_rsync=${shell.escape (concatStringsSep " && " [
-          "mkdir -m 0700 -p ${shell.escape plan.dst.path}/current"
-          "exec flock -n ${shell.escape plan.dst.path} rsync"
-        ])}
+        rsync_src=${shell.escape plan.src.path}
+        rsync_dst=$ssh_target:${shell.escape plan.dst.path}
+        echo >&2 "update snapshot current; $rsync_src -> $rsync_dst"
       '';
       pull = ''
-        identity=${shell.escape plan.dst.host.ssh.privkey.path}
-        src_user=root
-        src_host=$(${fastest-address plan.src.host})
-        src_port=$(${network-ssh-port plan.src.host "$src_host"})
-        src_path=${shell.escape plan.src.path}
-        src=$src_user@$src_host:$src_path
-        dst_path=${shell.escape plan.dst.path}
-        dst=$dst_path
-        echo "update snapshot: current; $dst <- $src" >&2
-        dst_shell() {
-          eval "$with_dst_path_lock_script"
-        }
-        rsh="ssh -F /dev/null -i $identity ''${src_port:+-p $src_port}"
-        local_rsync() {
-          mkdir -m 0700 -p ${shell.escape plan.dst.path}/current
-          flock -n ${shell.escape plan.dst.path} rsync "$@"
-        }
-        remote_rsync=rsync
+        rsync_src=$ssh_target:${shell.escape plan.src.path}
+        rsync_dst=${shell.escape plan.dst.path}
+        echo >&2 "update snapshot current; $rsync_dst <- $rsync_src"
       '';
     }}
-    # Note that this only works because we trust date +%s to produce output
-    # that doesn't need quoting when used to generate a command string.
-    # TODO relax this requirement by selectively allowing to inject variables
-    #   e.g.: ''${shell.quote "exec env NOW=''${shell.unquote "$NOW"} ..."}
-    with_dst_path_lock_script="exec env start_date=$(date +%s) "${shell.escape
-      "flock -n ${shell.escape plan.dst.path} /bin/sh"
-    }
-    local_rsync >&2 \
+    ${local.rsync} >&2 \
         -aAXF --delete \
-        --rsh="$rsh" \
-        --rsync-path="$remote_rsync" \
-        --link-dest="$dst_path/current" \
-        "$src/" \
-        "$dst/.partial"
-    dst_shell < ${toFile "backup.${plan.name}.take-snapshots" ''
+        --rsh=${shell.escape ssh} \
+        --rsync-path=${shell.escape remote.rsync} \
+        --link-dest=${shell.escape plan.dst.path}/current \
+        "$rsync_src/" \
+        "$rsync_dst/.partial"
+
+    dst_exec() {
+      ${getAttr plan.method {
+        push = ''exec ${ssh} "$ssh_target" -T "exec$(printf ' %q' "$@")"'';
+        pull = ''exec "$@"'';
+      }}
+    }
+    dst_exec env \
+        start_date="$start_date" \
+        flock -n ${shell.escape plan.dst.path} \
+        /bin/sh < ${toFile "backup.${plan.name}.take-snapshots" ''
       set -efu
       : $start_date
 
-      dst=${shell.escape plan.dst.path}
+      dst_path=${shell.escape plan.dst.path}
 
-      mv "$dst/current" "$dst/.previous"
-      mv "$dst/.partial" "$dst/current"
-      rm -fR "$dst/.previous"
+      mv "$dst_path/current" "$dst_path/.previous"
+      mv "$dst_path/.partial" "$dst_path/current"
+      rm -fR "$dst_path/.previous"
       echo >&2
 
       snapshot() {(
         : $ns $format $retain
         name=$(date --date="@$start_date" +"$format")
-        if ! test -e "$dst/$ns/$name"; then
+        if ! test -e "$dst_path/$ns/$name"; then
           echo >&2 "create snapshot: $ns/$name"
-          mkdir -m 0700 -p "$dst/$ns"
+          mkdir -m 0700 -p "$dst_path/$ns"
           rsync >&2 \
               -aAXF --delete \
-              --link-dest="$dst/current" \
-              "$dst/current/" \
-              "$dst/$ns/.partial.$name"
-          mv "$dst/$ns/.partial.$name" "$dst/$ns/$name"
+              --link-dest="$dst_path/current" \
+              "$dst_path/current/" \
+              "$dst_path/$ns/.partial.$name"
+          mv "$dst_path/$ns/.partial.$name" "$dst_path/$ns/$name"
           echo >&2
         fi
         case $retain in
           ([0-9]*)
             delete_from=$(($retain + 1))
-            ls -r "$dst/$ns" \
+            ls -r "$dst_path/$ns" \
               | sed -n "$delete_from,\$p" \
               | while read old_name; do
                   echo >&2 "delete snapshot: $ns/$old_name"
-                  rm -fR "$dst/$ns/$old_name"
+                  rm -fR "$dst_path/$ns/$old_name"
                 done
             ;;
           (ALL)
@@ -227,24 +215,12 @@ let
       | ${pkgs.coreutils}/bin/head -1; }
   '';
 
-  # Note that we don't escape word on purpose, so we can deref shell vars.
-  # TODO type word
-  network-ssh-port = host: word: ''
-    case ${word} in
-    ${concatStringsSep ";;\n" (mapAttrsToList
-      (_: net: "(${head net.aliases}) echo ${toString net.ssh.port}")
-      host.nets)};;
-    esac
-  '';
-
 in out
 # TODO ionice
-# TODO mail on failed push, pull
 # TODO mail on missing push
 # TODO don't cancel plans on activation
 #   also, don't hang while deploying at:
 #   starting the following units: backup.wu-home-xu.push.service, backup.wu-home-xu.push.timer
-# TODO make sure /bku is properly mounted
 # TODO make sure that secure hosts cannot backup to insecure ones
 # TODO optionally only backup when src and dst are near enough :)
 # TODO try using btrfs for snapshots (configurable)
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index d4c6b08df..9cd095622 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -88,6 +88,8 @@ let
           #! /bin/sh
           set -efu
 
+          export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+
           verbose() {
             printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
             "$@"
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index be530d46f..c114b74df 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -13,6 +13,7 @@ let
       ./buildbot/slave.nix
       ./build.nix
       ./current.nix
+      ./exim.nix
       ./exim-retiolum.nix
       ./exim-smarthost.nix
       ./fetchWallpaper.nix
@@ -157,7 +158,11 @@ let
           makefu
           tv
         ];
+        ciko.mail = "wieczorek.stefan@gmail.com";
+        Mic92.mail = "joerg@higgsboson.tk";
       in {
+        "*@eloop.org" = [{ mail = "eloop2016@krebsco.de"; }];
+        "eloop2016@krebsco.de" = spam-ml ++ [ ciko Mic92 ];
         "postmaster@krebsco.de" = spam-ml; # RFC 822
         "lass@krebsco.de" = lass;
         "makefu@krebsco.de" = makefu;
@@ -218,7 +223,7 @@ let
                             (filter (hasSuffix ".${cfg.search-domain}")
                                     longs);
                       add-port = a:
-                        if net.ssh.port != null
+                        if net.ssh.port != 22
                           then "[${a}]:${toString net.ssh.port}"
                           else a;
                     in
@@ -228,8 +233,25 @@ let
             publicKey = host.ssh.pubkey;
           })
           (filterAttrs (_: host: host.ssh.pubkey != null) cfg.hosts);
+
+      programs.ssh.extraConfig = concatMapStrings
+        (net: ''
+          Host ${toString (net.aliases ++ net.addrs)}
+            Port ${toString net.ssh.port}
+        '')
+        (filter
+          (net: net.ssh.port != 22)
+          (concatMap (host: attrValues host.nets)
+            (mapAttrsToList
+              (_: host: recursiveUpdate host
+                (optionalAttrs (hasAttr config.krebs.search-domain host.nets) {
+                  nets."" = host.nets.${config.krebs.search-domain} // {
+                    aliases = [host.name];
+                    addrs = [];
+                  };
+                }))
+              config.krebs.hosts)));
     }
   ];
 
-in
-out
+in out
diff --git a/krebs/3modules/exim-retiolum.nix b/krebs/3modules/exim-retiolum.nix
index 696c48baf..d6b7ab753 100644
--- a/krebs/3modules/exim-retiolum.nix
+++ b/krebs/3modules/exim-retiolum.nix
@@ -32,7 +32,7 @@ let
   };
 
   imp = {
-    services.exim = {
+    krebs.exim = {
       enable = true;
       config =
         # This configuration makes only sense for retiolum-enabled hosts.
@@ -57,6 +57,8 @@ let
           syslog_timestamp = false
           syslog_duplication = false
 
+          tls_advertise_hosts =
+
           begin acl
 
           acl_check_rcpt:
diff --git a/krebs/3modules/exim-smarthost.nix b/krebs/3modules/exim-smarthost.nix
index cee10ce7d..aba6ee0b5 100644
--- a/krebs/3modules/exim-smarthost.nix
+++ b/krebs/3modules/exim-smarthost.nix
@@ -105,7 +105,7 @@ let
         requires = [ "secret.service" ];
       };
     };
-    services.exim = {
+    krebs.exim = {
       enable = true;
       config = ''
         keep_environment =
@@ -136,6 +136,8 @@ let
         syslog_timestamp = false
         syslog_duplication = false
 
+        tls_advertise_hosts =
+
         begin acl
 
         acl_check_rcpt:
@@ -162,7 +164,7 @@ let
                   control       = dkim_disable_verify
 
           accept message = relay not permitted 2
-                  recipients = lsearch;${lsearch.internet-aliases}
+                  recipients = lsearch*@;${lsearch.internet-aliases}
 
           require message = relay not permitted
                   domains = +local_domains : +relay_to_domains
@@ -196,7 +198,7 @@ let
         internet_aliases:
           debug_print = "R: internet_aliases for $local_part@$domain"
           driver = redirect
-          data = ''${lookup{$local_part@$domain}lsearch{${lsearch.internet-aliases}}}
+          data = ''${lookup{$local_part@$domain}lsearch*@{${lsearch.internet-aliases}}}
 
         dnslookup:
           debug_print = "R: dnslookup for $local_part@$domain"
diff --git a/krebs/3modules/exim.nix b/krebs/3modules/exim.nix
new file mode 100644
index 000000000..7b18c72c1
--- /dev/null
+++ b/krebs/3modules/exim.nix
@@ -0,0 +1,80 @@
+{ config, lib, pkgs, ... }: with config.krebs.lib; let
+  cfg = config.krebs.exim;
+in {
+  options.krebs.exim = {
+    enable = mkEnableOption "krebs.exim";
+    config = mkOption {
+      type = types.str;
+      default = "";
+      description = ''
+        Verbatim Exim configuration.  This should not contain exim_user,
+        exim_group, exim_path, or spool_directory.
+      '';
+    };
+    user = mkOption {
+      type = types.user;
+      default = {
+        name = "exim";
+        home = "/var/spool/exim";
+      };
+      description = ''
+        User to use when no root privileges are required.
+        In particular, this applies when receiving messages and when doing
+        remote deliveries.  (Local deliveries run as various non-root users,
+        typically as the owner of a local mailbox.) Specifying this value
+        as root is not supported.
+      '';
+    };
+    group = mkOption {
+      type = types.group;
+      default = {
+        name = "exim";
+      };
+      description = ''
+        Group to use when no root privileges are required.
+      '';
+    };
+  };
+  config = lib.mkIf cfg.enable {
+    environment = {
+      etc."exim.conf".source = pkgs.writeEximConfig "exim.conf" ''
+        exim_user = ${cfg.user.name}
+        exim_group = ${cfg.group.name}
+        exim_path = /var/setuid-wrappers/exim
+        spool_directory = ${cfg.user.home}
+        ${cfg.config}
+      '';
+      systemPackages = [ pkgs.exim ];
+    };
+    krebs.setuid = {
+      exim = {
+        filename = "${pkgs.exim}/bin/exim";
+        mode = "4111";
+      };
+      sendmail = {
+        filename = "${pkgs.exim}/bin/exim";
+        mode = "4111";
+      };
+    };
+    systemd.services.exim = {
+      restartTriggers = [
+        config.environment.etc."exim.conf".source
+      ];
+      serviceConfig = {
+        ExecStart = "${pkgs.exim}/bin/exim -bdf -q30m";
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+    users = {
+      groups.${cfg.group.name} = {
+        inherit (cfg.group) name gid;
+      };
+      users.${cfg.user.name} = {
+        inherit (cfg.user) home name uid;
+        createHome = true;
+        group = cfg.group.name;
+      };
+    };
+  };
+}
diff --git a/krebs/3modules/lass/default.nix b/krebs/3modules/lass/default.nix
index 3d54900e4..b4686894e 100644
--- a/krebs/3modules/lass/default.nix
+++ b/krebs/3modules/lass/default.nix
@@ -8,15 +8,15 @@ with config.krebs.lib;
       cores = 4;
       nets = rec {
         internet = {
-          addrs4 = ["144.76.172.188"];
+          ip4.addr = "144.76.172.188";
           aliases = [
             "dishfire.internet"
           ];
         };
         retiolum = {
           via = internet;
-          addrs4 = ["10.243.133.99"];
-          addrs6 = ["42:0000:0000:0000:0000:0000:d15f:1233"];
+          ip4.addr = "10.243.133.99";
+          ip6.addr = "42:0000:0000:0000:0000:0000:d15f:1233";
           aliases = [
             "dishfire.retiolum"
             "dishfire.r"
@@ -40,15 +40,15 @@ with config.krebs.lib;
       cores = 2;
       nets = rec {
         internet = {
-          addrs4 = ["162.252.241.33"];
+          ip4.addr = "162.252.241.33";
           aliases = [
             "echelon.internet"
           ];
         };
         retiolum = {
           via = internet;
-          addrs4 = ["10.243.206.103"];
-          addrs6 = ["42:941e:2816:35f4:5c5e:206b:3f0b:f763"];
+          ip4.addr = "10.243.206.103";
+          ip6.addr = "42:941e:2816:35f4:5c5e:206b:3f0b:f763";
           aliases = [
             "echelon.retiolum"
             "echelon.r"
@@ -75,15 +75,15 @@ with config.krebs.lib;
       cores = 4;
       nets = rec {
         internet = {
-          addrs4 = ["213.239.205.240"];
+          ip4.addr = "213.239.205.240";
           aliases = [
             "prism.internet"
           ];
         };
         retiolum = {
           via = internet;
-          addrs4 = ["10.243.0.103"];
-          addrs6 = ["42:0000:0000:0000:0000:0000:0000:15ab"];
+          ip4.addr = "10.243.0.103";
+          ip6.addr = "42:0000:0000:0000:0000:0000:0000:15ab";
           aliases = [
             "prism.retiolum"
             "prism.r"
@@ -107,15 +107,15 @@ with config.krebs.lib;
     fastpoke = {
       nets = rec {
         internet = {
-          addrs4 = ["193.22.164.36"];
+          ip4.addr = "193.22.164.36";
           aliases = [
             "fastpoke.internet"
           ];
         };
         retiolum = {
           via = internet;
-          addrs4 = ["10.243.253.152"];
-          addrs6 = ["42:422a:194f:ff3b:e196:2f82:5cf5:bc00"];
+          ip4.addr = "10.243.253.152";
+          ip6.addr = "42:422a:194f:ff3b:e196:2f82:5cf5:bc00";
           aliases = [
             "fastpoke.retiolum"
             "fastpoke.r"
@@ -139,15 +139,15 @@ with config.krebs.lib;
       cores = 1;
       nets = rec {
         internet = {
-          addrs4 = ["104.167.113.104"];
+          ip4.addr = "104.167.113.104";
           aliases = [
             "cloudkrebs.internet"
           ];
         };
         retiolum = {
           via = internet;
-          addrs4 = ["10.243.206.102"];
-          addrs6 = ["42:941e:2816:35f4:5c5e:206b:3f0b:f762"];
+          ip4.addr = "10.243.206.102";
+          ip6.addr = "42:941e:2816:35f4:5c5e:206b:3f0b:f762";
           aliases = [
             "cloudkrebs.retiolum"
             "cloudkrebs.r"
@@ -172,12 +172,12 @@ with config.krebs.lib;
       cores = 1;
       nets = {
         gg23 = {
-          addrs4 = ["10.23.1.12"];
+          ip4.addr = "10.23.1.12";
           aliases = ["uriel.gg23"];
         };
         retiolum = {
-          addrs4 = ["10.243.81.176"];
-          addrs6 = ["42:dc25:60cf:94ef:759b:d2b6:98a9:2e56"];
+          ip4.addr = "10.243.81.176";
+          ip6.addr = "42:dc25:60cf:94ef:759b:d2b6:98a9:2e56";
           aliases = [
             "uriel.retiolum"
             "uriel.r"
@@ -203,12 +203,12 @@ with config.krebs.lib;
       cores = 2;
       nets = {
         gg23 = {
-          addrs4 = ["10.23.1.11"];
+          ip4.addr = "10.23.1.11";
           aliases = ["mors.gg23"];
         };
         retiolum = {
-          addrs4 = ["10.243.0.2"];
-          addrs6 = ["42:0:0:0:0:0:0:dea7"];
+          ip4.addr = "10.243.0.2";
+          ip6.addr = "42:0:0:0:0:0:0:dea7";
           aliases = [
             "mors.retiolum"
             "mors.r"
@@ -234,8 +234,8 @@ with config.krebs.lib;
       cores = 2;
       nets = {
         retiolum = {
-          addrs4 = ["10.243.0.3"];
-          addrs6 = ["42:0:0:0:0:0:0:7105"];
+          ip4.addr = "10.243.0.3";
+          ip6.addr = "42:0:0:0:0:0:0:7105";
           aliases = [
             "helios.retiolum"
             "helios.r"
diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix
index 418224138..79d9a59b4 100644
--- a/krebs/3modules/makefu/default.nix
+++ b/krebs/3modules/makefu/default.nix
@@ -8,8 +8,8 @@ with config.krebs.lib;
       cores = 1;
       nets = {
         retiolum = {
-          addrs4 = ["10.243.0.210"];
-          addrs6 = ["42:f9f1:0000:0000:0000:0000:0000:0001"];
+          ip4.addr = "10.243.0.210";
+          ip6.addr = "42:f9f1:0000:0000:0000:0000:0000:0001";
           aliases = [
             "pnp.retiolum"
             "cgit.pnp.retiolum"
@@ -31,8 +31,8 @@ with config.krebs.lib;
       cores = 4;
       nets = {
         retiolum = {
-          addrs4 = ["10.243.0.84"];
-          addrs6 = ["42:ff6b:5f0b:460d:2cee:4d05:73f7:5566"];
+          ip4.addr = "10.243.0.84";
+          ip6.addr = "42:ff6b:5f0b:460d:2cee:4d05:73f7:5566";
           aliases = [
             "darth.retiolum"
             "darth.r"
@@ -54,8 +54,8 @@ with config.krebs.lib;
       cores = 1;
       nets = {
         retiolum = {
-          addrs4 = ["10.243.0.212"];
-          addrs6 = ["42:f9f1:0000:0000:0000:0000:0000:0002"];
+          ip4.addr = "10.243.0.212";
+          ip6.addr = "42:f9f1:0000:0000:0000:0000:0000:0002";
           aliases = [
             "tsp.retiolum"
           ];
@@ -81,8 +81,8 @@ with config.krebs.lib;
       cores = 2;
       nets = {
         retiolum = {
-          addrs4 = ["10.243.0.91"];
-          addrs6 = ["42:0b2c:d90e:e717:03dc:9ac1:7c30:a4db"];
+          ip4.addr = "10.243.0.91";
+          ip6.addr = "42:0b2c:d90e:e717:03dc:9ac1:7c30:a4db";
           aliases = [
             "pornocauster.retiolum"
             "pornocauster.r"