Merge branch 'master' of prism.r:stockholm

69 files changed, 1179 insertions, 628 deletions

diff --git a/krebs/0tests/data/secrets/github-hosts-sync.ssh.id_ed25519 b/krebs/0tests/data/secrets/github-hosts-sync.ssh.id_ed25519
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/krebs/0tests/data/secrets/github-hosts-sync.ssh.id_ed25519
diff --git a/krebs/0tests/data/secrets/shackspace-gitlab-ci b/krebs/0tests/data/secrets/shackspace-gitlab-ci
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/krebs/0tests/data/secrets/shackspace-gitlab-ci
diff --git a/krebs/1systems/hotdog/config.nix b/krebs/1systems/hotdog/config.nix
index f68c8ce50..32e416831 100644
--- a/krebs/1systems/hotdog/config.nix
+++ b/krebs/1systems/hotdog/config.nix
@@ -18,6 +18,7 @@
   ];
 
   krebs.build.host = config.krebs.hosts.hotdog;
+  krebs.github-hosts-sync.enable = true;
 
   boot.isContainer = true;
   networking.useDHCP = false;
diff --git a/krebs/1systems/puyak/config.nix b/krebs/1systems/puyak/config.nix
index 67257eacd..af11c6944 100644
--- a/krebs/1systems/puyak/config.nix
+++ b/krebs/1systems/puyak/config.nix
@@ -73,6 +73,13 @@
   system.activationScripts."disengage fancontrol" = ''
     echo level disengaged > /proc/acpi/ibm/fan
   '';
+
+  # to access vorstand vm
+  users.users.root.openssh.authorizedKeys.keys = [
+    config.krebs.users.ulrich.pubkey
+    config.krebs.users.raute.pubkey
+  ];
+
   users.users.joerg = {
     openssh.authorizedKeys.keys = [ config.krebs.users.Mic92.pubkey ];
     isNormalUser = true;
diff --git a/krebs/1systems/wolf/config.nix b/krebs/1systems/wolf/config.nix
index ec8830711..7ca0f0ec1 100644
--- a/krebs/1systems/wolf/config.nix
+++ b/krebs/1systems/wolf/config.nix
@@ -11,83 +11,44 @@ in
     <stockholm/krebs>
     <stockholm/krebs/2configs>
     <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
-    <stockholm/krebs/2configs/collectd-base.nix>
-    <stockholm/krebs/2configs/stats/wolf-client.nix>
 
-    <stockholm/krebs/2configs/graphite.nix>
     <stockholm/krebs/2configs/binary-cache/nixos.nix>
     <stockholm/krebs/2configs/binary-cache/prism.nix>
 
+    # handle the worlddomination map via coap
     <stockholm/krebs/2configs/shack/worlddomination.nix>
+
+    # drivedroid.shack for shackphone
     <stockholm/krebs/2configs/shack/drivedroid.nix>
     # <stockholm/krebs/2configs/shack/nix-cacher.nix>
-    <stockholm/krebs/2configs/shack/mqtt_sub.nix>
+    # Say if muell will be collected
     <stockholm/krebs/2configs/shack/muell_caller.nix>
-    <stockholm/krebs/2configs/shack/radioactive.nix>
+
+    # create samba share for anonymous usage with the laser and 3d printer pc
     <stockholm/krebs/2configs/shack/share.nix>
+
+    # mobile.lounge.mpd.shack
     <stockholm/krebs/2configs/shack/mobile.mpd.nix>
-    {
-      systemd.services.telegraf.path = [ pkgs.net_snmp ]; # for snmptranslate
-      systemd.services.telegraf.environment = {
-        MIBDIRS = pkgs.fetchgit {
-          url = "http://git.shackspace.de/makefu/modem-mibs.git";
-          sha256 =
-          "1rhrpaascvj5p3dj29hrw79gm39rp0aa787x95m3r2jrcq83ln1k";
-        }; # extra mibs like ADSL
-      };
-      services.telegraf = {
-        enable = true;
-        extraConfig = {
-          inputs = {
-            snmp = {
-              agents = [ "10.0.1.3:161" ];
-              version = 2;
-              community = "shack";
-              name = "snmp";
-              field = [
-                {
-                  name = "hostname";
-                  oid = "RFC1213-MIB::sysName.0";
-                  is_tag = true;
-                }
-                {
-                  name = "load-percent"; #cisco
-                  oid = ".1.3.6.1.4.1.9.9.109.1.1.1.1.4.9";
-                }
-                {
-                  name = "uptime";
-                  oid = "DISMAN-EVENT-MIB::sysUpTimeInstance";
-                }
-              ];
-              table = [{
-                name = "snmp";
-                inherit_tags = [ "hostname" ];
-                oid = "IF-MIB::ifXTable";
-                field = [{
-                  name = "ifName";
-                  oid = "IF-MIB::ifName";
-                  is_tag = true;
-                }];
-              }];
-            };
-          };
-          outputs = {
-            influxdb = {
-              urls = [ "http://${influx-host}:8086" ];
-              database = "telegraf";
-              write_consistency = "any";
-              timeout = "5s";
-            };
-          };
-        };
-      };
-    }
+    # connect to git.shackspace.de as group runner for rz
+    <stockholm/krebs/2configs/shack/gitlab-runner.nix>
+
+    # Statistics collection and visualization
+    <stockholm/krebs/2configs/graphite.nix>
+    ## Collect data from mqtt.shack and store in graphite database
+    <stockholm/krebs/2configs/shack/mqtt_sub.nix>
+    ## Collect radioactive data and put into graphite
+    <stockholm/krebs/2configs/shack/radioactive.nix>
+    ## Collect local statistics via collectd and send to collectd
+    <stockholm/krebs/2configs/stats/wolf-client.nix>
+    ## write collectd statistics to wolf.shack
+    <stockholm/krebs/2configs/collectd-base.nix>
+    { services.influxdb.enable = true; }
 
+    <stockholm/krebs/2configs/shack/netbox.nix>
   ];
   # use your own binary cache, fallback use cache.nixos.org (which is used by
   # apt-cacher-ng in first place)
 
-  services.influxdb.enable = true;
 
   # local discovery in shackspace
   nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; };
@@ -156,10 +117,10 @@ in
   # fallout of ipv6calypse
   networking.extraHosts = ''
     hass.shack    10.42.2.191
-    heidi.shack   10.42.2.135
   '';
 
   users.extraUsers.root.openssh.authorizedKeys.keys = [
+    config.krebs.users."0x4a6f".pubkey
     config.krebs.users.ulrich.pubkey
     config.krebs.users.raute.pubkey
     config.krebs.users.makefu-omo.pubkey
diff --git a/krebs/2configs/shack/gitlab-runner.nix b/krebs/2configs/shack/gitlab-runner.nix
new file mode 100644
index 000000000..0fd06426a
--- /dev/null
+++ b/krebs/2configs/shack/gitlab-runner.nix
@@ -0,0 +1,21 @@
+{ pkgs, ... }:
+let
+  runner-src = builtins.fetchTarball {
+    url = "https://gitlab.com/arianvp/nixos-gitlab-runner/-/archive/master/nixos-gitlab-runner-master.tar.gz";
+    sha256 = "1s0fy5ny2ygcfvx35xws8xz5ih4z4kdfqlq3r6byxpylw7r52fyi";
+  };
+in
+{
+  systemd.services.gitlab-runner.path = [
+    "/run/wrappers" # /run/wrappers/bin/su
+    "/" # /bin/sh
+  ];
+  imports = [
+    "${runner-src}/gitlab-runner.nix"
+  ];
+  services.gitlab-runner2.enable = true;
+  ## registrationConfigurationFile contains:
+  # CI_SERVER_URL=<CI server URL>
+  # REGISTRATION_TOKEN=<registration secret>
+  services.gitlab-runner2.registrationConfigFile = <secrets/shackspace-gitlab-ci>;
+}
diff --git a/krebs/2configs/shack/netbox.nix b/krebs/2configs/shack/netbox.nix
new file mode 100644
index 000000000..4fb5a7dbc
--- /dev/null
+++ b/krebs/2configs/shack/netbox.nix
@@ -0,0 +1,39 @@
+{ pkgs, ... }:
+{
+  environment.systemPackages = [ pkgs.docker-compose ];
+  virtualisation.docker.enable = true;
+  services.nginx = {
+    enable = true;
+    virtualHosts."netbox.shack".locations."/".proxyPass = "http://localhost:18080";
+  };
+  # we store the netbox config there:
+  # state = [ "/var/lib/netbox" ];
+  systemd.services.backup-netbox = {
+    after = [ "netbox-docker-compose.service" ];
+    startAt = "daily";
+    path = with pkgs; [ docker-compose docker gzip coreutils ];
+    script = ''
+      cd /var/lib/netbox
+      mkdir -p backup
+      docker-compose exec -T -upostgres postgres pg_dumpall \
+        | gzip > backup/netdata_$(date -Iseconds).dump.gz
+    '';
+  };
+
+  systemd.services.netbox-docker-compose = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network-online.target" "docker.service" ];
+    environment.VERSION = "v2.5.13";
+    serviceConfig = {
+      WorkingDirectory = "/var/lib/netbox";
+      # TODO: grep -q NAPALM_SECRET env/netbox.env
+      # TODO: grep -q NAPALM_SECRET netbox-netprod-importer/switches.yml
+      ExecStartPre = "${pkgs.docker-compose}/bin/docker-compose pull";
+      ExecStart = "${pkgs.docker-compose}/bin/docker-compose up";
+      Restart = "always";
+      RestartSec = "10";
+      StartLimitIntervalSec = 60;
+      StartLimitBurst = 3;
+    };
+  };
+}
diff --git a/krebs/3modules/exim-retiolum.nix b/krebs/3modules/exim-retiolum.nix
index e08024977..118a8b2d5 100644
--- a/krebs/3modules/exim-retiolum.nix
+++ b/krebs/3modules/exim-retiolum.nix
@@ -1,15 +1,17 @@
-{ config, pkgs, lib, ... }:
-
 with import <stockholm/lib>;
-let
+{ config, pkgs, lib, ... }: let
   cfg = config.krebs.exim-retiolum;
 
-  out = {
-    options.krebs.exim-retiolum = api;
-    config = lib.mkIf cfg.enable imp;
-  };
+  # Due to improvements to the JSON notation, braces around top-level objects
+  # are not necessary^Wsupported by rspamd's parser when including files:
+  # https://github.com/rspamd/rspamd/issues/2674
+  toMostlyJSON = value:
+    assert typeOf value == "set";
+    (s: substring 1 (stringLength s - 2) s)
+    (toJSON value);
 
-  api = {
+in {
+  options.krebs.exim-retiolum = {
     enable = mkEnableOption "krebs.exim-retiolum";
     local_domains = mkOption {
       type = with types; listOf hostname;
@@ -28,22 +30,70 @@ let
         "*.r"
       ];
     };
+    rspamd = {
+      enable = mkEnableOption "krebs.exim-retiolum.rspamd" // {
+        default = false;
+      };
+      locals = {
+        logging = {
+          level = mkOption {
+            type = types.enum [
+              "error"
+              "warning"
+              "notice"
+              "info"
+              "debug"
+              "silent"
+            ];
+            default = "notice";
+          };
+        };
+        options = {
+          local_networks = mkOption {
+            type = types.listOf types.cidr;
+            default = [
+              config.krebs.build.host.nets.retiolum.ip4.prefix
+              config.krebs.build.host.nets.retiolum.ip6.prefix
+            ];
+          };
+        };
+      };
+    };
   };
-
-  imp = {
+  imports = [
+    {
+      config = lib.mkIf cfg.rspamd.enable {
+        services.rspamd.enable = true;
+        services.rspamd.locals =
+          mapAttrs'
+            (name: value: nameValuePair "${name}.inc" {
+              text = toMostlyJSON value;
+            })
+            cfg.rspamd.locals;
+        users.users.${config.krebs.exim.user.name}.extraGroups = [
+          config.services.rspamd.group
+        ];
+      };
+    }
+  ];
+  config = lib.mkIf cfg.enable {
     krebs.exim = {
       enable = true;
       config =
         # This configuration makes only sense for retiolum-enabled hosts.
         # TODO modular configuration
         assert config.krebs.tinc.retiolum.enable;
-        ''
+        /* exim */ ''
           keep_environment =
 
           primary_hostname = ${cfg.primary_hostname}
           domainlist local_domains = ${concatStringsSep ":" cfg.local_domains}
           domainlist relay_to_domains = ${concatStringsSep ":" cfg.relay_to_domains}
 
+          ${optionalString cfg.rspamd.enable /* exim */ ''
+            spamd_address = /run/rspamd/rspamd.sock variant=rspamd
+          ''}
+
           acl_smtp_rcpt = acl_check_rcpt
           acl_smtp_data = acl_check_data
 
@@ -72,6 +122,24 @@ let
 
 
           acl_check_data:
+            ${optionalString cfg.rspamd.enable /* exim */ ''
+              accept condition = ''${if eq{$interface_port}{587}}
+
+              warn remove_header = ${concatStringsSep " : " [
+                "x-spam"
+                "x-spam-report"
+                "x-spam-score"
+              ]}
+
+              warn
+                spam = nobody:true
+
+              warn
+                condition = ''${if !eq{$spam_action}{no action}}
+                add_header = X-Spam: Yes
+                add_header = X-Spam-Report: $spam_report
+                add_header = X-Spam-Score: $spam_score
+            ''}
             accept
 
 
@@ -118,4 +186,4 @@ let
         '';
     };
   };
-in out
+}
diff --git a/krebs/3modules/exim-smarthost.nix b/krebs/3modules/exim-smarthost.nix
index 5f93ae937..e988fb563 100644
--- a/krebs/3modules/exim-smarthost.nix
+++ b/krebs/3modules/exim-smarthost.nix
@@ -121,7 +121,7 @@ let
     };
     krebs.exim = {
       enable = true;
-      config = ''
+      config = /* exim */ ''
         keep_environment =
 
         primary_hostname = ${cfg.primary_hostname}
@@ -233,7 +233,7 @@ let
 
         remote_smtp:
           driver = smtp
-          ${optionalString (cfg.dkim != []) (indent ''
+          ${optionalString (cfg.dkim != []) (indent /* exim */ ''
             dkim_canon = relaxed
             dkim_domain = $sender_address_domain
             dkim_private_key = ''${lookup{$sender_address_domain}lsearch{${lsearch.dkim_private_key}}}
@@ -262,7 +262,7 @@ let
 
         begin rewrite
         begin authenticators
-        ${concatStringsSep "\n" (mapAttrsToList (name: text: ''
+        ${concatStringsSep "\n" (mapAttrsToList (name: text: /* exim */ ''
         ${name}:
           ${indent text}
         '') cfg.authenticators)}
diff --git a/krebs/3modules/exim.nix b/krebs/3modules/exim.nix
index cfcbbc438..83d88cb0d 100644
--- a/krebs/3modules/exim.nix
+++ b/krebs/3modules/exim.nix
@@ -37,7 +37,7 @@ in {
   };
   config = lib.mkIf cfg.enable {
     environment = {
-      etc."exim.conf".source = pkgs.writeEximConfig "exim.conf" ''
+      etc."exim.conf".source = pkgs.writeEximConfig "exim.conf" /* exim */ ''
         exim_user = ${cfg.user.name}
         exim_group = ${cfg.group.name}
         exim_path = /run/wrappers/bin/exim
diff --git a/krebs/3modules/external/default.nix b/krebs/3modules/external/default.nix
index 9bfc920a3..1720811d9 100644
--- a/krebs/3modules/external/default.nix
+++ b/krebs/3modules/external/default.nix
@@ -43,6 +43,31 @@ in {
         };
       };
     };
+    wilde = {
+      owner = config.krebs.users.kmein;
+      nets = {
+        retiolum = {
+          ip4.addr = "10.243.2.4";
+          aliases = [ "wilde.r" ];
+          tinc.pubkey = ''
+            -----BEGIN PUBLIC KEY-----
+            MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtz/MY5OSxJqrEMv6Iwjk
+            g/V58MATljj+2bmOuOuPui/AUYHEZX759lHW4MgLjYdNbZEoVq8UgkxNk0KPGlSg
+            2lsJ7FneCU7jBSE2iLT1aHuNFFa56KzSThFUl6Nj6Vyg5ghSmDF2tikurtG2q+Ay
+            uxf5/yEhFUPc1ZxmvJDqVHMeW5RZkuKXH00C7yN+gdcPuuFEFq+OtHNkBVmaxu7L
+            a8Q6b/QbrwQJAR9FAcm5WSQIj2brv50qnD8pZrU4loVu8dseQIicWkRowC0bzjAo
+            IHZTbF/S+CK0u0/q395sWRQJISkD+WAZKz5qOGHc4djJHBR3PWgHWBnRdkYqlQYM
+            C9zA/n4I+Y2BEfTWtgkD2g0dDssNGP5dlgFScGmRclR9pJ/7dsIbIeo9C72c6q3q
+            sg0EIWggQ8xyWrUTXIMoDXt37htlTSnTgjGsuwRzjotAEMJmgynWRf3br3yYChrq
+            10Exq8Lej+iOuKbdAXlwjKEk0qwN7JWft3OzVc2DMtKf7rcZQkBoLfWKzaCTQ4xo
+            1Y7d4OlcjbgrkLwHltTaShyosm8kbttdeinyBG1xqQcK11pMO43GFj8om+uKrz57
+            lQUVipu6H3WIVGnvLmr0e9MQfThpC1em/7Aq2exn1JNUHhCdEho/mK2x/doiiI+0
+            QAD64zPmuo9wsHnSMR2oKs0CAwEAAQ==
+            -----END PUBLIC KEY-----
+          '';
+        };
+      };
+    };
     dpdkm = {
       owner = config.krebs.users.Mic92;
       nets = rec {
@@ -167,6 +192,20 @@ in {