Merge remote-tracking branch 'gum/master'

15 files changed, 174 insertions, 64 deletions

diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix
index c85bf1ccd..9942ac043 100644
--- a/krebs/3modules/makefu/default.nix
+++ b/krebs/3modules/makefu/default.nix
@@ -453,6 +453,8 @@ TNs2RYfwDy/r6H/hDeB/BSngPouedEVcPwIDAQAB
           git.euer          IN A      ${nets.internet.ip4.addr}
           gum               IN A      ${nets.internet.ip4.addr}
           cgit.euer         IN A      ${nets.internet.ip4.addr}
+          o.euer            IN A      ${nets.internet.ip4.addr}
+          dl.euer           IN A      ${nets.internet.ip4.addr}
         '';
       };
       nets = rec {
@@ -470,6 +472,8 @@ TNs2RYfwDy/r6H/hDeB/BSngPouedEVcPwIDAQAB
             "gum.r"
             "gum.retiolum"
             "cgit.gum.retiolum"
+            "o.gum.r"
+            "o.gum.retiolum"
             "tracker.makefu.r"
             "tracker.makefu.retiolum"
           ];
diff --git a/krebs/3modules/rtorrent.nix b/krebs/3modules/rtorrent.nix
index 0c478aded..d85518993 100644
--- a/krebs/3modules/rtorrent.nix
+++ b/krebs/3modules/rtorrent.nix
@@ -224,13 +224,14 @@ let
             cp -f ${configFile} ${cfg.workDir}/.rtorrent.rc
           '';
           ExecStart = "${pkgs.tmux}/bin/tmux new-session -s rt -n rtorrent -d 'PATH=/bin:/usr/bin:${makeBinPath rutorrent-deps} ${cfg.package}/bin/rtorrent'";
+          Restart = "always";
+          RestartSec = "10";
 
           ## you can simply sudo -u rtorrent tmux a if privateTmp is set to false
           ## otherwise the tmux session is stored in some private folder in /tmp
           PrivateTmp = false;
 
           WorkingDirectory = cfg.workDir;
-          Restart = "on-failure";
           User = "${cfg.user}";
         };
       };
diff --git a/makefu/1systems/wry.nix b/makefu/1systems/wry.nix
index 6290ff6e9..03114c0e6 100644
--- a/makefu/1systems/wry.nix
+++ b/makefu/1systems/wry.nix
@@ -24,10 +24,10 @@ in {
       # ../2configs/nginx/euer.test.nix
 
       # collectd
-      # ../2configs/collectd/collectd-base.nix
+      ../2configs/logging/central-stats-client.nix
 
       ../2configs/tinc/retiolum.nix
-      ../2configs/torrent.nix
+      # ../2configs/torrent.nix
   ];
 
   krebs.build.host = config.krebs.hosts.wry;
diff --git a/makefu/2configs/backup.nix b/makefu/2configs/backup.nix
index 38fd93e2b..280dc1df4 100644
--- a/makefu/2configs/backup.nix
+++ b/makefu/2configs/backup.nix
@@ -29,6 +29,7 @@ let
   };
 in {
   krebs.backup.plans = {
-    wry-to-omo_var-www = defaultPull config.krebs.hosts.wry "/";
+    wry-to-omo_root = defaultPull config.krebs.hosts.wry "/";
+    gum-to-omo_root = defaultPull config.krebs.hosts.gum "/";
   };
 }
diff --git a/makefu/2configs/elchos/log.nix b/makefu/2configs/elchos/log.nix
index 3facd1ceb..34bd2494d 100644
--- a/makefu/2configs/elchos/log.nix
+++ b/makefu/2configs/elchos/log.nix
@@ -21,9 +21,9 @@ in {
         }
       '';
       outputConfig = ''
-        stdout {
-          codec => rubydebug
-        }
+        #stdout {
+        #  codec => rubydebug
+        #}
         elasticsearch { }
         '';
 	};
diff --git a/makefu/2configs/elchos/stats.nix b/makefu/2configs/elchos/stats.nix
index feb8c5bae..2036b391f 100644
--- a/makefu/2configs/elchos/stats.nix
+++ b/makefu/2configs/elchos/stats.nix
@@ -7,6 +7,11 @@
 with import <stockholm/lib>;
 {
 
+  networking.firewall = {
+    allowedTCPPorts = [ 2003 80 443 18080 ];
+    allowedUDPPorts = [ 2003 ];
+  };
+
   services.nginx = {
     enable = mkDefault true;
     virtualHosts = {
@@ -39,58 +44,11 @@ with import <stockholm/lib>;
   };
 
   services.graphite = {
-    beacon = {
-      enable = true;
-      config = {
-        graphite_url = "http://localhost:18080";
-
-        no_data = "critical";
-        loading_error = "normal";
-
-        prefix = "[elchos]";
-
-        cli = {
-          command = ''${pkgs.irc-announce}/bin/irc-announce irc.freenode.org 6667 alert0r \#elchos ' [elchos] ''${level} ''${name} ''${value}' '';
-        };
-        #smtp = {
-        #  from = "beacon@mors.r";
-        #  to = [
-        #    "lass@mors.r"
-        #  ];
-        #};
-        normal_handlers = [
-          # "smtp"
-          "cli"
-        ];
-        warning_handlers = [
-          # "smtp"
-          "cli"
-        ];
-        critical_handlers = [
-          # "smtp"
-          "cli"
-        ];
-        alerts = let
-          high-load = hostid: let
-              host = "elch-${toString hostid}"; in {
-              name = "high-cpu-load-${host}";
-              query = "aliasByNode(perSecond(elchos.${host}.cpu.0.cpu.idle),1)";
-              method = "average";
-              interval = "1minute";
-              logging = "info";
-              repeat_interval = "5minute";
-              rules = [
-  #              "warning: < 30.0"
-                "critical: < 1.0"
-              ];
-            };
-          in map high-load [ 1 2 3 4 5 6 7 8 ];
-      };
-   };
    api = {
       enable = true;
-      package = pkgs.graphiteApi;
-      listenAddress = "127.0.0.1";
+      # package = pkgs.graphiteApi;
+      #listenAddress = "127.0.0.1";
+      listenAddress = "0.0.0.0";
       port = 18080;
     };
     carbon = {
@@ -99,8 +57,11 @@ with import <stockholm/lib>;
       config = ''
         [cache]
         MAX_CACHE_SIZE = inf
-        MAX_UPDATES_PER_SECOND = 10
+        MAX_UPDATES_PER_SECOND = 3
         MAX_CREATES_PER_MINUTE = 5000
+        LOG_UPDATES = False
+        LOG_CACHE_HITS = False
+        LOG_CACHE_QUEUE_SORTS = False
         '';
       storageSchemas = ''
         [carbon]
@@ -122,8 +83,4 @@ with import <stockholm/lib>;
     };
   };
 
-  networking.firewall = {
-    allowedTCPPorts = [ 2003 80 443 ];
-    allowedUDPPorts = [ 2003 ];
-  };
 }
diff --git a/makefu/2configs/logging/central-stats-client.nix b/makefu/2configs/logging/central-stats-client.nix
new file mode 100644
index 000000000..dd6ddddaf
--- /dev/null
+++ b/makefu/2configs/logging/central-stats-client.nix
@@ -0,0 +1,60 @@
+{pkgs, config, ...}:
+{
+  services.collectd = {
+    enable = true;
+    autoLoadPlugin = true;
+    extraConfig = ''
+      Hostname ${config.krebs.build.host.name}
+      LoadPlugin load
+      LoadPlugin disk
+      LoadPlugin memory
+      LoadPlugin df
+      Interval 30.0
+
+      LoadPlugin interface
+      <Plugin "interface">
+        Interface "*Link"
+        Interface "lo"
+        Interface "vboxnet*"
+        Interface "virbr*"
+        IgnoreSelected true
+      </Plugin>
+
+      LoadPlugin df
+      <Plugin "df">
+        MountPoint "/nix/store"
+        # MountPoint "/run*"
+        # MountPoint "/sys*"
+        # MountPoint "/dev"
+        # MountPoint "/dev/shm"
+        # MountPoint "/tmp"
+        FSType "tmpfs"
+        FSType "binfmt_misc"
+        FSType "debugfs"
+        FSType "mqueue"
+        FSType "hugetlbfs"
+        FSType "systemd-1"
+        FSType "cgroup"
+        FSType "securityfs"
+        FSType "ramfs"
+        FSType "proc"
+        FSType "devpts"
+        FSType "devtmpfs"
+        MountPoint "/var/lib/docker/devicemapper"
+        IgnoreSelected true
+      </Plugin>
+
+      LoadPlugin cpu
+      <Plugin cpu>
+        ReportByCpu true
+        ReportByState true
+        ValuesPercentage true
+      </Plugin>
+
+      LoadPlugin network
+      <Plugin "network">
+          Server "${config.makefu.stats-server}" "25826"
+      </Plugin>
+    '';
+  };
+}
diff --git a/makefu/2configs/logging/central-stats-server.nix b/makefu/2configs/logging/central-stats-server.nix
new file mode 100644
index 000000000..69e37e493
--- /dev/null
+++ b/makefu/2configs/logging/central-stats-server.nix
@@ -0,0 +1,36 @@
+{pkgs, config, ...}:
+
+with import <stockholm/lib>;
+let
+  collectd-port = 25826;
+  influx-port = 8086;
+  grafana-port = 3000; # TODO nginx forward
+in {
+  services.grafana.enable = true;
+  services.grafana.addr = "0.0.0.0";
+
+  services.influxdb.enable = true;
+
+  # forward these via nginx
+  services.influxdb.extraConfig = {
+    meta.hostname = config.krebs.build.host.name;
+    # meta.logging-enabled = true;
+    http.bind-address = ":${toString influx-port}";
+    admin.bind-address = ":8083";
+    monitoring = {
+      enabled = false;
+      # write-interval = "24h";
+    };
+    collectd = [{
+      enabled = true;
+      typesdb = "${pkgs.collectd}/share/collectd/types.db";
+      database = "collectd_db";
+      port = collectd-port;
+    }];
+  };
+  networking.firewall.extraCommands = ''
+    iptables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT
+    iptables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT
+    iptables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT
+  '';
+}
diff --git a/makefu/2configs/nginx/share-download.nix b/makefu/2configs/nginx/share-download.nix
new file mode 100644
index 000000000..65c44b294
--- /dev/null
+++ b/makefu/2configs/nginx/share-download.nix
@@ -0,0 +1,18 @@
+{ config, lib, pkgs, ... }:
+
+with import <stockholm/lib>;
+{
+  services.nginx = {
+    enable = mkDefault true;
+    recommendedGzipSettings = true;
+    recommendedOptimisation = true;
+    virtualHosts."dl.euer.krebsco.de" = {
+        root = config.makefu.dl-dir;
+        extraConfig = "autoindex on;";
+        forceSSL = true;
+        enableSSL = true;
+        enableACME = true;
+        basicAuth = import <secrets/dl.euer.krebsco.de-auth.nix>;
+    };
+  };
+}
diff --git a/makefu/2configs/omo-share.nix b/makefu/2configs/omo-share.nix
index d1a9fabd7..8a3eab98a 100644
--- a/makefu/2configs/omo-share.nix
+++ b/makefu/2configs/omo-share.nix
@@ -42,6 +42,12 @@ in {
         browseable = "yes";
         "guest ok" = "yes";
       };
+      crypt0 = {
+        path = "/media/crypt0";
+        "read only" = "yes";
+        browseable = "yes";
+        "guest ok" = "yes";
+      };
       crypt0-rw = {
         path = "/media/crypt0/";
         "read only" = "no";
diff --git a/makefu/2configs/torrent.nix b/makefu/2configs/torrent.nix
index f3bc9091b..69d40094f 100644
--- a/makefu/2configs/torrent.nix
+++ b/makefu/2configs/torrent.nix
@@ -8,13 +8,13 @@ let
   peer-port = 51412;
   web-port = 8112;
   daemon-port = 58846;
-  dl-dir = "/var/download";
+  dl-dir = config.makefu.dl-dir;
 in {
   # prepare secrets
   krebs.build.source.torrent-secrets.file =
     if getEnv "dummy_secrets" == "true"
     then toString <stockholm/makefu/6tests/data/secrets>
-    else "/home/makefu/secrets/torrent";
+    else config.makefu.torrent-secrets ;
 
   users.users = {
     download = {
diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix
index 16215b27a..546fed109 100644
--- a/makefu/3modules/default.nix
+++ b/makefu/3modules/default.nix
@@ -8,9 +8,11 @@ _:
     ./forward-journal.nix
     ./opentracker.nix
     ./ps3netsrv.nix
+    ./logging-config.nix
     ./server-config.nix
     ./snapraid.nix
     ./taskserver.nix
+    ./torrent.nix
     ./udpt.nix
     ./umts.nix
   ];
diff --git a/makefu/3modules/logging-config.nix b/makefu/3modules/logging-config.nix
new file mode 100644
index 000000000..cd6924f61
--- /dev/null
+++ b/makefu/3modules/logging-config.nix
@@ -0,0 +1,11 @@
+{config, lib, pkgs, ... }:
+
+with import <stockholm/lib>;
+{
+  options.makefu.stats-server = lib.mkOption {
+    type = types.str;
+    default = "omo.retiolum";
+    description = "Central stats server (collectd)";
+  };
+}
+
diff --git a/makefu/3modules/torrent.nix b/makefu/3modules/torrent.nix
new file mode 100644
index 000000000..9f1cf3398
--- /dev/null
+++ b/makefu/3modules/torrent.nix
@@ -0,0 +1,13 @@
+{config, lib, pkgs, ... }:
+
+{
+  options.makefu.dl-dir = lib.mkOption {
+    type = lib.types.str;
+    description = "Default download directory";
+    default = "/media/cryptX/torrent";
+  };
+  options.makefu.torrent-secrets = lib.mkOption {
+    type = lib.types.str;
+    default = "/home/makefu/secrets/torrent";
+  };
+}
diff --git a/makefu/6tests/data/secrets/dl.euer.krebsco.de-auth.nix b/makefu/6tests/data/secrets/dl.euer.krebsco.de-auth.nix
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/makefu/6tests/data/secrets/dl.euer.krebsco.de-auth.nix
@@ -0,0 +1 @@
+{}