Merge remote-tracking branch 'pnp/master'

20 files changed, 352 insertions, 26 deletions

diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix
index e36a083f2..2d33b9275 100644
--- a/krebs/3modules/makefu/default.nix
+++ b/krebs/3modules/makefu/default.nix
@@ -164,6 +164,7 @@ with lib;
       dc = "makefu"; #dc = "cac";
       extraZones = {
         "krebsco.de" = ''
+          wiki.euer      IN A  ${head nets.internet.addrs4}
           wry            IN A  ${head nets.internet.addrs4}
           io             IN NS wry.krebsco.de.
           graphs         IN A  ${head nets.internet.addrs4}
@@ -189,6 +190,7 @@ with lib;
             "paste.wry.retiolum"
             "paste.retiolum"
             "wry.retiolum"
+            "wiki.makefu.retiolum"
           ];
           tinc.pubkey = ''
             -----BEGIN RSA PUBLIC KEY-----
@@ -214,8 +216,8 @@ with lib;
 
       extraZones = {
         "krebsco.de" = ''
-          omo               IN A      ${head nets.internet.addrs4}
           euer              IN A      ${head nets.internet.addrs4}
+          share.euer        IN A      ${head nets.internet.addrs4}
           gum               IN A      ${head nets.internet.addrs4}
         '';
       };
diff --git a/krebs/3modules/tinc_graphs.nix b/krebs/3modules/tinc_graphs.nix
index a6c628353..e415d20ab 100644
--- a/krebs/3modules/tinc_graphs.nix
+++ b/krebs/3modules/tinc_graphs.nix
@@ -95,8 +95,12 @@ let
 
         ExecStartPre = pkgs.writeScript "tinc_graphs-init" ''
           #!/bin/sh
+          mkdir -p "${internal_dir}" "${external_dir}"
           if ! test -e "${cfg.workingDir}/internal/index.html"; then
-            cp -fr "$(${pkgs.tinc_graphs}/bin/tincstats-static-dir)/internal/" "${internal_dir}"
+            cp -fr "$(${pkgs.tinc_graphs}/bin/tincstats-static-dir)/internal/." "${internal_dir}"
+          fi
+          if ! test -e "${cfg.workingDir}/external/index.html"; then
+            cp -fr "$(${pkgs.tinc_graphs}/bin/tincstats-static-dir)/external/." "${external_dir}"
           fi
         '';
 
@@ -118,7 +122,6 @@ let
     users.extraUsers.tinc_graphs = {
       uid = 3925439960; #genid tinc_graphs
       home = "/var/spool/tinc_graphs";
-      createHome = true;
     };
 
     krebs.nginx.servers = mkIf cfg.nginx.enable {
diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix
index 80d9f5e93..206bc5697 100644
--- a/krebs/3modules/urlwatch.nix
+++ b/krebs/3modules/urlwatch.nix
@@ -56,6 +56,13 @@ let
         https://nixos.org/channels/nixos-unstable/git-revision
       ];
     };
+    verbose = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        verbose output of urlwatch
+      '';
+    };
   };
 
   urlsFile = toFile "urls" (concatStringsSep "\n" cfg.urls);
@@ -106,7 +113,7 @@ let
 
           cd /tmp
 
-          urlwatch -e --urls="$urlsFile" > changes 2>&1 || :
+          urlwatch -e ${optionalString cfg.verbose "-v"} --urls="$urlsFile" > changes || :
 
           if test -s changes; then
             date=$(date -R)
diff --git a/krebs/5pkgs/collectd-connect-time/default.nix b/krebs/5pkgs/collectd-connect-time/default.nix
new file mode 100644
index 000000000..b07a119db
--- /dev/null
+++ b/krebs/5pkgs/collectd-connect-time/default.nix
@@ -0,0 +1,15 @@
+{lib, pkgs, pythonPackages, fetchurl, ... }:
+
+pythonPackages.buildPythonPackage rec {
+  name = "collectd-connect-time-${version}";
+  version = "0.2.1";
+  src = fetchurl {
+    url = "https://pypi.python.org/packages/source/c/collectd-connect-time/collectd-connect-time-${version}.tar.gz";
+    sha256 = "0611h53ww9lk1qm27njsffckkibirmq7p8cxlq02dgl1zbh7583d";
+  };
+  meta = {
+    homepage = https://pypi.python.org/pypi/collectd-connect-time/;
+    description = "TCP Connection time plugin for collectd";
+    license = lib.licenses.wtfpl;
+  };
+}
diff --git a/makefu/5pkgs/tinc_graphs/default.nix b/krebs/5pkgs/tinc_graphs/default.nix
index 62a787d30..e5f1e40e8 100644
--- a/makefu/5pkgs/tinc_graphs/default.nix
+++ b/krebs/5pkgs/tinc_graphs/default.nix
@@ -2,14 +2,14 @@
 
 python3Packages.buildPythonPackage rec {
   name = "tinc_graphs-${version}";
-  version = "0.3.6";
+  version = "0.3.9";
   propagatedBuildInputs = with pkgs;[
     python3Packages.pygeoip
     ## ${geolite-legacy}/share/GeoIP/GeoIPCity.dat
   ];
   src = fetchurl {
     url = "https://pypi.python.org/packages/source/t/tinc_graphs/tinc_graphs-${version}.tar.gz";
-    sha256 = "0ghdx9aaipmppvc2b6cgks4nxw6zsb0fhjrmnisbx7rz0vjvzc74";
+    sha256 = "0hjmkiclvyjb3707285x4b8mk5aqjcvh383hvkad1h7p1n61qrfx";
   };
   preFixup = with pkgs;''
     wrapProgram $out/bin/build-graphs --prefix PATH : "$out/bin"
diff --git a/makefu/1systems/gum.nix b/makefu/1systems/gum.nix
new file mode 100644
index 000000000..c4fa064b3
--- /dev/null
+++ b/makefu/1systems/gum.nix
@@ -0,0 +1,44 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  external-ip = head config.krebs.build.host.nets.internet.addrs4;
+  internal-ip = head config.krebs.build.host.nets.retiolum.addrs4;
+in {
+  imports = [
+      # TODO: copy this config or move to krebs
+      ../2configs/base.nix
+      ../2configs/base-sources.nix
+      ../2configs/tinc-basic-retiolum.nix
+      ../2configs/headless.nix
+      # ../2configs/iodined.nix
+
+      # Reaktor
+      ../2configs/Reaktor/simpleExtend.nix
+  ];
+
+  krebs.build = {
+    user = config.krebs.users.makefu;
+    target = "root@gum.krebsco.de";
+    host = config.krebs.hosts.gum;
+  };
+
+  krebs.Reaktor.enable = true;
+
+  # prepare graphs
+  krebs.nginx.enable = true;
+
+  networking = {
+    firewall.allowPing = true;
+    firewall.allowedTCPPorts = [ 80 443 655 ];
+    firewall.allowedUDPPorts = [ 655 ];
+    interfaces.enp2s1.ip4 = [{
+      address = external-ip;
+      prefixLength = 24;
+    }];
+    defaultGateway = "195.154.108.1";
+    nameservers = [ "8.8.8.8" ];
+  };
+
+  # based on ../../tv/2configs/CAC-Developer-2.nix
+}
diff --git a/makefu/1systems/pnp.nix b/makefu/1systems/pnp.nix
index e7ceca60d..9c7be3b79 100644
--- a/makefu/1systems/pnp.nix
+++ b/makefu/1systems/pnp.nix
@@ -11,6 +11,7 @@
       ../2configs/base.nix
       ../2configs/base-sources.nix
       ../2configs/tinc-basic-retiolum.nix
+      ../2configs/headless.nix
 
       # HW/FS
       <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
@@ -32,6 +33,8 @@
 
       # ../2configs/graphite-standalone.nix
     ];
+  krebs.urlwatch.verbose = true;
+
   krebs.Reaktor.enable = true;
   krebs.Reaktor.debug = true;
   krebs.Reaktor.nickname = "Reaktor|bot";
diff --git a/makefu/1systems/wry.nix b/makefu/1systems/wry.nix
index 3baf31dc5..c90b84451 100644
--- a/makefu/1systems/wry.nix
+++ b/makefu/1systems/wry.nix
@@ -11,6 +11,7 @@ in {
       ../../tv/2configs/CAC-CentOS-7-64bit.nix
       ../2configs/base.nix
       ../2configs/unstable-sources.nix
+      ../2configs/headless.nix
       ../2configs/tinc-basic-retiolum.nix
 
       ../2configs/bepasty-dual.nix
@@ -19,6 +20,11 @@ in {
 
       # Reaktor
       ../2configs/Reaktor/simpleExtend.nix
+
+      # other nginx
+      ../2configs/nginx/euer.wiki.nix
+      # collectd
+      ../2configs/collectd/collectd-base.nix
   ];
 
   krebs.build = {
@@ -27,8 +33,6 @@ in {
     host = config.krebs.hosts.wry;
   };
 
-
-
   krebs.Reaktor.enable = true;
 
   # bepasty to listen only on the correct interfaces
@@ -58,6 +62,7 @@ in {
   networking = {
     firewall.allowPing = true;
     firewall.allowedTCPPorts = [ 53 80 443 ];
+    firewall.allowedUDPPorts = [ 655 ];
     interfaces.enp2s1.ip4 = [{
       address = external-ip;
       prefixLength = 24;
@@ -67,6 +72,4 @@ in {
   };
 
 
-  # based on ../../tv/2configs/CAC-Developer-2.nix
-  sound.enable = false;
 }
diff --git a/makefu/2configs/base-sources.nix b/makefu/2configs/base-sources.nix
index 7e6bebec3..65c6e8e76 100644
--- a/makefu/2configs/base-sources.nix
+++ b/makefu/2configs/base-sources.nix
@@ -1,6 +1,7 @@
 { config, lib, pkgs, ... }:
 
 {
+  system.stateVersion = "15.09";
   krebs.build.source = {
     git.nixpkgs = {
       #url = https://github.com/NixOS/nixpkgs;
diff --git a/makefu/2configs/bepasty-dual.nix b/makefu/2configs/bepasty-dual.nix
index fb90f4c9f..123ae3cf9 100644
--- a/makefu/2configs/bepasty-dual.nix
+++ b/makefu/2configs/bepasty-dual.nix
@@ -11,7 +11,11 @@
 #   bepasty-secret.nix     <- contains single string
 
 with lib;
-{
+let
+  sec = toString <secrets>;
+  # secKey is nothing worth protecting on a local machine
+  secKey = import <secrets/bepasty-secret.nix>;
+in {
 
   krebs.nginx.enable = mkDefault true;
   krebs.bepasty = {
@@ -24,28 +28,28 @@ with lib;
           server-names = [ "paste.retiolum" "paste.${config.krebs.build.host.name}" ];
         };
         defaultPermissions = "admin,list,create,read,delete";
-        secretKey = import <secrets/bepasty-secret.nix>;
+        secretKey = secKey;
       };
 
       external = {
         nginx = {
           server-names = [ "paste.krebsco.de" ];
           extraConfig = ''
-            ssl_session_cache    shared:SSL:1m;
-            ssl_session_timeout  10m;
-            ssl_certificate     /root/secrets/wildcard.krebsco.de.crt;
-            ssl_certificate_key /root/secrets/wildcard.krebsco.de.key;
-            ssl_verify_client off;
-            proxy_ssl_session_reuse off;
-            ssl_protocols        TLSv1 TLSv1.1 TLSv1.2;
-            ssl_ciphers RC4:HIGH:!aNULL:!MD5;
-            ssl_prefer_server_ciphers on;
-            if ($scheme = http){
-              return 301 https://$server_name$request_uri;
+          ssl_session_cache    shared:SSL:1m;
+          ssl_session_timeout  10m;
+          ssl_certificate     ${sec}/wildcard.krebsco.de.crt;
+          ssl_certificate_key ${sec}/wildcard.krebsco.de.key;
+          ssl_verify_client off;
+          proxy_ssl_session_reuse off;
+          ssl_protocols        TLSv1 TLSv1.1 TLSv1.2;
+          ssl_ciphers RC4:HIGH:!aNULL:!MD5;
+          ssl_prefer_server_ciphers on;
+          if ($scheme = http){
+            return 301 https://$server_name$request_uri;
           }'';
         };
         defaultPermissions = "read";
-        secretKey = import <secrets/bepasty-secret.nix>;
+        secretKey = secKey;
       };
     };
   };
diff --git a/makefu/2configs/collectd/collectd-base.nix b/makefu/2configs/collectd/collectd-base.nix
new file mode 100644
index 000000000..7909c1be5
--- /dev/null
+++ b/makefu/2configs/collectd/collectd-base.nix
@@ -0,0 +1,42 @@
+{ config, lib, pkgs, ... }:
+
+# graphite-web on port 8080
+# carbon cache on port 2003 (tcp/udp)
+with lib;
+let
+  connect-time-cfg = with pkgs; writeText "collectd-connect-time.cfg" ''
+    LoadPlugin python
+    <Plugin python>
+      ModulePath "${collectd-connect-time}/lib/${python.libPrefix}/site-packages/"
+      Import "collectd_connect_time"
+      <Module collectd_connect_time>
+        target "wry.retiolum" "localhost" "google.com"
+        interval 30
+      </Module>
+    </Plugin>
+  '';
+  graphite-cfg = pkgs.writeText "collectd-graphite-cfg" ''
+    LoadPlugin write_graphite
+    <Plugin "write_graphite">
+      <Carbon>
+        Host "heidi.retiolum"
+        Port "2003"
+        Prefix "retiolum."
+        EscapeCharacter "_"
+        StoreRates false
+        AlwaysAppendDS false
+      </Carbon>
+    </Plugin>
+  '';
+in {
+  imports = [ ];
+
+  nixpkgs.config.packageOverrides = pkgs: with pkgs; {
+    collectd = pkgs.collectd.override { python= pkgs.python; };
+  };
+  services.collectd = {
+    enable = true;
+    include = [ (toString connect-time-cfg) (toString graphite-cfg) ];
+  };
+
+}
diff --git a/makefu/2configs/headless.nix b/makefu/2configs/headless.nix
new file mode 100644
index 000000000..33847c5e1
--- /dev/null
+++ b/makefu/2configs/headless.nix
@@ -0,0 +1,4 @@
+_:
+{
+  sound.enable = false;
+}
diff --git a/makefu/2configs/nginx/euer.blog.nix b/makefu/2configs/nginx/euer.blog.nix
new file mode 100644
index 000000000..e97050ec4
--- /dev/null
+++ b/makefu/2configs/nginx/euer.blog.nix
@@ -0,0 +1,30 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  sec = toString <secrets>;
+  ssl_cert = "${sec}/wildcard.krebsco.de.crt";
+  ssl_key  = "${sec}/wildcard.krebsco.de.key";
+  hostname = krebs.build.host.name;
+in {
+  krebs.nginx = {
+    enable = mkDefault true;
+    servers = {
+      euer-blog = {
+        listen = [ "80" "443 ssl" ];
+        server-names = [ "euer.krebsco.de" "euer.blog.krebsco.de" "blog.${hostname}" ];
+        extraConfig = ''
+          gzip on;
+          gzip_buffers 4 32k;
+          gzip_types  text/plain application/x-javascript text/css;
+          ssl_certificate ${ssl_cert};
+          ssl_certificate_key ${ssl_key};
+          default_type text/plain;
+        '';
+        locations = singleton (nameValuePair "/" ''
+          root /var/www/euer.blog/;
+        '');
+      };
+    };
+  };
+}
diff --git a/makefu/2configs/nginx/euer.wiki.nix b/makefu/2configs/nginx/euer.wiki.nix
new file mode 100644
index 000000000..fbcfe2047
--- /dev/null
+++ b/makefu/2configs/nginx/euer.wiki.nix
@@ -0,0 +1,115 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  sec = toString <secrets>;
+  ssl_cert = "${sec}/wildcard.krebsco.de.crt";
+  ssl_key  = "${sec}/wildcard.krebsco.de.key";
+  user = config.services.nginx.user;
+  group = config.services.nginx.group;
+  fpm-socket = "/var/run/php5-fpm.sock";
+  hostname = config.krebs.build.host.name;
+  tw-upload = pkgs.tw-upload-plugin;
+  base-dir = "/var/www/wiki.euer";
+  base-cfg = "${base-dir}/twconf.ini";
+  wiki-dir = "${base-dir}/store/";
+  backup-dir = "${base-dir}/backup/";
+  # contains:
+  #  user1 = pass1
+  #  userN = passN
+  tw-pass-file = "${sec}/tw-pass.ini";
+  external-ip = head config.krebs.build.host.nets.internet.addrs4;
+  internal-ip = head config.krebs.build.host.nets.retiolum.addrs4;
+in {
+  services.phpfpm = {
+    # phpfpm does not have an enable option
+    poolConfigs  = {
+      euer-wiki = ''
+        user =  ${user}
+        group =  ${group}
+        listen = ${fpm-socket}
+        listen.owner = ${user}
+        listen.group = ${group}
+        env[twconf] = ${base-cfg};
+        pm = dynamic
+        pm.max_children = 5
+        pm.start_servers = 2
+        pm.min_spare_servers = 1
+        pm.max_spare_servers = 3
+        chdir = /
+        # errors to journal
+        php_admin_value[error_log] = 'stderr'
+        php_admin_flag[log_errors] = on
+        catch_workers_output = yes
+      '';
+    };
+  };
+
+  systemd.services.prepare-tw = {
+    wantedBy = [ "local-fs.target" ];
+    before = [ "phpfpm.service" ];
+    serviceConfig = {
+      ExecStart = pkgs.writeScript "prepare-tw-service" ''
+        #!/bin/sh
+        mkdir -p "${wiki-dir}" "${backup-dir}"
+
+        # write the base configuration
+        cat > "${base-cfg}" <<EOF
+        [users]
+        $(cat "${tw-pass-file}")
+        [directories]
+        backupdir = ${backup-dir}
+        savedir = ${wiki-dir}
+        EOF
+        chown -R ${user}:${group} "${base-dir}"
+        chmod 700  -R "${base-dir}"
+      '';
+      Type = "oneshot";
+      RemainAfterExit = "yes";
+      TimeoutSec = "0";
+    };
+  };
+
+  krebs.nginx = {
+    enable = mkDefault true;
+    servers = {
+      euer-wiki = {
+        listen = [ "${external-ip}:80" "${external-ip}:443 ssl"
+                   "${internal-ip}:80" "${internal-ip}:443 ssl" ];
+        server-names = [
+          "wiki.euer.krebsco.de"
+          "wiki.makefu.retiolum"
+          "wiki.makefu"
+        ];
+        extraConfig = ''
+          gzip on;
+          gzip_buffers 4 32k;
+          gzip_types  text/plain application/x-javascript text/css;
+          ssl_certificate ${ssl_cert};
+          ssl_certificate_key ${ssl_key};
+          default_type text/plain;
+
+          if ($scheme = http){
+            return 301 https://$server_name$request_uri;
+          }
+
+        '';
+        locations = [
+          (nameValuePair "/" ''
+            root ${wiki-dir};
+            expires -1;
+            autoindex on;
+          '')
+          (nameValuePair "/store.php" ''
+            root ${tw-upload};
+            client_max_body_size 200M;
+            fastcgi_split_path_info ^(.+\.php)(/.+)$;
+            fastcgi_pass unix:${fpm-socket};
+            include ${pkgs.nginx}/conf/fastcgi_params;
+            include ${pkgs.nginx}/conf/fastcgi.conf;
+          '')
+        ];
+      };
+    };
+  };
+}
diff --git a/makefu/2configs/unstable-sources.nix b/makefu/2configs/unstable-sources.nix
index f2d28dcaf..7bd5f50cb 100644
--- a/makefu/2configs/unstable-sources.nix
+++ b/makefu/2configs/unstable-sources.nix
@@ -1,10 +1,11 @@
 { config, lib, pkgs, ... }:
 
 {
+  system.stateVersion = "15.09";
   krebs.build.source = {
     git.nixpkgs = {
       url = https://github.com/makefu/nixpkgs;
-      rev = "984d33884d63d404ff2da76920b8bc8b15471552";
+      rev = "15b5bbfbd1c8a55e7d9e05dd9058dc102fac04fe"; # cherry-picked collectd
     };
 
     dir.secrets = {
diff --git a/makefu/2configs/urlwatch.nix b/makefu/2configs/urlwatch.nix
index 933cb93c5..cd05f0114 100644
--- a/makefu/2configs/urlwatch.nix
+++ b/makefu/2configs/urlwatch.nix
@@ -10,6 +10,8 @@
       https://api.github.com/repos/ovh/python-ovh/tags
       https://api.github.com/repos/embray/d2to1/tags
       http://git.sysphere.org/vicious/log/?qt=grep&q=Next+release
+      https://pypi.python.org/simple/bepasty/
+      https://pypi.python.org/simple/xstatic/
 
     ];
   };
diff --git a/makefu/5pkgs/default.nix b/makefu/5pkgs/default.nix
index 7945b6ebd..436c52fcd 100644
--- a/makefu/5pkgs/default.nix
+++ b/makefu/5pkgs/default.nix
@@ -7,6 +7,6 @@ in
   alsa-hdspmixer = callPackage ./alsa-tools { alsaToolTarget="hdspmixer";};
   alsa-hdspconf = callPackage ./alsa-tools { alsaToolTarget="hdspconf";};
   alsa-hdsploader = callPackage ./alsa-tools { alsaToolTarget="hdsploader";};
-  tinc_graphs = callPackage ./tinc_graphs {};
   awesomecfg = callPackage ./awesomecfg {};
+  tw-upload-plugin = callPackage ./tw-upload-plugin {};
 }
diff --git a/makefu/5pkgs/tw-upload-plugin/default.nix b/makefu/5pkgs/tw-upload-plugin/default.nix
new file mode 100644
index 000000000..a68dc09dc
--- /dev/null
+++ b/makefu/5pkgs/tw-upload-plugin/default.nix
@@ -0,0 +1,8 @@
+{pkgs}:
+
+pkgs.fetchFromGitHub {
+  owner = "makefu";
+  repo = "tw-upload-plugin";
+  rev = "a00aac";
+  sha256 = "0kazqs24kzjxqzr33kg1jbfx8xyvmrnrdxh6g27kgkgbl1d2qknh";
+}
diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix
index aeaeee288..73552e705 100644
--- a/shared/1systems/wolf.nix
+++ b/shared/1systems/wolf.nix
@@ -5,6 +5,7 @@ with lib;
 {
   imports = [
     <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
+    ../2configs/collectd-base.nix
   ];
 
   krebs.build.host = config.krebs.hosts.wolf;
diff --git a/shared/2configs/collectd-base.nix b/shared/2configs/collectd-base.nix
new file mode 100644
index 000000000..b2ec40b28
--- /dev/null
+++ b/shared/2configs/collectd-base.nix
@@ -0,0 +1,41 @@
+{ config, lib, pkgs, ... }:
+
+# TODO: krebs.collectd.plugins
+with lib;
+let
+  connect-time-cfg = with pkgs; writeText "collectd-connect-time.conf" ''
+    LoadPlugin python
+    <Plugin python>
+      ModulePath "${collectd-connect-time}/lib/${python.libPrefix}/site-packages/"
+      Import "collectd_connect_time"
+      <Module collectd_connect_time>
+        target "heidi.retiolum:8080" "localhost" "google.com" "google.de" "omo.retiolum" "gum.retiolum" "gum.krebsco.de"
+        interval 10
+      </Module>
+    </Plugin>
+  '';
+  graphite-cfg = pkgs.writeText "collectd-graphite.conf" ''
+    LoadPlugin write_graphite
+    <Plugin "write_graphite">
+      <Carbon>
+        Host "heidi.retiolum"
+        Port "2003"
+        Prefix "retiolum."
+        EscapeCharacter "_"
+        StoreRates false
+        AlwaysAppendDS false
+      </Carbon>
+    </Plugin>
+  '';
+in {
+  imports = [ ];
+
+  nixpkgs.config.packageOverrides = pkgs: with pkgs; {
+    collectd = pkgs.collectd.override { python= pkgs.python; };
+  };
+  services.collectd = {
+    enable = true;
+    include = [ (toString connect-time-cfg) (toString graphite-cfg) ];
+  };
+
+}