Merge remote-tracking branch 'prism/master'

44 files changed, 427 insertions, 204 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 000000000..3f2f28d65
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,5 @@
+nix-shell test:
+  script:
+    - env
+    - nix-shell --pure --command 'true' -p stdenv && echo success
+    - nix-shell --pure --command 'false' -p stdenv || echo success
diff --git a/krebs/1systems/hotdog/config.nix b/krebs/1systems/hotdog/config.nix
index 4807307ff..2ad22f49c 100644
--- a/krebs/1systems/hotdog/config.nix
+++ b/krebs/1systems/hotdog/config.nix
@@ -10,6 +10,7 @@
     <stockholm/krebs/2configs>
 
     <stockholm/krebs/2configs/buildbot-all.nix>
+    <stockholm/krebs/2configs/gitlab-runner-shackspace.nix>
     <stockholm/krebs/2configs/binary-cache/nixos.nix>
   ];
 
diff --git a/krebs/2configs/gitlab-runner-shackspace.nix b/krebs/2configs/gitlab-runner-shackspace.nix
new file mode 100644
index 000000000..d9b4cd589
--- /dev/null
+++ b/krebs/2configs/gitlab-runner-shackspace.nix
@@ -0,0 +1,34 @@
+{ config, ... }:
+let
+  url = "https://git.shackspace.de/";
+  # generate token from CI-token via:
+  ## gitlab-runner register
+  ## cat /etc/gitlab-runner/config.toml
+  token = import <secrets/shackspace-gitlab-ci-token.nix> ;
+in {
+  systemd.services.gitlab-runner.path = [ 
+    "/run/wrappers" # /run/wrappers/bin/su
+    "/" # /bin/sh
+  ];
+  systemd.services.gitlab-runner.serviceConfig.PrivateTmp = true;
+  virtualisation.docker.enable = true;
+  services.gitlab-runner = {
+    enable = true;
+    # configFile, configOptions and gracefulTimeout not yet in stable
+    # gracefulTimeout = "120min";
+    configText = ''
+    concurrent = 1
+    check_interval = 0
+
+    [[runners]]
+      name = "krebs-shell"
+      url = "${url}"
+      token = "${token}"
+      executor = "shell"
+      shell = "sh"
+      environment = ["PATH=/bin:/run/wrappers/bin:/etc/per-user/gitlab-runner/bin:/etc/per-user-pkgs/gitlab-runner/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin"]
+      [runners.cache]
+
+    '';
+  };
+}
diff --git a/krebs/3modules/newsbot-js.nix b/krebs/3modules/newsbot-js.nix
index 2ff9a5ebb..dd3e5647a 100644
--- a/krebs/3modules/newsbot-js.nix
+++ b/krebs/3modules/newsbot-js.nix
@@ -13,6 +13,11 @@ let
 
   api = {
     enable = mkEnableOption "Enable krebs newsbot";
+    package = mkOption {
+      type = types.package;
+      default = pkgs.newsbot-js;
+      description = "newsbot package to use";
+    };
     ircServer = mkOption {
       type = types.str;
       default = "echelon.retiolum";
@@ -79,7 +84,7 @@ let
       serviceConfig = {
         User = "newsbot-js";
         Restart = "always";
-        ExecStart = "${pkgs.newsbot-js}/bin/newsbot";
+        ExecStart = "${cfg.package}/bin/newsbot";
       };
     };
   };
diff --git a/krebs/3modules/nin/default.nix b/krebs/3modules/nin/default.nix
index d5d13cd1a..aab568352 100644
--- a/krebs/3modules/nin/default.nix
+++ b/krebs/3modules/nin/default.nix
@@ -3,7 +3,10 @@
 with import <stockholm/lib>;
 
 {
-  hosts = mapAttrs (_: setAttr "owner" config.krebs.users.nin) {
+  hosts = mapAttrs (_: recursiveUpdate {
+    owner = config.krebs.users.nin;
+    ci = true;
+  }) {
     hiawatha = {
       cores = 2;
       nets = {
diff --git a/krebs/3modules/rtorrent.nix b/krebs/3modules/rtorrent.nix
index d85518993..472accef9 100644
--- a/krebs/3modules/rtorrent.nix
+++ b/krebs/3modules/rtorrent.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, pkgs, options, ... }:
 
 with import <stockholm/lib>;
 let
@@ -73,6 +73,15 @@ let
       # authentication also applies to rtorrent.rutorrent
       enable = mkEnableOption "rtorrent nginx web RPC";
 
+      addr = mkOption {
+        type = types.addr4;
+        default = "0.0.0.0";
+        description = ''
+          the address to listen on
+          default is 0.0.0.0
+        '';
+      };
+
       port = mkOption {
         type = types.nullOr types.int;
         description =''
@@ -290,7 +299,7 @@ let
     services.nginx.enable = mkDefault true;
     services.nginx.virtualHosts.rtorrent = {
       default = mkDefault true;
-      inherit (webcfg) basicAuth port;
+      inherit (webcfg) basicAuth;
       root = optionalString rucfg.enable webdir;
 
       locations = {
@@ -310,7 +319,15 @@ let
           include ${pkgs.nginx}/conf/fastcgi.conf;
         ''; }
       );
-    };
+    # workaround because upstream nginx api changed
+    # TODO remove when nobody uses 17.03 anymore
+    } // (if hasAttr "port" (head options.services.nginx.virtualHosts.type.getSubModules).submodule.options then {
+      port = webcfg.port;
+    } else {
+      listen = [
+        { inherit (webcfg) addr port; }
+      ];
+    });
   };
 
   rutorrent-imp = {
diff --git a/krebs/5pkgs/simple/newsbot-js/default.nix b/krebs/5pkgs/simple/newsbot-js/default.nix
index b52454ca4..fa39823d3 100644
--- a/krebs/5pkgs/simple/newsbot-js/default.nix
+++ b/krebs/5pkgs/simple/newsbot-js/default.nix
@@ -32,6 +32,7 @@ in np.buildNodePackage {
 
   phases = [
     "unpackPhase"
+    "patchPhase"
     "installPhase"
   ];
 
diff --git a/krebs/6tests/data/secrets/shackspace-gitlab-ci-token.nix b/krebs/6tests/data/secrets/shackspace-gitlab-ci-token.nix
new file mode 100644
index 000000000..963e6db8b
--- /dev/null
+++ b/krebs/6tests/data/secrets/shackspace-gitlab-ci-token.nix
@@ -0,0 +1 @@
+"lol"
diff --git a/lass/source.nix b/lass/source.nix
index 52b0d99f2..01631bef1 100644
--- a/lass/source.nix
+++ b/lass/source.nix
@@ -15,7 +15,7 @@ in
         #   87a4615 & 334ac4f
         # + acme permissions for groups
         #   fd7a8f1
-        ref = "fe46ffc";
+        ref = "d151161";
       };
       secrets.file = getAttr builder {
         buildbot = toString <stockholm/lass/2configs/tests/dummy-secrets>;
diff --git a/makefu/1systems/gum/config.nix b/makefu/1systems/gum/config.nix
index 110edc130..934bfa685 100644
--- a/makefu/1systems/gum/config.nix
+++ b/makefu/1systems/gum/config.nix
@@ -43,6 +43,7 @@ in {
       <stockholm/makefu/2configs/torrent.nix>
       <stockholm/makefu/2configs/iodined.nix>
       <stockholm/makefu/2configs/vpn/openvpn-server.nix>
+      <stockholm/makefu/2configs/dnscrypt/server.nix>
 
       ## Web
       <stockholm/makefu/2configs/nginx/share-download.nix>
@@ -55,9 +56,7 @@ in {
       <stockholm/makefu/2configs/deployment/mycube.connector.one.nix>
       <stockholm/makefu/2configs/deployment/graphs.nix>
       <stockholm/makefu/2configs/deployment/owncloud.nix>
-      <stockholm/makefu/2configs/deployment/wiki-irc-bot>
       <stockholm/makefu/2configs/deployment/boot-euer.nix>
-      <stockholm/makefu/2configs/deployment/hound>
       {
         services.taskserver.enable = true;
         services.taskserver.fqdn = config.krebs.build.host.name;
diff --git a/makefu/1systems/omo/config.nix b/makefu/1systems/omo/config.nix
index 0df2ba6e6..4c93a7a3e 100644
--- a/makefu/1systems/omo/config.nix
+++ b/makefu/1systems/omo/config.nix
@@ -194,16 +194,6 @@ in {
 
   zramSwap.enable = true;
 
-  krebs.Reaktor.reaktor = {
-    nickname = "Reaktor|krebs";
-    workdir = "/var/lib/Reaktor/krebs";
-    channels = [ "#krebs" ];
-    plugins = with pkgs.ReaktorPlugins;[
-                               stockholm-issue
-                               nixos-version
-                               sed-plugin
-                               random-emoji ];
-  };
   krebs.Reaktor.reaktor-shack = {
     nickname = "Reaktor|shack";
     workdir = "/var/lib/Reaktor/shack";
diff --git a/makefu/1systems/x/config.nix b/makefu/1systems/x/config.nix
index 8e8c8a736..faa29f3db 100644
--- a/makefu/1systems/x/config.nix
+++ b/makefu/1systems/x/config.nix
@@ -12,7 +12,7 @@ with import <stockholm/lib>;
       <stockholm/makefu/2configs/extra-fonts.nix>
       <stockholm/makefu/2configs/tools/all.nix>
       <stockholm/makefu/2configs/laptop-backup.nix>
-      <stockholm/makefu/2configs/dnscrypt.nix>
+      <stockholm/makefu/2configs/dnscrypt/client.nix>
       <stockholm/makefu/2configs/avahi.nix>
 
       # Debugging
@@ -20,6 +20,7 @@ with import <stockholm/lib>;
 
       # Testing
       # <stockholm/makefu/2configs/deployment/gitlab.nix>
+      # <stockholm/makefu/2configs/deployment/wiki-irc-bot>
 
       # <stockholm/makefu/2configs/torrent.nix>
       # <stockholm/makefu/2configs/lanparty/lancache.nix>
@@ -54,6 +55,7 @@ with import <stockholm/lib>;
       # Services
       <stockholm/makefu/2configs/git/brain-retiolum.nix>
       <stockholm/makefu/2configs/tor.nix>
+      <stockholm/makefu/2configs/vpn/vpngate.nix>
       <stockholm/makefu/2configs/steam.nix>
       # <stockholm/makefu/2configs/buildbot-standalone.nix>
 
diff --git a/makefu/2configs/bepasty-dual.nix b/makefu/2configs/bepasty-dual.nix
index ecf5f8a38..701bf5b1d 100644
--- a/makefu/2configs/bepasty-dual.nix
+++ b/makefu/2configs/bepasty-dual.nix
@@ -36,7 +36,6 @@ in {
 
       "${ext-dom}" = {
         nginx = {
-          enableSSL = true;
           forceSSL = true;
           enableACME = true;
         };
diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix
index 547b73597..25f9f63bf 100644
--- a/makefu/2configs/default.nix
+++ b/makefu/2configs/default.nix
@@ -39,10 +39,10 @@ with import <stockholm/lib>;
   };
 
   networking.hostName = config.krebs.build.host.name;
-  nix.maxJobs = config.krebs.build.host.cores;
+  nix.maxJobs = 2;
+  nix.buildCores = config.krebs.build.host.cores;
 
   time.timeZone = "Europe/Berlin";
-  #nix.maxJobs = 1;
 
   programs.ssh = {
     startAgent = false;
@@ -145,15 +145,6 @@ with import <stockholm/lib>;
     "net.ipv6.conf.default.use_tempaddr" = 2;
   };
 
-  system.activationScripts.nix-defexpr = ''
-    (set -euf
-     for i in /home/makefu /root/;do
-       f="$i/.nix-defexpr"
-       rm -fr "$f"
-       ln -s /var/src/nixpkgs "$f"
-     done)
-  '';
-
   i18n = {
     consoleKeyMap = "us";
     defaultLocale = "en_US.UTF-8";
diff --git a/makefu/2configs/deployment/boot-euer.nix b/makefu/2configs/deployment/boot-euer.nix
index be24209bc..f890ea7ad 100644
--- a/makefu/2configs/deployment/boot-euer.nix
+++ b/makefu/2configs/deployment/boot-euer.nix
@@ -16,7 +16,6 @@ in {
   services.nginx = {
     enable = mkDefault true;
     virtualHosts."boot.euer.krebsco.de" = {
-      enableSSL = true;
       forceSSL = true;
       enableACME = true;
       locations."/" = {
diff --git a/makefu/2configs/deployment/graphs.nix b/makefu/2configs/deployment/graphs.nix
index b33ddece0..bde9892cd 100644
--- a/makefu/2configs/deployment/graphs.nix
+++ b/makefu/2configs/deployment/graphs.nix
@@ -28,7 +28,6 @@ in {
         ];
       };
       anonymous = {
-        enableSSL = true;
         forceSSL = true;
         enableACME = true;
       };
diff --git a/makefu/2configs/deployment/owncloud.nix b/makefu/2configs/deployment/owncloud.nix
index 65ac5c184..3a9d57dbb 100644
--- a/makefu/2configs/deployment/owncloud.nix
+++ b/makefu/2configs/deployment/owncloud.nix
@@ -21,7 +21,6 @@ let
       services.nginx.virtualHosts."${domain}" = {
         forceSSL = true;
         enableACME = true;
-        enableSSL = true;
         serverAliases = domains;
         extraConfig = ''
 
diff --git a/makefu/2configs/deployment/wiki-irc-bot/default.nix b/makefu/2configs/deployment/wiki-irc-bot/default.nix
index 7ab31e698..12686efba 100644
--- a/makefu/2configs/deployment/wiki-irc-bot/default.nix
+++ b/makefu/2configs/deployment/wiki-irc-bot/default.nix
@@ -1,67 +1,19 @@
-{ pkgs, lib, ... }:
+{ config, pkgs, ... }:
 
-with lib;
 let
-  port = 18872;
+  pkg = pkgs.lib.overrideDerivation pkgs.newsbot-js (original: {
+    patches = [ ./wiki-output.patch ];
+  });
+  newsfile = pkgs.writeText "feeds" ''
+    nixoswiki-bot|https://nixos.wiki/api.php?days=7&limit=50&hidecategorization=1&action=feedrecentchanges&feedformat=rss|#krebs
+  '';
 in {
-  nixpkgs.config.packageOverrides = pkgs: with pkgs; {
-    logstash = pkgs.stdenv.lib.overrideDerivation  pkgs.logstash (old: {
-      patches = [ ./irc-out-notice.patch ]; });
-  };
-  services.logstash = {
+  krebs.newsbot-js = {
     enable = true;
-    inputConfig = ''
-      http {
-        port => ${toString port}
-        host => "127.0.0.1"
-      }
-    '';
-    filterConfig = ''
-      if ([pages]) {
-        ruby {
-          code => '
-            require "net/http"
-            require "net/https"
-            http = Net::HTTP.new("git.io", 443)
-            http.use_ssl = true
-            lines = []
-            event["pages"].each {|p|
-              url = "#{p["html_url"]}/_compare/#{p["sha"]}"
-              short_url = begin
-                request = Net::HTTP::Post.new "/"
-                request.set_form_data ({"url" => url })
-                response = http.request(request)
-                response["location"]
-              end
-              lines << "\"#{p["title"]}\" #{p["action"]} by #{event["sender"]["login"]} #{short_url}"
-            }
-            event["output"] = lines.join("\n")
-          '
-        }
-      }
-    '';
-    outputConfig = ''
-      file { path => "/tmp/logs.json" codec => "json_lines" }
-      if [output] {
-        irc {
-          channels => [ "#krebs", "#nixos" ]
-          host => "irc.freenode.net"
-          nick => "nixos-users-wiki"
-          format => "%{output}"
-          notice => true
-        }
-      }
-    '';
-    plugins = [ ];
-  };
-
-  services.nginx = {
-    enable = lib.mkDefault true;
-    virtualHosts."ghook.krebsco.de" = {
-      locations."/".proxyPass = "http://localhost:${toString port}/";
-      enableSSL = true;
-      enableACME = true;
-      forceSSL = true;
-    };
+    package = pkg;
+    ircServer = "chat.freenode.net";
+    feeds = newsfile;
+    urlShortenerHost = "go";
+    urlShortenerPort = "80";
   };
 }
diff --git a/makefu/2configs/deployment/wiki-irc-bot/irc-out-notice.patch b/makefu/2configs/deployment/wiki-irc-bot/irc-out-notice.patch
deleted file mode 100644
index 040643f81..000000000
--- a/makefu/2configs/deployment/wiki-irc-bot/irc-out-notice.patch
+++ /dev/null
@@ -1,26 +0,0 @@
-index b63339d..8c8c747 100644
---- a/vendor/bundle/jruby/1.9/gems/logstash-output-irc-2.0.4/lib/logstash/outputs/irc.rb
-+++ b/vendor/bundle/jruby/1.9/gems/logstash-output-irc-2.0.4/lib/logstash/outputs/irc.rb
-@@ -48,6 +48,9 @@ class LogStash::Outputs::Irc < LogStash::Outputs::Base
-   # Static string after event
-   config :post_string, :validate => :string, :required => false
- 
-+  # Set this to true to send messages as notice
-+  config :notice, :validate => :boolean, :default => false
-+
-   public
- 
-   def inject_bot(bot)
-@@ -90,9 +93,9 @@ class LogStash::Outputs::Irc < LogStash::Outputs::Base
- 
-     @bot.channels.each do |channel|
-       @logger.debug("Sending to...", :channel => channel, :text => text)
--      channel.msg(pre_string) if !@pre_string.nil?
--      channel.msg(text)
--      channel.msg(post_string) if !@post_string.nil?
-+      channel.send(pre_string, :notice => @notice) if !@pre_string.nil?
-+      channel.send(text, :notice => @notice)
-+      channel.send(post_string, :notice => @notice) if !@post_string.nil?
-     end # channels.each
-   end # def receive
- end # class LogStash::Outputs::Irc
diff --git a/makefu/2configs/deployment/wiki-irc-bot/wiki-output.patch b/makefu/2configs/deployment/wiki-irc-bot/wiki-output.patch
new file mode 100644
index 000000000..6e1e27853
--- /dev/null
+++ b/makefu/2configs/deployment/wiki-irc-bot/wiki-output.patch
@@ -0,0 +1,45 @@
+diff --git a/newsbot.js b/newsbot.js
+index 42d0666..a284011 100644
+--- a/newsbot.js
++++ b/newsbot.js
+@@ -92,8 +92,9 @@ function create_feedbot (nick, uri, channels) {
+   }
+ 
+   function broadcast_new_item (item) {
++    console.log('Broadcasting item ',item.link)
+     return getShortLink(item.link, function (error, shortlink) {
+-      return broadcast(item.title + ' ' + shortlink)
++      return broadcast('"'+ item.title + '" edited by ' + item.author +  ' ' + shortlink)
+     })
+   }
+   
+@@ -152,15 +153,18 @@ function create_feedbot (nick, uri, channels) {
+ 
+       if (client.lastItems) {
+         items.forEach(function (item) {
+-          if (!client.lastItems.hasOwnProperty(item.title)) {
++
++          if (!client.lastItems.hasOwnProperty(item.guid)) {
+             broadcast_new_item(item)
++          }else {
++            console.log("Item already seen:",item.guid)
+           }
+         })
+       }
+ 
+       client.lastItems = {}
+       items.forEach(function (item) {
+-        client.lastItems[item.title] = true
++        client.lastItems[item.guid] = true
+       })
+ 
+       return continue_loop()
+@@ -199,6 +203,8 @@ function run_command (methodname, params, callback) {
+ }
+ 
+ function getShortLink (link, callback) {
++  callback(null,link)
++  return
+   var form = new FormData()
+   try {
+     form.append('uri', link)
diff --git a/makefu/2configs/dnscrypt.nix b/makefu/2configs/dnscrypt.nix
deleted file mode 100644
index 6e7ef0f82..000000000
--- a/makefu/2configs/dnscrypt.nix
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-  services.dnscrypt-proxy.enable = true;
-  services.dnscrypt-proxy.resolverName = "cs-de";
-  networking.extraResolvconfConf = ''
-    name_servers='127.0.0.1'
-  '';
-}
diff --git a/makefu/2configs/dnscrypt/client.nix b/makefu/2configs/dnscrypt/client.nix
new file mode 100644
index 000000000..988fb4a7d
--- /dev/null
+++ b/makefu/2configs/dnscrypt/client.nix
@@ -0,0 +1,19 @@
+{ config, ... }:
+let
+  customResolver = {
+    # TODO: put this somewhere else
+    address = config.krebs.hosts.gum.nets.internet.ip4.addr;
+    port = 15251;
+    name = "2.dnscrypt-cert.euer.krebsco.de";
+    # dnscrypt-wrapper --show-provider-publickey --provider-publickey-file public.key
+    key = "1AFC:E58D:F242:0FBB:9EE9:4E51:47F4:5373:D9AE:C2AB:DD96:8448:333D:5D79:272C:A44C";
+  };
+in {
+  services.dnscrypt-proxy = {
+    enable = true;
+    inherit customResolver;
+  };
+  networking.extraResolvconfConf = ''
+    name_servers='127.0.0.1'
+  '';
+}
diff --git a/makefu/2configs/dnscrypt/server.nix b/makefu/2configs/dnscrypt/server.nix
new file mode 100644
index 000000000..79305e727
--- /dev/null
+++ b/makefu/2configs/dnscrypt/server.nix
@@ -0,0 +1,26 @@
+{ config, ... }:
+let
+    # TODO: dataDir is currently not provided by upstream
+    # data = config.services.dnscrypt-wrapper.dataDir;
+    data = "/var/lib/dnscrypt-wrapper";
+    sec = toString <secrets>;
+    port = 15251;
+    user = "dnscrypt-wrapper";
+in {
+  services.dnscrypt-wrapper = {
+    enable = true;
+    address = "0.0.0.0";
+    upstream.address = "8.8.8.8";
+    providerName = "2.dnscrypt-cert.euer.krebsco.de";
+    inherit port;
+  };
+  networking.firewall.allowedUDPPorts = [ port ];
+  systemd.services.prepare-dnscrypt-wrapper-keys = {
+    wantedBy = [ "dnscrypt-wrapper.service" ];
+    before = [ "dnscrypt-wrapper.service" ];
+    script = ''
+      install -m700 -o ${user} -v ${sec}/dnscrypt-public.key ${data}/public.key
+      install -m700 -o ${user} -v ${sec}/dnscrypt-secret.key ${data}/secret.key
+    '';
+  };
+}
diff --git a/makefu/2configs/git/cgit-retiolum.nix b/makefu/2configs/git/cgit-retiolum.nix
index b89bfadfd..30c0b0b87 100644
--- a/makefu/2configs/git/cgit-retiolum.nix
+++ b/makefu/2configs/git/cgit-retiolum.nix
@@ -22,6 +22,7 @@ let
       cgit.desc = "Build new Stockholm hosts";
     };
     cac-api = { };
+    euer_blog = { };
     ampel = { };
     init-stockholm = {
       cgit.desc = "Init stuff for stockholm";