Merge remote-tracking branch 'origin/staging/jeschli'

50 files changed, 711 insertions, 624 deletions

diff --git a/krebs/6tests/data/secrets/grafana_security.nix b/krebs/0tests/data/secrets/grafana_security.nix
index 0967ef424..0967ef424 100644
--- a/krebs/6tests/data/secrets/grafana_security.nix
+++ b/krebs/0tests/data/secrets/grafana_security.nix
diff --git a/krebs/6tests/data/secrets/hashedPasswords.nix b/krebs/0tests/data/secrets/hashedPasswords.nix
index 0967ef424..0967ef424 100644
--- a/krebs/6tests/data/secrets/hashedPasswords.nix
+++ b/krebs/0tests/data/secrets/hashedPasswords.nix
diff --git a/krebs/6tests/data/secrets/retiolum.rsa_key.priv b/krebs/0tests/data/secrets/retiolum.rsa_key.priv
index e69de29bb..e69de29bb 100644
--- a/krebs/6tests/data/secrets/retiolum.rsa_key.priv
+++ b/krebs/0tests/data/secrets/retiolum.rsa_key.priv
diff --git a/krebs/6tests/data/secrets/shackspace-gitlab-ci-token.nix b/krebs/0tests/data/secrets/shackspace-gitlab-ci-token.nix
index 963e6db8b..963e6db8b 100644
--- a/krebs/6tests/data/secrets/shackspace-gitlab-ci-token.nix
+++ b/krebs/0tests/data/secrets/shackspace-gitlab-ci-token.nix
diff --git a/krebs/6tests/data/secrets/ssh.id_ed25519 b/krebs/0tests/data/secrets/ssh.id_ed25519
index e69de29bb..e69de29bb 100644
--- a/krebs/6tests/data/secrets/ssh.id_ed25519
+++ b/krebs/0tests/data/secrets/ssh.id_ed25519
diff --git a/krebs/6tests/data/test-config.nix b/krebs/0tests/data/test-config.nix
index f0927ddd9..f0927ddd9 100644
--- a/krebs/6tests/data/test-config.nix
+++ b/krebs/0tests/data/test-config.nix
diff --git a/krebs/6tests/data/test-source.nix b/krebs/0tests/data/test-source.nix
index dfc6b3297..dfc6b3297 100644
--- a/krebs/6tests/data/test-source.nix
+++ b/krebs/0tests/data/test-source.nix
diff --git a/krebs/6tests/default.nix b/krebs/0tests/default.nix
index c0ca00296..c0ca00296 100644
--- a/krebs/6tests/default.nix
+++ b/krebs/0tests/default.nix
diff --git a/krebs/6tests/deploy.nix b/krebs/0tests/deploy.nix
index 156e9239f..d96963500 100644
--- a/krebs/6tests/deploy.nix
+++ b/krebs/0tests/deploy.nix
@@ -3,7 +3,7 @@ import <nixpkgs/nixos/tests/make-test.nix> ({ ... }:
 
 let
   pkgs = import <nixpkgs> { overlays = [(import ../5pkgs)]; };
-  test-config = <stockholm/krebs/6tests/data/test-config.nix>;
+  test-config = <stockholm/krebs/0tests/data/test-config.nix>;
   privKey = ''
     -----BEGIN OPENSSH PRIVATE KEY-----
     b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
diff --git a/krebs/1systems/hope/config.nix b/krebs/1systems/hope/config.nix
deleted file mode 100644
index c19b210c5..000000000
--- a/krebs/1systems/hope/config.nix
+++ /dev/null
@@ -1,41 +0,0 @@
-with import <stockholm/lib>;
-{ config, pkgs, ... }: let
-
-  ip = config.krebs.build.host.nets.internet.ip4.addr;
-  bestGuessGateway = addr: elemAt (match "(.*)(\.[^.])" addr) 0 + ".1";
-
-in {
-  imports = [
-    <stockholm/krebs>
-    <stockholm/krebs/2configs>
-    <stockholm/krebs/2configs/os-templates/CAC-CentOS-7-64bit.nix>
-
-    <stockholm/krebs/2configs/secret-passwords.nix>
-    {
-      users.extraUsers = {
-        satan = {
-          name = "satan";
-          uid = 1338;
-          home = "/home/satan";
-          group = "users";
-          createHome = true;
-          useDefaultShell = true;
-          initialPassword = "test";
-        };
-      };
-    }
-  ];
-
-  krebs.build.host = config.krebs.hosts.hope;
-
-  networking = let
-    address = config.krebs.build.host.nets.internet.ip4.addr;
-  in {
-    defaultGateway = bestGuessGateway address;
-    interfaces.enp2s1.ip4 = singleton {
-      inherit address;
-      prefixLength = 24;
-    };
-    nameservers = ["8.8.8.8"];
-  };
-}
diff --git a/krebs/1systems/hope/source.nix b/krebs/1systems/hope/source.nix
deleted file mode 100644
index 7121d1d9d..000000000
--- a/krebs/1systems/hope/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/krebs/source.nix> {
-  name = "hope";
-}
diff --git a/krebs/1systems/hotdog/config.nix b/krebs/1systems/hotdog/config.nix
index 98fb88702..18b751a7e 100644
--- a/krebs/1systems/hotdog/config.nix
+++ b/krebs/1systems/hotdog/config.nix
@@ -9,16 +9,17 @@
     <stockholm/krebs>
     <stockholm/krebs/2configs>
 
-    <stockholm/krebs/2configs/buildbot-all.nix>
+    <stockholm/krebs/2configs/buildbot-stockholm.nix>
     <stockholm/krebs/2configs/gitlab-runner-shackspace.nix>
     <stockholm/krebs/2configs/binary-cache/nixos.nix>
     <stockholm/krebs/2configs/ircd.nix>
     <stockholm/krebs/2configs/reaktor-retiolum.nix>
+    <stockholm/krebs/2configs/reaktor-krebs.nix>
   ];
 
   krebs.build.host = config.krebs.hosts.hotdog;
 
   boot.isContainer = true;
   networking.useDHCP = false;
-  krebs.ci.stockholmSrc = "http://cgit.prism.r/stockholm";
+  environment.variables.NIX_REMOTE = "daemon";
 }
diff --git a/krebs/1systems/onebutton/config.nix b/krebs/1systems/onebutton/config.nix
new file mode 100644
index 000000000..dca00a206
--- /dev/null
+++ b/krebs/1systems/onebutton/config.nix
@@ -0,0 +1,48 @@
+{ config, pkgs, lib, ... }:
+{
+  # :l <nixpkgs>
+  # builtins.readDir (pkgs.fetchFromGitHub { owner = "nixos"; repo = "nixpkgs-channels"; rev = "6c064e6b"; sha256 = "1rqzh475xn43phagrr30lb0fd292c1s8as53irihsnd5wcksnbyd"; })
+  imports = [
+    <stockholm/krebs>
+    <stockholm/krebs/2configs>
+    { # flag to rebuild everything yourself:
+      # environment.noXlibs = true;
+
+      # minimal disk usage
+      nix.gc.automatic = true;
+      nix.gc.dates = "03:10";
+      documentation.man.enable = false;
+      documentation.info.enable = false;
+      services.nixosManual.enable = false;
+      services.journald.extraConfig = "SystemMaxUse=50M";
+    }
+  ];
+  krebs.build.host = config.krebs.hosts.onebutton;
+  # NixOS wants to enable GRUB by default
+  boot.loader.grub.enable = false;
+
+  # Enables the generation of /boot/extlinux/extlinux.conf
+  boot.loader.generic-extlinux-compatible.enable = true;
+
+  boot.kernelPackages = pkgs.linuxPackages_rpi;
+
+  nix.binaryCaches = [ "http://nixos-arm.dezgeg.me/channel" ];
+  nix.binaryCachePublicKeys = [ "nixos-arm.dezgeg.me-1:xBaUKS3n17BZPKeyxL4JfbTqECsT+ysbDJz29kLFRW0=%" ];
+
+  fileSystems = {
+    "/boot" = {
+      device = "/dev/disk/by-label/NIXOS_BOOT";
+      fsType = "vfat";
+    };
+    "/" = {
+      device = "/dev/disk/by-label/NIXOS_SD";
+      fsType = "ext4";
+    };
+  };
+
+  swapDevices = [ { device = "/swapfile"; size = 1024; } ];
+  services.openssh.enable = true;
+
+  networking.wireless.enable = true;
+  hardware.enableRedistributableFirmware = true;
+}
diff --git a/krebs/1systems/onebutton/source.nix b/krebs/1systems/onebutton/source.nix
new file mode 100644
index 000000000..8f25881c9
--- /dev/null
+++ b/krebs/1systems/onebutton/source.nix
@@ -0,0 +1,16 @@
+with import <stockholm/lib>;
+let
+  pkgs = import <nixpkgs> {};
+  nixpkgs = pkgs.fetchFromGitHub {
+    owner = "nixos";
+    repo = "nixpkgs-channels";
+    rev = "6c064e6b"; # only binary cache for unstable arm6
+    sha256 = "1rqzh475xn43phagrr30lb0fd292c1s8as53irihsnd5wcksnbyd";
+  };
+in import <stockholm/krebs/source.nix> {
+  name = "onebutton";
+  override.nixpkgs = mkForce {
+    file = toString nixpkgs;
+  };
+
+}
diff --git a/krebs/2configs/buildbot-all.nix b/krebs/2configs/buildbot-all.nix
deleted file mode 100644
index d85cde175..000000000
--- a/krebs/2configs/buildbot-all.nix
+++ /dev/null
@@ -1,9 +0,0 @@
-with import <stockholm/lib>;
-{ lib, config, pkgs, ... }:
-{
-  networking.firewall.allowedTCPPorts = [ 80 8010 9989 ];
-  krebs.ci.enable = true;
-  krebs.ci.treeStableTimer = 1;
-  krebs.ci.hosts = filter (getAttr "ci") (attrValues config.krebs.hosts);
-}
-
diff --git a/krebs/2configs/buildbot-krebs.nix b/krebs/2configs/buildbot-krebs.nix
deleted file mode 100644
index a09b3b98b..000000000
--- a/krebs/2configs/buildbot-krebs.nix
+++ /dev/null
@@ -1,12 +0,0 @@
-with import <stockholm/lib>;
-{ lib, config, pkgs, ... }:
-{
-  imports = [
-    <stockholm/krebs/2configs/repo-sync.nix>
-  ];
-
-  networking.firewall.allowedTCPPorts = [ 80 8010 9989 ];
-  krebs.ci.enable = true;
-  krebs.ci.treeStableTimer = 120;
-  krebs.ci.hosts = [ config.krebs.build.host ];
-}
diff --git a/krebs/2configs/buildbot-stockholm.nix b/krebs/2configs/buildbot-stockholm.nix
new file mode 100644
index 000000000..04b1c999f
--- /dev/null
+++ b/krebs/2configs/buildbot-stockholm.nix
@@ -0,0 +1,178 @@
+{ config, pkgs, ... }: with import <stockholm/lib>;
+
+let
+
+  hostname = config.networking.hostName;
+
+in
+{
+  networking.firewall.allowedTCPPorts = [ 80 ];
+  services.nginx = {
+    enable = true;
+    virtualHosts.build = {
+      serverAliases = [ "build.${hostname}.r" ];
+      locations."/".extraConfig = ''
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port};
+      '';
+    };
+  };
+
+  krebs.buildbot.master = {
+    slaves = {
+      testslave = "lasspass";
+    };
+    change_source.stockholm = ''
+      stockholm_repo = 'http://cgit.prism.r/stockholm'
+      cs.append(
+          changes.GitPoller(
+              stockholm_repo,
+              workdir='stockholm-poller', branches=True,
+              project='stockholm',
+              pollinterval=10
+          )
+      )
+    '';
+    scheduler = {
+      auto-scheduler = ''
+        sched.append(
+              schedulers.SingleBranchScheduler(
+                  change_filter=util.ChangeFilter(branch_re=".*"),
+                  treeStableTimer=60,
+                  name="build-all-branches",
+                  builderNames=[
+                      "hosts",
+                  ]
+              )
+        )
+      '';
+      force-scheduler = ''
+        sched.append(
+            schedulers.ForceScheduler(
+                  name="hosts",
+                  builderNames=[
+                      "hosts",
+                  ]
+            )
+        )
+      '';
+    };
+    builder_pre = ''
+      # prepare grab_repo step for stockholm
+      grab_repo = steps.Git(
+          repourl=stockholm_repo,
+          mode='full',
+      )
+    '';
+    builder = {
+      hosts = ''
+        from buildbot import interfaces
+        from buildbot.steps.shell import ShellCommand
+
+        class StepToStartMoreSteps(ShellCommand):
+            def __init__(self, **kwargs):
+                ShellCommand.__init__(self, **kwargs)
+
+            def addBuildSteps(self, steps_factories):
+                for sf in steps_factories:
+                    step = interfaces.IBuildStepFactory(sf).buildStep()
+                    step.setBuild(self.build)
+                    step.setBuildSlave(self.build.slavebuilder.slave)
+                    step_status = self.build.build_status.addStepWithName(step.name)
+                    step.setStepStatus(step_status)
+                    self.build.steps.append(step)
+
+            def start(self):
+                props = self.build.getProperties()
+                hosts = json.loads(props.getProperty('hosts_json'))
+                for host in hosts:
+                    user = hosts[host]['owner']
+
+                    self.addBuildSteps([steps.ShellCommand(
+                        name=str(host),
+                        env={
+                          "NIX_PATH": "secrets=/var/src/stockholm/null:stockholm=./:/var/src",
+                          "NIX_REMOTE": "daemon",
+                          "dummy_secrets": "true",
+                        },
+                        command=[
+                          "nix-shell", "-I", "stockholm=.", "--run", " ".join(["test",
+                            "--user={}".format(user),
+                            "--system={}".format(host),
+                            "--force-populate",
+                            "--target=$LOGNAME@${config.krebs.build.host.name}$HOME/{}".format(user),
+                          ])
+                        ],
+                        timeout=90001,
+                        workdir='build', # TODO figure out why we need this?
+                    )])
+
+                ShellCommand.start(self)
+
+
+        f = util.BuildFactory()
+        f.addStep(grab_repo)
+
+        f.addStep(steps.SetPropertyFromCommand(
+            env={
+              "NIX_PATH": "secrets=/var/src/stockholm/null:stockholm=./:/var/src",
+              "NIX_REMOTE": "daemon",
+            },
+            name="get_hosts",
+            command=["nix-instantiate", "--json", "--strict", "--eval", "-E", """
+                with import <nixpkgs> {};
+                let
+                  eval-config = cfg:
+                    import <nixpkgs/nixos/lib/eval-config.nix> {
+                      modules = [
+                        (import cfg)
+                      ];
+                    }
+                  ;
+
+                  system = eval-config ./krebs/1systems/hotdog/config.nix; # TODO put a better config here
+
+                  ci-systems = lib.filterAttrs (_: v: v.ci) system.config.krebs.hosts;
+
+                  filtered-attrs = lib.mapAttrs ( n: v: {
+                    owner = v.owner.name;
+                  }) ci-systems;
+
+                in filtered-attrs
+            """],
+            property="hosts_json"
+        ))
+        f.addStep(StepToStartMoreSteps(command=["echo"])) # TODO remove dummy command from here
+
+        bu.append(
+            util.BuilderConfig(
+                name="hosts",
+                slavenames=slavenames,
+                factory=f
+            )
+        )
+      '';
+    };
+    enable = true;
+    web.enable = true;
+    irc = {
+      enable = true;
+      nick = "build|${hostname}";
+      server = "irc.r";
+      channels = [ "noise" "xxx" ];
+      allowForce = true;
+    };
+    extraConfig = ''
+      c['buildbotURL'] = "http://build.${hostname}.r/"
+    '';
+  };
+
+  krebs.buildbot.slave = {
+    enable = true;
+    masterhost = "localhost";
+    username = "testslave";
+    password = "lasspass";
+    packages = with pkgs; [ gnumake jq nix populate ];
+  };
+}
diff --git a/krebs/2configs/default.nix b/krebs/2configs/default.nix
index 90aaa254a..7b970923d 100644
--- a/krebs/2configs/default.nix
+++ b/krebs/2configs/default.nix
@@ -50,6 +50,7 @@ with import <stockholm/lib>;
   users.extraUsers.root.openssh.authorizedKeys.keys = [
     # TODO
     config.krebs.users.lass.pubkey
+    config.krebs.users.lass-mors.pubkey
     config.krebs.users.makefu.pubkey
     # TODO HARDER:
     config.krebs.users.makefu-omo.pubkey
diff --git a/krebs/2configs/news-spam.nix b/krebs/2configs/news-spam.nix
index 63848c234..a3f39b40e 100644
--- a/krebs/2configs/news-spam.nix
+++ b/krebs/2configs/news-spam.nix
@@ -2,6 +2,7 @@
 
 {
   krebs.newsbot-js.news-spam = {
+    urlShortenerHost = "go.lassul.us";
     feeds = pkgs.writeText "feeds" ''
       [SPAM]aje|http://www.aljazeera.com/Services/Rss/?PostingId=2007731105943979989|#snews
       [SPAM]allafrica|http://allafrica.com/tools/headlines/rdf/latest/headlines.rdf|#snews
@@ -93,7 +94,7 @@
       [SPAM]npr_world|http://www.npr.org/rss/rss.php?id=1004|#snews
       [SPAM]nsa|https://www.nsa.gov/rss.xml|#snews #bullerei
       [SPAM]nytimes|http://rss.nytimes.com/services/xml/rss/nyt/World.xml|#snews
-      [SPAM]painload|https://github.com/krebscode/painload/commits/master.atom|#snews
+      [SPAM]painload|https://github.com/krebs/painload/commits/master.atom|#snews
       [SPAM]phys|http://phys.org/rss-feed/|#snews
       [SPAM]piraten|https://www.piratenpartei.de/feed/|#snews
       [SPAM]polizei_berlin|http://www.berlin.de/polizei/presse-fahndung/_rss_presse.xml|#snews
@@ -120,7 +121,7 @@
       [SPAM]sciencemag|http://news.sciencemag.org/rss/current.xml|#snews
       [SPAM]scmp|http://www.scmp.com/rss/91/feed|#snews
       [SPAM]sec-db|http://feeds.security-database.com/SecurityDatabaseToolsWatch|#snews
-      [SPAM]shackspace|http://blog.shackspace.de/?feed=rss2|#snews
+      [SPAM]shackspace|http://shackspace.de/atom.xml|#snews
       [SPAM]shz_news|http://www.shz.de/nachrichten/newsticker/rss|#snews
       [SPAM]sky_busi|http://feeds.skynews.com/feeds/rss/business.xml|#snews
       [SPAM]sky_pol|http://feeds.skynews.com/feeds/rss/politics.xml|#snews
diff --git a/krebs/2configs/news.nix b/krebs/2configs/news.nix
index 2628c7986..6c59f4d84 100644
--- a/krebs/2configs/news.nix
+++ b/krebs/2configs/news.nix
@@ -8,15 +8,15 @@
       ethereum|http://blog.ethereum.org/feed|#news
       LtU|http://lambda-the-ultimate.org/rss.xml|#news
       mongrel2_master|https://github.com/zedshaw/mongrel2/commits/master.atom|#news
-      painload|https://github.com/krebscode/painload/commits/master.atom|#news
+      painload|https://github.com/krebs/painload/commits/master.atom|#news
       reddit_haskell|http://www.reddit.com/r/haskell/.rss|#news
       reddit_nix|http://www.reddit.com/r/nixos/.rss|#news
-      shackspace|http://blog.shackspace.de/?feed=rss2|#news
+      shackspace|http://shackspace.de/atom.xml|#news
       tinc|http://tinc-vpn.org/news/index.rss|#news
       vimperator|https://sites.google.com/a/vimperator.org/www/blog/posts.xml|#news
       weechat|http://dev.weechat.org/feed/atom|#news
       xkcd|https://xkcd.com/rss.xml|#news
-      painload|https://github.com/krebscode/painload/commits/master.atom|#news
+      painload|https://github.com/krebs/painload/commits/master.atom|#news
     '';
   };
 }
diff --git a/krebs/2configs/reaktor-krebs.nix b/krebs/2configs/reaktor-krebs.nix
index 6b17b457d..fa51b84f0 100644
--- a/krebs/2configs/reaktor-krebs.nix
+++ b/krebs/2configs/reaktor-krebs.nix
@@ -13,13 +13,8 @@ with import <stockholm/lib>;
     };
     plugins = with pkgs.ReaktorPlugins; [
       sed-plugin
-      wiki-todo-add
-      wiki-todo-done
-      wiki-todo-show
-    ];
+    ] ++
+      (attrValues (todo "agenda"))
+    ;
   };
-  services.nginx.virtualHosts."lassul.us".locations."/wiki-todo".extraConfig = ''
-    default_type "text/plain";
-    alias /var/lib/Reaktor/state/wiki-todo;
-  '';
 }
diff --git a/krebs/2configs/reaktor-retiolum.nix b/krebs/2configs/reaktor-retiolum.nix
index 144b7d484..b32d39b7e 100644
--- a/krebs/2configs/reaktor-retiolum.nix
+++ b/krebs/2configs/reaktor-retiolum.nix
@@ -10,6 +10,8 @@ with import <stockholm/lib>;
     };
     plugins = with pkgs.ReaktorPlugins; [
       sed-plugin
-    ];
+    ] ++
+      (attrValues (todo "agenda"))
+    ;
   };
 }
diff --git a/krebs/2configs/shack/worlddomination.nix b/krebs/2configs/shack/worlddomination.nix
index 828b6cd70..44176a341 100644
--- a/