Merge remote-tracking branch 'prism/master'

10 files changed, 230 insertions, 18 deletions

diff --git a/shared/1systems/test-all-krebs-modules.nix b/shared/1systems/test-all-krebs-modules.nix
new file mode 100644
index 000000000..b98004dfe
--- /dev/null
+++ b/shared/1systems/test-all-krebs-modules.nix
@@ -0,0 +1,45 @@
+{ config, pkgs, lib, ... }:
+let
+  en = { enable = true;};
+in {
+  krebs = {
+    enable = true;
+    build.user = config.krebs.users.shared;
+    build.host = config.krebs.hosts.test-all-krebs-modules;
+    Reaktor.enable = true;
+    apt-cacher-ng.enable = true;
+    backup.enable = true;
+    bepasty.enable = true;
+    buildbot.master.enable = true;
+    buildbot.slave = {
+      enable = true;
+      username = "lol";
+      password = "wut";
+    };
+    exim-retiolum.enable = true;
+    exim-smarthost = {
+      enable = true;
+      system-aliases = [ { from = "dick"; to = "butt"; } ];
+    };
+    go.enable = true;
+    iptables = {
+      enable = true;
+      tables = {};
+    };
+    nginx.enable = true;
+    realwallpaper.enable = true;
+    retiolum.enable = true;
+    retiolum-bootstrap.enable = true;
+    tinc_graphs.enable = true;
+    urlwatch.enable = true;
+    fetchWallpaper = {
+      enable = true;
+      url ="localhost";
+    };
+  };
+  # just get the system running
+  boot.loader.grub.devices = ["/dev/sda"];
+  fileSystems."/" = {
+    device = "/dev/lol";
+  };
+}
diff --git a/shared/1systems/test-centos7.nix b/shared/1systems/test-centos7.nix
index 077a5d61b..48cecc877 100644
--- a/shared/1systems/test-centos7.nix
+++ b/shared/1systems/test-centos7.nix
@@ -7,7 +7,8 @@ in {
   imports = [
     ../2configs/base.nix
     ../2configs/os-templates/CAC-CentOS-7-64bit.nix
-    ../2configs/os-templates/temp-networking.nix
+    ../2configs/temp/networking.nix
+    ../2configs/temp/dirs.nix
   ];
 
   sound.enable = false;
diff --git a/shared/1systems/test-failing.nix b/shared/1systems/test-failing.nix
new file mode 100644
index 000000000..81a9e48d6
--- /dev/null
+++ b/shared/1systems/test-failing.nix
@@ -0,0 +1,6 @@
+{ config, pkgs, ... }:
+
+{
+  programs.ssh.startAgent = true;
+  programs.ssh.startAgent = false;
+}
diff --git a/shared/1systems/test-minimal-deploy.nix b/shared/1systems/test-minimal-deploy.nix
new file mode 100644
index 000000000..ddd96f6b5
--- /dev/null
+++ b/shared/1systems/test-minimal-deploy.nix
@@ -0,0 +1,13 @@
+{ config, pkgs, lib, ... }:
+{
+  krebs = {
+    enable = true;
+    build.user = config.krebs.users.shared;
+    build.host = config.krebs.hosts.test-all-krebs-modules;
+  };
+  # just get the system running
+  boot.loader.grub.devices = ["/dev/sda"];
+  fileSystems."/" = {
+    device = "/dev/lol";
+  };
+}
diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix
index 2c51ac8fe..f05356f0f 100644
--- a/shared/1systems/wolf.nix
+++ b/shared/1systems/wolf.nix
@@ -11,7 +11,7 @@ in
     ../2configs/collectd-base.nix
     ../2configs/shack-nix-cacher.nix
     ../2configs/shack-drivedroid.nix
-    ../2configs/cac-ci.nix
+    ../2configs/buildbot-standalone.nix
     ../2configs/graphite.nix
   ];
   # use your own binary cache, fallback use cache.nixos.org (which is used by
@@ -33,8 +33,6 @@ in
   # uninteresting stuff
   #####################
   krebs.build.host = config.krebs.hosts.wolf;
-  # TODO rename shared user to "krebs"
-  krebs.build.user = config.krebs.users.shared;
   krebs.build.target = "wolf";
 
   boot.kernel.sysctl = {
diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix
index df41eae1a..4d509d7a6 100644
--- a/shared/2configs/base.nix
+++ b/shared/2configs/base.nix
@@ -13,18 +13,22 @@ with lib;
     ];
   };
 
+  # TODO rename shared user to "krebs"
+  krebs.build.user = mkDefault config.krebs.users.shared;
   krebs.build.source = {
     git.nixpkgs = {
       url = https://github.com/NixOS/nixpkgs;
       rev = "6d31e9b81dcd4ab927bb3dc91b612dd5abfa2f80";
+      target-path = "/var/src/nixpkgs";
     };
     dir.secrets = {
       host = config.krebs.current.host;
-      path = "${getEnv "HOME"}/secrets/krebs/wolf";
+      path = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
     };
     dir.stockholm = {
       host = config.krebs.current.host;
-      path = "${getEnv "HOME"}/stockholm";
+      path = mkDefault "${getEnv "HOME"}/stockholm";
+      target-path = "/var/src/stockholm";
     };
   };
 
@@ -65,7 +69,7 @@ with lib;
     config.krebs.users.lass.pubkey
     config.krebs.users.makefu.pubkey
     # TODO HARDER:
-    (readFile ../../krebs/Zpubkeys/makefu_omo.ssh.pub)
+    config.krebs.users.makefu-omo.pubkey
     config.krebs.users.tv.pubkey
   ];
 
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
new file mode 100644
index 000000000..6ffd7fe8a
--- /dev/null
+++ b/shared/2configs/buildbot-standalone.nix
@@ -0,0 +1,154 @@
+{ lib, config, pkgs, ... }:
+let
+    pkgs-unst = import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz) {};
+in {
+  nixpkgs.config.packageOverrides = pkgs: {
+    buildbot = pkgs-unst.buildbot;
+    buildbot-slave = pkgs-unst.buildbot-slave;
+  };
+  networking.firewall.allowedTCPPorts = [ 8010 9989 ];
+  krebs.buildbot.master = {
+    secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ];
+    slaves = {
+      testslave =  "krebspass";
+    };
+    change_source.stockholm = ''
+  stockholm_repo = 'http://cgit.gum/stockholm'
+  cs.append(changes.GitPoller(
+          stockholm_repo,
+          workdir='stockholm-poller', branch='master',
+          project='stockholm',
+          pollinterval=120))
+    '';
+    scheduler = {
+        force-scheduler = ''
+  sched.append(schedulers.ForceScheduler(
+                              name="force",
+                              builderNames=["full-tests"]))
+        '';
+        fast-tests-scheduler = ''
+  # test the master real quick
+  sched.append(schedulers.SingleBranchScheduler(
+                              change_filter=util.ChangeFilter(branch="master"),
+                              name="fast-master-test",
+                              builderNames=["fast-tests"]))
+        '';
+        test-cac-infest-master = ''
+  # files everyone depends on or are part of the share branch
+  def shared_files(change):
+    r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)")
+    for file in change.files:
+      if r.match(file):
+        return True
+    return False
+
+  sched.append(schedulers.SingleBranchScheduler(
+                              change_filter=util.ChangeFilter(branch="master"),
+                              fileIsImportant=shared_files,
+                              treeStableTimer=60*60, # master was stable for the last hour
+                              name="full-master-test",
+                              builderNames=["full-tests"]))
+        '';
+    };
+    builder_pre = ''
+  # prepare grab_repo step for stockholm
+  stockholm_repo = "http://cgit.gum.retiolum/stockholm"
+  grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+
+  env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"}
+
+  # prepare nix-shell
+  # the dependencies which are used by the test script
+  deps = [ "gnumake", "jq","nix","rsync",
+            "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
+  # TODO: --pure , prepare ENV in nix-shell command:
+  #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
+  nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
+
+  # prepare addShell function
+  def addShell(factory,**kwargs):
+    factory.addStep(steps.ShellCommand(**kwargs))
+    '';
+    builder = {
+      fast-tests = ''
+  f = util.BuildFactory()
+  f.addStep(grab_repo)
+  addShell(f,name="deploy-eval-centos7",env=env,
+            command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
+
+  addShell(f,name="deploy-eval-wolf",env=env,
+            command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
+
+  addShell(f,name="deploy-eval-cross-check",env=env,
+            command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
+
+  addShell(f,name="instantiate-test-all-modules",env=env,
+            command=nixshell + \
+                      ["touch retiolum.rsa_key.priv; \
+                        nix-instantiate --eval -A \
+                            users.shared.test-all-krebs-modules.system \
+                            -I stockholm=. \
+                            -I secrets=. '<stockholm>' \
+                            --argstr current-date lol \
+                            --argstr current-user-name shared \
+                            --argstr current-host-name lol \
+                            --strict --json"])
+
+  addShell(f,name="instantiate-test-minimal-deploy",env=env,
+            command=nixshell + \
+                      ["nix-instantiate --eval -A \
+                            users.shared.test-minimal-deploy.system \
+                            -I stockholm=. \
+                            -I secrets=. '<stockholm>' \
+                            --argstr current-date lol \
+                            --argstr current-user-name shared \
+                            --argstr current-host-name lol \
+                            --strict --json"])
+
+  bu.append(util.BuilderConfig(name="fast-tests",
+        slavenames=slavenames,
+        factory=f))
+      '';
+      slow-tests = ''
+  s = util.BuildFactory()
+  s.addStep(grab_repo)
+
+  # slave needs 2 files:
+  # * cac.json
+  # * retiolum
+  s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json"))
+  s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv"))
+
+  addShell(s, name="infest-cac-centos7",env=env,
+              sigtermTime=60,           # SIGTERM 1 minute before SIGKILL
+              timeout=7200,             # 2h
+              command=nixshell + ["infest-cac-centos7"])
+
+  bu.append(util.BuilderConfig(name="full-tests",
+        slavenames=slavenames,
+        factory=s))
+      '';
+    };
+    enable = true;
+    web = {
+      enable = true;
+    };
+    irc = {
+      enable = true;
+      nick = "shared-buildbot";
+      server = "cd.retiolum";
+      channels = [ "retiolum" ];
+      allowForce = true;
+    };
+  };
+
+  krebs.buildbot.slave = {
+    enable = true;
+    masterhost = "localhost";
+    username = "testslave";
+    password = "krebspass";
+    packages = with pkgs;[ git nix ];
+    # all nix commands will need a working nixpkgs installation
+    extraEnviron = { NIX_PATH="nixpkgs=${toString <nixpkgs>}"; };
+  };
+}
diff --git a/shared/2configs/cac-ci.nix b/shared/2configs/cac-ci.nix
deleted file mode 100644
index 06cce2746..000000000
--- a/shared/2configs/cac-ci.nix
+++ /dev/null
@@ -1,11 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-{
-  environment.systemPackages = with pkgs;[
-    get
-    cac
-    cacpanel
-    jq
-  ];
-}
diff --git a/shared/2configs/temp/dirs.nix b/shared/2configs/temp/dirs.nix
new file mode 100644
index 000000000..958608a54
--- /dev/null
+++ b/shared/2configs/temp/dirs.nix
@@ -0,0 +1 @@
+_: { }
diff --git a/shared/2configs/temp/networking.nix b/shared/2configs/temp/networking.nix
new file mode 100644
index 000000000..958608a54
--- /dev/null
+++ b/shared/2configs/temp/networking.nix
@@ -0,0 +1 @@
+_: { }