Merge remote-tracking branch 'gum/master'

17 files changed, 209 insertions, 122 deletions

diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix
index 7b28ffca8..11cf21b5f 100644
--- a/krebs/3modules/git.nix
+++ b/krebs/3modules/git.nix
@@ -92,7 +92,7 @@ let
         }
       '';
       description = ''
-        Rules.
+        access and permission rules for git repositories.
       '';
     };
   };
diff --git a/krebs/3modules/shared/default.nix b/krebs/3modules/shared/default.nix
index 518e46587..91d92857b 100644
--- a/krebs/3modules/shared/default.nix
+++ b/krebs/3modules/shared/default.nix
@@ -50,6 +50,7 @@ in {
           addrs6 = ["42:0:0:0:0:0:77:1"];
           aliases = [
             "wolf.retiolum"
+            "cgit.wolf.retiolum"
           ];
           tinc.pubkey = ''
             -----BEGIN RSA PUBLIC KEY-----
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index 7b9cbb46f..b3beb392f 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -1,6 +1,4 @@
-#! /bin/sh
-
-# nix-shell -p gnumake jq openssh cac-api cacpanel
+# nix-shell -p gnumake jq openssh cac-api cac-panel
 set -eufx
 
 # 2 secrets are required:
@@ -40,11 +38,11 @@ defer "rm -r $krebs_secrets"
 
 cat > $sec_file <<EOF
 cac_login="$(jq -r .email $krebs_cred)"
-cac_key="$(cac-cli --config $krebs_cred panel settings | jq -r .apicode)"
+cac_key="$(cac-panel --config $krebs_cred settings | jq -r .apicode)"
 EOF
 
 export cac_secrets=$sec_file
-cac-cli --config $krebs_cred panel  add-api-ip
+cac-panel --config $krebs_cred add-api-ip
 
 # test login:
 cac-api update
diff --git a/makefu/1systems/filepimp.nix b/makefu/1systems/filepimp.nix
index 2d008cee6..fb9324ee9 100644
--- a/makefu/1systems/filepimp.nix
+++ b/makefu/1systems/filepimp.nix
@@ -1,10 +1,14 @@
-# Edit this configuration file to define what should be installed on
-# your system.  Help is available in the configuration.nix(5) man page
-# and in the NixOS manual (accessible by running ‘nixos-help’).
-
-{ config, pkgs, ... }:
-
-{
+{ config, pkgs, lib, ... }:
+let
+  byid = dev: "/dev/disk/by-id/" + dev;
+  part1 = disk: disk + "-part1";
+  rootDisk = byid "ata-SanDisk_SDSSDP064G_140237402890";
+  jDisk0 = byid "ata-ST4000DM000-1F2168_Z303HVSG";
+  jDisk1 = byid "ata-ST4000DM000-1F2168_Z3040NEA";
+  jDisk2 = byid "ata-WDC_WD40EFRX-68WT0N0_WD-WCC4E0621363";
+  jDisk3 = byid "ata-TOSHIBA_MD04ACA400_156GK89OFSBA";
+  allDisks = [ rootDisk jDisk0 jDisk1 jDisk2 jDisk3 ];
+in {
   imports =
     [ # Include the results of the hardware scan.
       ../2configs/fs/single-partition-ext4.nix
@@ -12,16 +16,9 @@
       ../2configs/smart-monitor.nix
     ];
   krebs.build.host = config.krebs.hosts.filepimp;
-  services.smartd.devices = [
-    { device = "/dev/sda"; }
-    { device = "/dev/sdb"; }
-    { device = "/dev/sdc"; }
-    { device = "/dev/sdd"; }
-    { device = "/dev/sde"; }
-  ];
   # AMD N54L
   boot = {
-    loader.grub.device = "/dev/sde";
+    loader.grub.device = rootDisk;
 
     initrd.availableKernelModules = [
       "ahci"
@@ -40,4 +37,28 @@
 
   zramSwap.enable = true;
   zramSwap.numDevices = 2;
+
+  makefu.snapraid = let
+    toMedia = name: "/media/" + name;
+  in {
+    enable = true;
+    # todo combine creation when enabling the mount point
+    disks = map toMedia [ "j0" "j1" "j2" ];
+    parity = toMedia "par0";
+  };
+  # TODO: refactor, copy-paste from omo
+  services.smartd.devices = builtins.map (x: { device = x; }) allDisks;
+  powerManagement.powerUpCommands = lib.concatStrings (map (disk: ''
+      ${pkgs.hdparm}/sbin/hdparm -S 100 ${disk}
+      ${pkgs.hdparm}/sbin/hdparm -B 127 ${disk}
+      ${pkgs.hdparm}/sbin/hdparm -y ${disk}
+    '') allDisks);
+  fileSystems = let
+    xfsmount = name: dev:
+      { "/media/${name}" = { device = dev; fsType = "xfs"; }; };
+  in
+        (xfsmount "j0" (part1 jDisk0))
+    //  (xfsmount "j1" (part1 jDisk1))
+    //  (xfsmount "j2" (part1 jDisk2))
+    //  (xfsmount "par0" (part1 jDisk3));
 }
diff --git a/makefu/1systems/gum.nix b/makefu/1systems/gum.nix
index ac7524506..c4dfbf4b7 100644
--- a/makefu/1systems/gum.nix
+++ b/makefu/1systems/gum.nix
@@ -15,6 +15,7 @@ in {
       ../2configs/git/cgit-retiolum.nix
       ../2configs/mattermost-docker.nix
       ../2configs/nginx/euer.test.nix
+      ../2configs/nginx/update.connector.one.nix
 
       ../2configs/exim-retiolum.nix
       ../2configs/urlwatch.nix
diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix
index 19183fea8..34d5a394d 100644
--- a/makefu/1systems/omo.nix
+++ b/makefu/1systems/omo.nix
@@ -28,8 +28,7 @@ in {
       ../2configs/smart-monitor.nix
       ../2configs/mail-client.nix
       ../2configs/share-user-sftp.nix
-      ../2configs/nginx/omo-share.nix
-      ../3modules
+      ../2configs/omo-share.nix
     ];
   networking.firewall.trustedInterfaces = [ "enp3s0" ];
   # udp:137 udp:138 tcp:445 tcp:139 - samba, allowed in local net
@@ -40,35 +39,7 @@ in {
   networking.firewall.allowedTCPPorts = [ 80 655 8080 ];
 
   # services.openssh.allowSFTP = false;
-  krebs.build.source.git.nixpkgs.rev = "d0e3cca04edd5d1b3d61f188b4a5f61f35cdf1ce";
-
-  # samba share /media/crypt1/share
-  users.users.smbguest = {
-    name = "smbguest";
-    uid = config.ids.uids.smbguest;
-    description = "smb guest user";
-    home = "/var/empty";
-  };
-  services.samba = {
-    enable = true;
-    shares = {
-      winshare = {
-        path = "/media/crypt1/share";
-        "read only" = "no";
-        browseable = "yes";
-        "guest ok" = "yes";
-      };
-    };
-    extraConfig = ''
-      guest account = smbguest
-      map to guest = bad user
-      # disable printing
-      load printers = no
-      printing = bsd
-      printcap name = /dev/null
-      disable spoolss = yes
-    '';
-  };
+  krebs.build.source.nixpkgs.rev = "d0e3cca04edd5d1b3d61f188b4a5f61f35cdf1ce";
 
   # copy config from <secrets/sabnzbd.ini> to /var/lib/sabnzbd/
   services.sabnzbd.enable = true;
diff --git a/makefu/1systems/vbob.nix b/makefu/1systems/vbob.nix
index e8a2959d0..90b490802 100644
--- a/makefu/1systems/vbob.nix
+++ b/makefu/1systems/vbob.nix
@@ -18,27 +18,8 @@
     tinc = pkgs.tinc_pre;
   };
 
-  makefu.buildbot.master = {
-    enable = false;
-    irc = {
-      enable = true;
-      server = "cd.retiolum";
-      channel = "retiolum";
-      allowForce = true;
-    };
-  };
-  # services.logstash.enable = true;
-  makefu.buildbot.slave = {
-    enable = false;
-    masterhost = "localhost";
-    username = "testslave";
-    password = "krebspass";
-    packages = with pkgs;[ git nix ];
-    extraEnviron = { NIX_PATH="nixpkgs=${toString <nixpkgs>}"; };
-  };
-
-  krebs.build.source.git.nixpkgs = {
-    #url = https://github.com/nixos/nixpkgs;
+  krebs.build.source.nixpkgs = {
+    # url = https://github.com/nixos/nixpkgs;
     # HTTP Everywhere + libredir
     rev = "8239ac6";
   };
diff --git a/makefu/2configs/backup.nix b/makefu/2configs/backup.nix
new file mode 100644
index 000000000..ed6d1f4a7
--- /dev/null
+++ b/makefu/2configs/backup.nix
@@ -0,0 +1,30 @@
+{ config, lib, ... }:
+with lib;
+let
+  startAt = "0,6,12,18:00";
+  defaultBackupServer = config.krebs.hosts.omo;
+  defaultBackupDir = "/home/backup";
+  defaultPull = host: src: {
+    method = "pull";
+    src = {
+      inherit host;
+      path = src;
+    };
+    dst = {
+      host = defaultBackupServer;
+      path = defaultBackupDir + src;
+    };
+    startAt = "0,6,12,18:00";
+    snapshots = {
+      hourly   = { format = "%Y-%m-%dT%H";    retain =  4; };
+      daily    = { format = "%Y-%m-%d";       retain =  7; };
+      weekly   = { format = "%YW%W";          retain =  4; };
+      monthly  = { format = "%Y-%m";          retain = 12; };
+      yearly   = { format = "%Y";                          };
+    };
+  };
+in {
+  krebs.backup.plans = addNames {
+    wry-to-omo_var-www = defaultPull wry "/var/www";
+  };
+}
diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix
index ec1100582..2b4e31119 100644
--- a/makefu/2configs/default.nix
+++ b/makefu/2configs/default.nix
@@ -20,24 +20,18 @@ with lib;
     build =  {
       target = mkDefault "root@${config.krebs.build.host.name}";
       user = config.krebs.users.makefu;
-      source = {
-        git.nixpkgs = {
-          #url = https://github.com/NixOS/nixpkgs;
-          url = mkDefault https://github.com/nixos/nixpkgs;
-          rev = mkDefault "93d8671e2c6d1d25f126ed30e5e6f16764330119"; # unstable @ 2015-01-03, tested on filepimp
-          target-path = "/var/src/nixpkgs";
+      source =  mapAttrs (_: mkDefault) {
+        upstream-nixpkgs = {
+          url = https://github.com/nixos/nixpkgs;
+          rev = "93d8671e2c6d1d25f126ed30e5e6f16764330119"; # unstable @ 2015-01-03, tested on filepimp
         };
+        secrets = "/home/makefu/secrets/${config.krebs.build.host.name}/";
+        stockholm = "/home/makefu/stockholm";
 
-        dir.secrets = {
-          host = config.krebs.hosts.pornocauster;
-          path = "/home/makefu/secrets/${config.krebs.build.host.name}/";
-        };
-
-        dir.stockholm = {
-          host = config.krebs.hosts.pornocauster;
-          path = "/home/makefu/stockholm" ;
-          target-path = "/var/src/stockholm";
-        };
+        # Defaults for all stockholm users?
+        nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix";
+        nixpkgs = symlink:stockholm/nixpkgs;
+        stockholm-user = "symlink:stockholm/${config.krebs.build.user.name}";
       };
     };
   };
@@ -86,11 +80,7 @@ with lib;
   ];
 
   environment.variables = {
-    NIX_PATH = with config.krebs.build.source; with dir; with git;
-      mkForce (concatStringsSep ":" [
-        "nixpkgs=${nixpkgs.target-path}"
-        "${nixpkgs.target-path}"
-      ]);
+    NIX_PATH = mkForce "/var/src";
     EDITOR = mkForce "vim";
   };
 
diff --git a/makefu/2configs/nginx/update.connector.one.nix b/makefu/2configs/nginx/update.connector.one.nix
new file mode 100644
index 000000000..eb39a1668
--- /dev/null
+++ b/makefu/2configs/nginx/update.connector.one.nix
@@ -0,0 +1,26 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  hostname = config.krebs.build.host.name;
+  external-ip = head config.krebs.build.host.nets.internet.addrs4;
+in {
+  krebs.nginx = {
+    enable = mkDefault true;
+    servers = {
+      omo-share = {
+        listen = [ "${external-ip}:80" ];
+        server-names = [
+          "update.connector.one"
+          "firmware.connector.one"
+        ];
+        locations = singleton (nameValuePair "/" ''
+          autoindex on;
+          root /var/www/update.connector.one;
+          sendfile on;
+          gzip on;
+        '');
+      };
+    };
+  };
+}
diff --git a/makefu/2configs/nginx/omo-share.nix b/makefu/2configs/omo-share.nix
index ce85e0442..1e0975e1d 100644
--- a/makefu/2configs/nginx/omo-share.nix
+++ b/makefu/2configs/omo-share.nix
@@ -31,4 +31,38 @@ in {
       };
     };
   };
+
+  # samba share /media/crypt1/share
+  users.users.smbguest = {
+    name = "smbguest";
+    uid = config.ids.uids.smbguest;
+    description = "smb guest user";
+    home = "/var/empty";
+  };
+  services.samba = {
+    enable = true;
+    shares = {
+      winshare = {
+        path = "/media/crypt1/share";
+        "read only" = "no";
+        browseable = "yes";
+        "guest ok" = "yes";
+      };
+      usenet = {
+        path = "/media/crypt0/usenet/dst";
+        "read only" = "yes";
+        browseable = "yes";
+        "guest ok" = "yes";
+      };
+    };
+    extraConfig = ''
+      guest account = smbguest
+      map to guest = bad user
+      # disable printing
+      load printers = no
+      printing = bsd
+      printcap name = /dev/null
+      disable spoolss = yes
+    '';
+  };
 }
diff --git a/makefu/2configs/unstable-sources.nix b/makefu/2configs/unstable-sources.nix
index 7a9a8a81c..a34377683 100644
--- a/makefu/2configs/unstable-sources.nix
+++ b/makefu/2configs/unstable-sources.nix
@@ -1,7 +1,7 @@
 _:
 
 {
-  krebs.build.source.git.nixpkgs = {
+  krebs.build.source.nixpkgs = {
       url = https://github.com/makefu/nixpkgs;
       rev = "15b5bbfbd1c8a55e7d9e05dd9058dc102fac04fe"; # cherry-picked collectd
     };
diff --git a/makefu/2configs/wwan.nix b/makefu/2configs/wwan.nix
index 1e76cd28a..0eb0c97d7 100644
--- a/makefu/2configs/wwan.nix
+++ b/makefu/2configs/wwan.nix
@@ -1,7 +1,6 @@
 _:
 
 {
-  imports = [ ../3modules ];
   makefu.umts = {
     enable = true;
     modem-device = "/dev/serial/by-id/usb-Lenovo_H5321_gw_2D5A51BA0D3C3A90-if01";
diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix
index 8cf5be71c..bcfbd6810 100644
--- a/shared/1systems/wolf.nix
+++ b/shared/1systems/wolf.nix
@@ -11,7 +11,8 @@ in
     ../2configs/collectd-base.nix
     ../2configs/shack-nix-cacher.nix
     ../2configs/shack-drivedroid.nix
-    ../2configs/buildbot-standalone.nix
+    ../2configs/shared-buildbot.nix
+    ../2configs/cgit-mirror.nix
     # ../2configs/graphite.nix
   ];
   # use your own binary cache, fallback use cache.nixos.org (which is used by
diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix
index 5e6072661..dd698ba97 100644
--- a/shared/2configs/base.nix
+++ b/shared/2configs/base.nix
@@ -16,20 +16,16 @@ with lib;
   # TODO rename shared user to "krebs"
   krebs.build.user = mkDefault config.krebs.users.shared;
   krebs.build.source = {
-    git.nixpkgs = {
+    upstream-nixpkgs = mkDefault {
       url = https://github.com/NixOS/nixpkgs;
       rev = "d0e3cca";
-      target-path = "/var/src/nixpkgs";
-    };
-    dir.secrets = {
-      host = config.krebs.current.host;
-      path = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
-    };
-    dir.stockholm = {
-      host = config.krebs.current.host;
-      path = mkDefault "${getEnv "HOME"}/stockholm";
-      target-path = "/var/src/stockholm";
     };
+    secrets =  mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}";
+    stockholm = mkDefault "${getEnv "HOME"}/stockholm";
+
+    nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix";
+    nixpkgs = symlink:stockholm/nixpkgs;
+    stockholm-user = "symlink:stockholm/${config.krebs.build.user.name}";
   };
 
   networking.hostName = config.krebs.build.host.name;
diff --git a/shared/2configs/cgit-mirror.nix b/shared/2configs/cgit-mirror.nix
new file mode 100644
index 000000000..4ff1902f9
--- /dev/null
+++ b/shared/2configs/cgit-mirror.nix
@@ -0,0 +1,40 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  rules = with git; singleton {
+    user = [ git-sync ];
+    repo = [ stockholm-mirror ];
+    perm = push ''refs/*'' [ non-fast-forward create delete merge ];
+  };
+
+  stockholm-mirror = {
+    public = true;
+    name = "stockholm-mirror";
+    desc = "mirror for all stockholm branches";
+    hooks = {
+      post-receive = pkgs.git-hooks.irc-announce {
+        nick = config.networking.hostName;
+        verbose = false;
+        channel = "#retiolum";
+        server = "cd.retiolum";
+      };
+    };
+  };
+
+  git-sync = {
+    name = "git-sync";
+    mail = "spam@krebsco.de";
+    # TODO put git-sync pubkey somewhere more appropriate
+    pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzUuzyoAhMgJmsiaTVWNSXqcrZNTpKpv0nfFBOMcNXUWEbvfAq5eNpg5cX+P8eoYl6UQgfftbYi06flKK3yJdntxoZKLwJGgJt9NZr8yZTsiIfMG8XosvGNQtGPkBtpLusgmPpu7t2RQ9QrqumBvoUDGYEauKTslLwupp1QeyWKUGEhihn4CuqQKiPrz+9vbNd75XOfVZMggk3j4F7HScatmA+p1EQXWyq5Jj78jQN5ZIRnHjMQcIZ4DOz1U96atwSKMviI1xEZIODYfgoGjjiWYeEtKaLVPtSqtLRGI7l+RNouMfwHLdTWOJSlIdFncfPXC6R19hTll3UHeHLtqLP git-sync'';
+  };
+
+in {
+  krebs.git = {
+    enable = true;
+    root-title = "Shared Repos";
+    root-desc = "keep on krebsing";
+    inherit rules;
+    repos.stockholm-mirror = stockholm-mirror;
+  };
+}
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/shared-buildbot.nix
index 9982dd915..50b279036 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/shared-buildbot.nix
@@ -1,5 +1,9 @@
 { lib, config, pkgs, ... }:
+# The buildbot config is seilf-contained and provides a way to test "shared"
+# configuration (infrastructure to be used by every krebsminister).
 
+# You can add your own test, test steps as required. Deploy the config on a
+# shared host like wolf and everything should be fine.
 {
   networking.firewall.allowedTCPPorts = [ 8010 9989 ];
   krebs.buildbot.master = {
@@ -59,7 +63,10 @@
             "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
   # TODO: --pure , prepare ENV in nix-shell command:
   #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
-  nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ]
+  nixshell = ["nix-shell",
+                "-I", "stockholm=.",
+                "-I", "nixpkgs=/var/src/upstream-nixpkgs",
+                "-p" ] + deps + [ "--run" ]
 
   # prepare addShell function
   def addShell(factory,**kwargs):
@@ -69,14 +76,9 @@
       fast-tests = ''
   f = util.BuildFactory()
   f.addStep(grab_repo)
-  addShell(f,name="deploy-eval-centos7",env=env,
-            command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"])
-
-  addShell(f,name="deploy-eval-wolf",env=env,
-            command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"])
-
-  addShell(f,name="deploy-eval-cross-check",env=env,
-            command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"])
+  for i in [ "test-centos7", "wolf", "test-failing" ]:
+    addShell(f,name="populate-{}".format(i),env=env,
+            command=nixshell + ["set -o pipefail;{}( nix-instantiate --arg configuration shared/1systems/{}.nix --eval --readonly-mode --show-trace -A config.krebs.build.populate --strict | jq -r .)".format("!" if "failing" in i else "",i)])
 
   addShell(f,name="instantiate-test-all-modules",env=env,
             command=nixshell + \
@@ -86,8 +88,6 @@
                             -I stockholm=. \
                             --show-trace \
                             -I secrets=. '<stockholm>' \
-                            --argstr current-user-name shared \
-                            --argstr current-host-name lol \
                             --strict --json"])
 
   addShell(f,name="instantiate-test-minimal-deploy",env=env,
@@ -97,8 +97,6 @@
                             -I stockholm=. \
                             -I secrets=. '<stockholm>' \
                             --show-trace \
-                            --argstr current-user-name shared \
-                            --argstr current-host-name lol \
                             --strict --json"])
 
   bu.append(util.BuilderConfig(name="fast-tests",
@@ -145,6 +143,6 @@
     password = "krebspass";
     packages = with pkgs;[ git nix ];
     # all nix commands will need a working nixpkgs installation
-    extraEnviron = { NIX_PATH="nixpkgs=${toString <nixpkgs>}"; };
+    extraEnviron = { NIX_PATH="/var/src"; };
   };
 }