Merge remote-tracking branch 'cd/master'

12 files changed, 275 insertions, 99 deletions

diff --git a/Makefile b/Makefile
index 384c872ab..60dfe8030 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,32 @@
-ifndef system
-$(error unbound variable: system)
+stockholm ?= .
+
+ifndef nixos-config
+$(if $(system),,$(error unbound variable: system))
+nixos-config = ./$(LOGNAME)/1systems/$(system).nix
+endif
+
+# target = [target_user@]target_host[:target_port][/target_path]
+ifdef target
+_target_user != echo $(target) | sed -n 's/@.*//p'
+_target_path != echo $(target) | sed -n 's/^[^/]*//p'
+_target_port != echo $(target) | sed -En 's|^.*:([^/]*)(/.*)?$$|\1|p'
+_target_host != echo $(target) | sed -En 's/^(.*@)?([^:/]*).*/\2/p'
+ifneq ($(_target_host),)
+$(if $(target_host),$(error cannot define both, target_host and host in target))
+target_host ?= $(_target_host)
+endif
+ifneq ($(_target_user),)
+$(if $(target_user),$(error cannot define both, target_user and user in target))
+target_user ?= $(_target_user)
+endif
+ifneq ($(_target_port),)
+$(if $(target_port),$(error cannot define both, target_port and port in target))
+target_port ?= $(_target_port)
+endif
+ifneq ($(_target_path),)
+$(if $(target_path),$(error cannot define both, target_path and path in target))
+target_path ?= $(_target_path)
+endif
 endif
 
 export target_host ?= $(system)
@@ -7,13 +34,18 @@ export target_user ?= root
 export target_port ?= 22
 export target_path ?= /var/src
 
+$(if $(target_host),,$(error unbound variable: target_host))
+$(if $(target_user),,$(error unbound variable: target_user))
+$(if $(target_port),,$(error unbound variable: target_port))
+$(if $(target_path),,$(error unbound variable: target_path))
+
 evaluate = \
 	nix-instantiate \
 		--eval \
 		--readonly-mode \
 		--show-trace \
-		-I nixos-config=./$(LOGNAME)/1systems/$(system).nix \
-		-I stockholm=. \
+		-I nixos-config=$(nixos-config) \
+		-I stockholm=$(stockholm) \
 		$(1)
 
 execute = \
@@ -22,9 +54,10 @@ execute = \
 	echo "$$script" | sh
 
 # usage: make deploy system=foo [target_host=bar]
+deploy: ssh ?= ssh
 deploy:
 	$(call execute,populate)
-	ssh $(target_user)@$(target_host) -p $(target_port) \
+	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		nixos-rebuild switch --show-trace -I $(target_path)
 
 # usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name
@@ -41,3 +74,21 @@ install:
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		env NIXOS_CONFIG=$(target_path)/nixos-config \
 			nixos-install
+
+# usage: make test system=foo [target=bar] [method={eval,build}]
+method ?= eval
+ifeq ($(method),build)
+test: command = nix-build --no-out-link
+else
+ifeq ($(method),eval)
+test: command ?= nix-instantiate --eval --json --readonly-mode --strict
+else
+$(error bad method: $(method))
+endif
+endif
+test: ssh ?= ssh
+test:
+	$(call execute,populate)
+	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
+		$(command) --show-trace -I $(target_path) \
+			-A config.system.build.toplevel $(target_path)/stockholm
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index a1f446188..c700fbc56 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -20,35 +20,19 @@ let
       type = types.user;
     };
 
-    options.krebs.build.source = let
-      raw = types.either types.str types.path;
-      url = types.submodule {
+    options.krebs.build.source = mkOption {
+      type = with types; attrsOf (either str (submodule {
         options = {
-          url = mkOption {
-            type = types.str;
-          };
-          rev = mkOption {
-            type = types.str;
-          };
-          dev = mkOption {
-            type = types.str;
-          };
+          url = str;
+          rev = str;
         };
-      };
-    in mkOption {
-      type = types.attrsOf (types.either types.str url);
-      apply = let f = mapAttrs (_: value: {
-        string = value;
-        path = toString value;
-        set = f value;
-      }.${typeOf value}); in f;
+      }));
       default = {};
     };
 
     options.krebs.build.populate = mkOption {
       type = types.str;
       default = let
-        source = config.krebs.build.source;
         target-user = maybeEnv "target_user" "root";
         target-host = maybeEnv "target_host" config.krebs.build.host.name;
         target-port = maybeEnv "target_port" "22";
@@ -75,24 +59,21 @@ let
           tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
           chmod 0755 "$tmpdir"
 
-          ${concatStringsSep "\n"
-            (mapAttrsToList
-              (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
-                "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
-              symlink-specs)}
+          ${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
+            verbose ln -s ${shell.escape symlink.target} \
+                          "$tmpdir"/${shell.escape name}
+          '') source-by-method.symlink)}
 
           verbose proot \
-              -b $tmpdir:${shell.escape target-path} \
-              ${concatStringsSep " \\\n    "
-                (mapAttrsToList
-                  (name: spec:
-                    "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
-                  file-specs)} \
+              -b "$tmpdir":${shell.escape target-path} \
+              ${concatStringsSep " \\\n    " (mapAttrsToList (name: file:
+                "-b ${shell.escape "${file.path}:${target-path}/${name}"}"
+              ) source-by-method.file)} \
               rsync \
                   -f ${shell.escape "P /*"} \
-                  ${concatMapStringsSep " \\\n        "
-                    (name: "-f ${shell.escape "R /${name}"}")
-                    (attrNames file-specs)} \
+                  ${concatMapStringsSep " \\\n        " (name:
+                    "-f ${shell.escape "R /${name}"}"
+                  ) (attrNames source-by-method.file)} \
                   --delete \
                   -vFrlptD \
                   -e ${shell.escape "ssh -p ${target-port}"} \
@@ -100,30 +81,6 @@ let
                   ${shell.escape "${target-user}@${target-host}:${target-path}"}
         '';
 
-        get-schema = uri:
-          if substring 0 1 uri == "/"
-            then "file"
-            else head (splitString ":" uri);
-
-        has-schema = schema: uri: get-schema uri == schema;
-
-        get-url = spec: {
-          string = spec;
-          path = toString spec;
-          set = get-url spec.url;
-        }.${typeOf spec};
-
-        git-specs =
-          filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
-          filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
-          filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
-
-        file-specs =
-          filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
-
-        symlink-specs =
-          filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
-
         git-script = ''
           #! /bin/sh
           set -efu
@@ -162,20 +119,42 @@ let
             git clean -dxf
           )}
 
-          ${concatStringsSep "\n"
-            (mapAttrsToList
-              (name: spec: toString (map shell.escape [
-                "verbose"
-                "fetch_git"
-                "${target-path}/${name}"
-                spec.url
-                spec.rev
-              ]))
-              git-specs)}
+          ${concatStringsSep "\n" (mapAttrsToList (name: git: ''
+            verbose fetch_git ${concatMapStringsSep " " shell.escape [
+              "${target-path}/${name}"
+              git.url
+              git.rev
+            ]}
+          '') source-by-method.git)}
         '';
       in out;
     };
 
   };
 
+  source-by-method = let
+    known-methods = ["git" "file" "symlink"];
+  in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
+    git = source-by-scheme.http or {} //
+          source-by-scheme.https or {};
+  };
+
+  source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
+    ${v.scheme}.${k} = v;
+  }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
+
+  normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
+    path = f (toString x);
+    string = f {
+      url = if substring 0 1 x == "/" then "file://${x}" else x;
+    };
+    set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
+      inherit scheme;
+    } // {
+      symlink.target = removePrefix "symlink:" x.url;
+      file.path = # TODO file://host/...
+                  assert hasPrefix "file:///" x.url;
+                  removePrefix "file://" x.url;
+    }.${scheme} or {};
+  }; in f) config.krebs.build.source;
 in out
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index 825cb3413..2a1dbe31a 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -338,8 +338,8 @@ let
         SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
       };
       serviceConfig = let
-        workdir="${lib.shell.escape cfg.workDir}";
-        secretsdir="${lib.shell.escape (toString <secrets>)}";
+        workdir = shell.escape cfg.workDir;
+        secretsdir = shell.escape (toString <secrets>);
       in {
         PermissionsStartOnly = true;
         Type = "forking";
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
index 7705ac31c..248b46132 100644
--- a/krebs/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -149,9 +149,9 @@ let
       } // cfg.extraEnviron;
 
       serviceConfig = let
-        workdir = "${lib.shell.escape cfg.workDir}";
-        contact = "${lib.shell.escape cfg.contact}";
-        description = "${lib.shell.escape cfg.description}";
+        workdir = shell.escape cfg.workDir;
+        contact = shell.escape cfg.contact;
+        description = shell.escape cfg.description;
         buildbot = pkgs.buildbot-slave;
         # TODO:make this
       in {
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 16a74e7c1..c06f3754e 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -31,6 +31,7 @@ let
       ./setuid.nix
       ./tinc_graphs.nix
       ./urlwatch.nix
+      ./repo-sync.nix
     ];
     options.krebs = api;
     config = lib.mkIf cfg.enable imp;
diff --git a/krebs/3modules/repo-sync.nix b/krebs/3modules/repo-sync.nix
new file mode 100644
index 000000000..7a7c80a75
--- /dev/null
+++ b/krebs/3modules/repo-sync.nix
@@ -0,0 +1,109 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.krebs.repo-sync;
+
+  out = {
+    options.krebs.repo-sync = api;
+    config = mkIf cfg.enable imp;
+  };
+
+  api = {
+    enable = mkEnableOption "repo-sync";
+    config = mkOption {
+      type = with types;attrsOf (attrsOf (attrsOf str));
+      example = literalExample ''
+        # see `repo-sync --help`
+        #   `ref` provides sane defaults and can be omitted
+
+        # attrset will be converted to json and be used as config
+        {
+            makefu = {
+                origin = {
+                    url = http://github.com/makefu/repo ;
+                    ref = "heads/dev" ;
+                };
+                mirror = {
+                    url = "git@internal:mirror" ;
+                    ref = "heads/github-mirror-dev" ;
+                };
+            };
+            lass = {
+                origin = {
+                    url = http://github.com/lass/repo ;
+                };
+                mirror = {
+                    url = "git@internal:mirror" ;
+                };
+            };
+            "@latest" = {
+                mirror = {
+                    url = "git@internal:mirror";
+                    ref = "heads/master";
+                };
+            };
+        };
+      '';
+    };
+    timerConfig = mkOption {
+      type = types.attrsOf types.str;
+      default = {
+        OnCalendar = "*:00,15,30,45";
+      };
+    };
+    stateDir = mkOption {
+      type = types.str;
+      default = "/var/lib/repo-sync";
+    };
+    privateKeyFile = mkOption {
+      type = types.str;
+      description = ''
+        used by repo-sync to identify with ssh service
+      '';
+      default = toString <secrets/wolf-repo-sync.rsa_key.priv>;
+    };
+  };
+  repo-sync-config = pkgs.writeText "repo-sync-config.json"
+    (builtins.toJSON cfg.config);
+
+  imp = {
+    users.users.repo-sync = {
+      name = "repo-sync";
+      uid = config.krebs.lib.genid "repo-sync";
+      description = "repo-sync user";
+      home = cfg.stateDir;
+      createHome = true;
+    };
+
+    systemd.timers.repo-sync = {
+      description = "repo-sync timer";
+      wantedBy = [ "timers.target" ];
+
+      timerConfig = cfg.timerConfig;
+    };
+    systemd.services.repo-sync = {
+      description = "repo-sync";
+      after = [ "network.target" ];
+
+      path = with pkgs; [ ];
+
+      environment = {
+        GIT_SSH_COMMAND = "${pkgs.openssh}/bin/ssh -i ${cfg.stateDir}/ssh.priv";
+      };
+
+      serviceConfig = {
+        Type = "simple";
+        PermissionsStartOnly = true;
+        ExecStartPre = pkgs.writeScript "prepare-repo-sync-user" ''
+          #! /bin/sh
+          cp -v ${config.krebs.lib.shell.escape cfg.privateKeyFile} ${cfg.stateDir}/ssh.priv
+          chown repo-sync ${cfg.stateDir}/ssh.priv
+        '';
+        ExecStart = "${pkgs.repo-sync}/bin/repo-sync ${repo-sync-config}";
+        WorkingDirectory = cfg.stateDir;
+        User = "repo-sync";
+      };
+    };
+  };
+in out
diff --git a/krebs/5pkgs/repo-sync/default.nix b/krebs/5pkgs/repo-sync/default.nix
index 90f838de9..789c03f36 100644
--- a/krebs/5pkgs/repo-sync/default.nix
+++ b/krebs/5pkgs/repo-sync/default.nix
@@ -1,15 +1,17 @@
 { lib, pkgs, python3Packages, fetchurl, ... }:
+
 with python3Packages; buildPythonPackage rec {
   name = "repo-sync-${version}";
-  version = "0.1.1";
+  version = "0.2.5";
   disabled = isPy26 || isPy27;
   propagatedBuildInputs = [
     docopt
     GitPython
+    pkgs.git
   ];
   src = fetchurl {
     url = "https://pypi.python.org/packages/source/r/repo-sync/repo-sync-${version}.tar.gz";
-    sha256 = "01r30l2bbsld90ps13ip0zi2a41b53dv4q6fxrzvkfrprr64c0vv";
+    sha256 = "1a59bj0vc5ajq8indkvkdk022yzvvv5mjb57hk3xf1j3wpr85p84";
   };
   meta = {
     homepage = http://github.com/makefu/repo-sync;
diff --git a/shared/1systems/wolf.nix b/shared/1systems/wolf.nix
index 317591433..96691aed8 100644
--- a/shared/1systems/wolf.nix
+++ b/shared/1systems/wolf.nix
@@ -14,6 +14,7 @@ in
     ../2configs/shack-drivedroid.nix
     ../2configs/shared-buildbot.nix
     ../2configs/cgit-mirror.nix
+    ../2configs/repo-sync.nix
     # ../2configs/graphite.nix
   ];
   # use your own binary cache, fallback use cache.nixos.org (which is used by
diff --git a/shared/2configs/cgit-mirror.nix b/shared/2configs/cgit-mirror.nix
index d30f1444f..b984535c9 100644
--- a/shared/2configs/cgit-mirror.nix
+++ b/shared/2configs/cgit-mirror.nix
@@ -3,7 +3,7 @@
 with config.krebs.lib;
 let
   rules = with git; singleton {
-    user = [ git-sync ];
+    user = [ wolf-repo-sync ];
     repo = [ stockholm-mirror ];
     perm = push ''refs/*'' [ non-fast-forward create delete merge ];
   };
@@ -22,14 +22,15 @@ let
     };
   };
 
-  git-sync = {
-    name = "git-sync";
+  wolf-repo-sync = {
+    name = "wolf-repo-sync";
     mail = "spam@krebsco.de";
     # TODO put git-sync pubkey somewhere more appropriate
-    pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzUuzyoAhMgJmsiaTVWNSXqcrZNTpKpv0nfFBOMcNXUWEbvfAq5eNpg5cX+P8eoYl6UQgfftbYi06flKK3yJdntxoZKLwJGgJt9NZr8yZTsiIfMG8XosvGNQtGPkBtpLusgmPpu7t2RQ9QrqumBvoUDGYEauKTslLwupp1QeyWKUGEhihn4CuqQKiPrz+9vbNd75XOfVZMggk3j4F7HScatmA+p1EQXWyq5Jj78jQN5ZIRnHjMQcIZ4DOz1U96atwSKMviI1xEZIODYfgoGjjiWYeEtKaLVPtSqtLRGI7l+RNouMfwHLdTWOJSlIdFncfPXC6R19hTll3UHeHLtqLP git-sync'';
+    pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwuAZB3wtAvBJFYh+gWdyGaZU4mtqM2dFXmh2rORlbXeh02msu1uv07ck1VKkQ4LgvCBcBsAOeVa1NTz99eLqutwgcqMCytvRNUCibcoEWwHObsK53KhDJj+zotwlFhnPPeK9+EpOP4ngh/tprJikttos5BwBwe2K+lfiid3fmVPZcTTYa77nCwijimMvWEx6CEjq1wiXMUc4+qcEn8Swbwomz/EEQdNE2hgoC3iMW9RqduTFdIJWnjVi0KaxenX9CvQRGbVK5SSu2gwzN59D/okQOCP6+p1gL5r3QRHSLSSRiEHctVQTkpKOifrtLZGSr5zArEmLd/cOVyssHQPCX repo-sync@wolf'';
   };
 
 in {
+  krebs.users.wolf-repo-sync = wolf-repo-sync;
   krebs.git = {
     enable = true;
     root-title = "Shared Repos";
diff --git a/shared/2configs/repo-sync.nix b/shared/2configs/repo-sync.nix
new file mode 100644
index 000000000..b23cb1675
--- /dev/null
+++ b/shared/2configs/repo-sync.nix
@@ -0,0 +1,28 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+{
+  krebs.repo-sync = let
+    # TODO addMirrorURL function
+    mirror = "git@wolf:stockholm-mirror";
+  in {
+    enable = true;
+    config = {
+      makefu = {
+        origin.url = http://cgit.gum/stockholm ;
+        mirror.url = mirror;
+      };
+      tv = {
+        origin.url = http://cgit.cd/stockholm ;
+        mirror.url = mirror;
+      };
+      lassulus = {
+        origin.url = http://cgit.cloudkrebs/stockholm ;
+        mirror.url = mirror;
+      };
+      "@latest" = {
+        mirror.url = mirror;
+      };
+    };
+  };
+}
diff --git a/shared/2configs/shack-drivedroid.nix b/shared/2configs/shack-drivedroid.nix
index 08a6b0697..6133ccc99 100644
--- a/shared/2configs/shack-drivedroid.nix
+++ b/shared/2configs/shack-drivedroid.nix
@@ -1,7 +1,8 @@
 { pkgs, lib, config, ... }:
+with config.krebs.lib;
 let
   repodir = "/var/srv/drivedroid";
-  srepodir = lib.shell.escape repodir;
+  srepodir = shell.escape repodir;
 in
 {
   environment.systemPackages = [ pkgs.drivedroid-gen-repo ];
@@ -40,5 +41,4 @@ in
       };
     };
   };
-
 }
diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix
index af877f5d8..ebf5f4a1e 100644
--- a/shared/2configs/shared-buildbot.nix
+++ b/shared/2configs/shared-buildbot.nix
@@ -1,18 +1,22 @@
 { lib, config, pkgs, ... }:
-# The buildbot config is seilf-contained and provides a way to test "shared"
-# configuration (infrastructure to be used by every krebsminister).
+# The buildbot config is self-contained and currently provides a way 
+# to test "shared" configuration (infrastructure to be used by every krebsminister).
 
 # You can add your own test, test steps as required. Deploy the config on a
 # shared host like wolf and everything should be fine.
+
+# TODO for all users schedule a build for fast tests
 {
   networking.firewall.allowedTCPPorts = [ 8010 9989 ];
-  krebs.buildbot.master = {
+  krebs.buildbot.master = let
+    stockholm-mirror-url = http://cgit.wolf/stockholm-mirror ;
+  in {
     secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ];
     slaves = {
       testslave =  "krebspass";
     };
     change_source.stockholm = ''
-  stockholm_repo = 'http://cgit.wolf/stockholm-mirror'
+  stockholm_repo = '${stockholm-mirror-url}'
   cs.append(changes.GitPoller(
           stockholm_repo,
           workdir='stockholm-poller', branches=True,
@@ -23,16 +27,15 @@
         force-scheduler = ''
   sched.append(schedulers.ForceScheduler(
                               name="force",
-                              builderNames=["full-tests"]))
+                              builderNames=["full-tests","fast-tests"]))
         '';
         fast-tests-scheduler = ''
-  # test the master real quick
+  # test everything real quick
   sched.append(schedulers.SingleBranchScheduler(
                               ## all branches
                               change_filter=util.ChangeFilter(branch_re=".*"),
-                              # change_filter=util.ChangeFilter(branch="master"),
-                              treeStableTimer=10, #only test the latest push
-                              name="fast-master-test",
+                              # treeStableTimer=10,
+                              name="fast-test-all-branches",
                               builderNames=["fast-tests"]))
         '';
         test-cac-infest-master = ''
@@ -61,7 +64,7 @@
   # prepare nix-shell
   # the dependencies which are used by the test script
   deps = [ "gnumake", "jq","nix","rsync",
-            "(import <stockholm> {}).pkgs.test.infest-cac-centos7" ]
+            "(import <stockholm>).pkgs.test.infest-cac-centos7" ]
   # TODO: --pure , prepare ENV in nix-shell command:
   #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
   nixshell = ["nix-shell",
@@ -133,7 +136,7 @@
     };
     irc = {
       enable = true;
-      nick = "shared-buildbot";
+      nick = "wolfbot";
       server = "cd.retiolum";
       channels = [ "retiolum" ];
       allowForce = true;
@@ -147,6 +150,7 @@
     password = "krebspass";
     packages = with pkgs;[ git nix ];
     # all nix commands will need a working nixpkgs installation
-    extraEnviron = { NIX_PATH="/var/src"; };
+    extraEnviron = {
+      NIX_PATH="nixpkgs=/var/src/upstream-nixpkgs:nixos-config=./shared/1systems/wolf.nix"; };
   };
 }