Merge 'cd/master' - update krebs.build.source

62 files changed, 772 insertions, 730 deletions

diff --git a/.rsync-filter b/.rsync-filter
new file mode 100644
index 000000000..d7657cd00
--- /dev/null
+++ b/.rsync-filter
@@ -0,0 +1,2 @@
+- /.git
+- /.graveyard
diff --git a/Makefile b/Makefile
index 5b898c54c..e82e39c7e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,53 +1,51 @@
-#
-# usage:
-#		make infest system=foo [target=bar]
-#		make [deploy] system=foo [target=bar]
-#		make [deploy] systems='foo bar'
-#		make eval get=users.tv.wu.config.time.timeZone [filter=json]
-#
-
 .ONESHELL:
 .SHELLFLAGS := -eufc
 
-ifdef systems
-$(systems):
-	@
-	unset target
-	parallel \
-		--line-buffer \
-		-j0 \
-		--no-notice \
-		--tagstring {} \
-		-q make -s systems= system={} ::: $(systems)
-else ifdef system
-.PHONY: deploy infest
-deploy infest:;@
-	export get=krebs.$@
-	export filter=json
-	make -s eval | sh
+ifndef system
+$(error unbound variable: system)
+endif
+
+export target_host ?= $(system)
+export target_user ?= root
+export target_path ?= /var/src
+
+# usage: make deploy system=foo [target_host=bar]
+.PHONY: deploy
+deploy: populate ;@set -x
+	ssh "$$target_user@$$target_host" nixos-rebuild switch -I "$$target_path"
 
+# usage: make populate system=foo [target_host=bar]
+.PHONY: populate
+populate:;@
+	result=$$(make -s eval get=config.krebs.build.populate filter=json)
+	echo "$$result" | sh
+
+# usage: make eval system=foo get=config.krebs.build [LOGNAME=tv] [filter=json]
 .PHONY: eval
-eval:
-	@
+eval:;@
 ifeq ($(filter),json)
 	extraArgs='--json --strict'
-	filter() { jq -r .; }
+	filter() { echo "$$1" | jq -r .; }
 else
-	filter() { cat; }
+	filter() { echo "$$1"; }
 endif
 	result=$$(nix-instantiate \
 		$${extraArgs-} \
+		--show-trace \
+		--readonly-mode \
 		--eval \
 		-A "$$get" \
-		-I stockholm="$$PWD" \
-		'<stockholm>' \
-		--argstr current-date "$$(date -Is)" \
-		--argstr current-host-name "$$HOSTNAME" \
-		--argstr current-user-name "$$LOGNAME" \
-		$${system+--argstr system "$$system"} \
-		$${target+--argstr target "$$target"})
-	echo "$$result" | filter
+		--arg configuration "./$$LOGNAME/1systems/$$system.nix")
+	filter "$$result"
 
-else
-$(error unbound variable: system[s])
-endif
+## usage: make install system=foo target=
+#.PHONY: install
+#install: ssh = ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+#install:;@set -x
+#	$(ssh) "$$target_user@$$target_host" \
+#		env target_path=/var/src \
+#			sh -s prepare < krebs/4lib/infest/prepare.sh
+#	make -s populate target_path=/mnt"$$target_path"
+#	$(ssh) "$$target_user@$$target_host" \
+#		env NIXOS_CONFIG=/var/src/nixos-config \
+#			nixos-install
diff --git a/default.nix b/default.nix
index 1637aa464..278f1d14d 100644
--- a/default.nix
+++ b/default.nix
@@ -1,6 +1,15 @@
-{ current-date ? abort "current-date not defined"
-, current-host-name ? abort "current-host-name not defined"
-, current-user-name ? builtins.getEnv "LOGNAME"
+{ configuration ? import (nixpkgs-path + "/nixos/lib/from-env.nix") "NIXOS_CONFIG" <nixos-config>
+, system ? builtins.currentSystem
+, current-host-name ?
+    let v = builtins.getEnv "HOSTNAME"; in
+    if v != "" then v else builtins.readFile /proc/sys/kernel/hostname
+, current-user-name ?
+    let v = builtins.getEnv "LOGNAME"; in
+    if v != "" then v else abort "undefined variable: LOGNAME"
+, nixpkgs-path ?
+    if (builtins.tryEval <nixpkgs/krebs>).success
+      then <upstream-nixpkgs>
+      else <nixpkgs>
 , StrictHostKeyChecking ? "yes"
 }@args:
 
@@ -8,26 +17,26 @@ let stockholm = {
     inherit krebs;
     inherit users;
     inherit lib;
-    inherit pkgs;
+    inherit config options pkgs;
+    system = config.system.build.toplevel;
   };
 
   krebs = import ./krebs (args // { inherit lib stockholm; });
 
   lib = let
-    nlib = import <nixpkgs/lib>;
+    nlib = import (slib.npath "lib");
     klib = import (slib.kpath "4lib") { lib = nlib; };
     slib = rec {
-      stockholm-path = ./.;
-      nspath = ns: p: stockholm-path + "/${ns}/${p}";
-      kpath = nspath "krebs";
-      upath = nspath current-user-name;
+      npath = p: nixpkgs-path + "/${p}";
+      kpath = p: ./. + "/krebs/${p}";
+      upath = p: ./. + "/${current-user-name}/${p}";
     };
     ulib = let p = slib.upath "4lib"; in
       nlib.optionalAttrs (klib.dir.has-default-nix p)
                          (import p { lib = nlib // klib; });
   in nlib // klib // slib // ulib // builtins;
 
-  inherit (eval {}) pkgs;
+  inherit (eval configuration) config options pkgs;
 
   base-module = { config, ... }: {
     imports = builtins.filter lib.dir.has-default-nix (lib.concatLists [
@@ -45,7 +54,8 @@ let stockholm = {
     in kpkgs // upkgs;
   };
 
-  eval = config: import <nixpkgs/nixos/lib/eval-config.nix> {
+  eval = config: import (lib.npath "nixos/lib/eval-config.nix") {
+    inherit system;
     specialArgs = {
       inherit lib;
     };
diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix
index 01bb16a2b..a1f335905 100644
--- a/krebs/3modules/backup.nix
+++ b/krebs/3modules/backup.nix
@@ -28,9 +28,17 @@ let
             type = types.krebs.file-location;
           };
           startAt = mkOption {
-            type = types.str;
+            default = "hourly";
+            type = types.str; # TODO systemd.time(7)'s calendar event
           };
           snapshots = mkOption {
+            default = {
+              hourly   = { format = "%Y-%m-%dT%H"; retain =  4; };
+              daily    = { format = "%Y-%m-%d";    retain =  7; };
+              weekly   = { format = "%YW%W";       retain =  4; };
+              monthly  = { format = "%Y-%m";       retain = 12; };
+              yearly   = { format = "%Y";                       };
+            };
             type = types.attrsOf (types.submodule {
               options = {
                 format = mkOption {
@@ -284,3 +292,15 @@ let
   };
 
 in out
+# TODO ionice
+# TODO mail on failed push, pull
+# TODO mail on missing push
+# TODO don't cancel plans on activation
+#   also, don't hang while deploying at:
+#   starting the following units: backup.wu-home-xu.push.service, backup.wu-home-xu.push.timer
+# TODO make sure /bku is properly mounted
+# TODO make sure that secure hosts cannot backup to insecure ones
+# TODO optionally only backup when src and dst are near enough :)
+# TODO try using btrfs for snapshots (configurable)
+# TODO warn if partial snapshots are found
+# TODO warn if unknown stuff is found in dst path
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index 7f004cd81..00142acdd 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -28,48 +28,159 @@ let
       type = types.user;
     };
 
-    options.krebs.build.source.dir = mkOption {
-      type = let
-        default-host = config.krebs.current.host;
-      in types.attrsOf (types.submodule ({ config, ... }: {
+    options.krebs.build.source = let
+      raw = types.either types.str types.path;
+      url = types.submodule {
         options = {
-          host = mkOption {
-            type = types.host;
-            default = default-host;
-          };
-          path = mkOption {
-            type = types.str;
-          };
-          target-path = mkOption {
-            type = types.str;
-            default = "/root/${config._module.args.name}";
-          };
           url = mkOption {
             type = types.str;
-            default = "file://${config.host.name}${config.path}";
-          };
-        };
-      }));
-      default = {};
-    };
-
-    options.krebs.build.source.git = mkOption {
-      type = with types; attrsOf (submodule ({ config, ... }: {
-        options = {
-          url = mkOption {
-            type = types.str; # TODO must be shell safe
           };
           rev = mkOption {
             type = types.str;
           };
-          target-path = mkOption {
+          dev = mkOption {
             type = types.str;
-            default = "/root/${config._module.args.name}";
           };
         };
-      }));
+      };
+    in mkOption {
+      type = types.attrsOf (types.either types.str url);
+      apply = let f = mapAttrs (_: value: {
+        string = value;
+        path = toString value;
+        set = f value;
+      }.${typeOf value}); in f;
       default = {};
     };
+
+    options.krebs.build.populate = mkOption {
+      type = types.str;
+      default = let
+        source = config.krebs.build.source;
+        target-user = maybeEnv "target_user" "root";
+        target-host = maybeEnv "target_host" config.krebs.build.host.name;
+        target-path = maybeEnv "target_path" "/var/src";
+        out = ''
+          #! /bin/sh
+          set -eu
+
+          verbose() {
+            printf '+%s\n' "$(printf ' %q' "$@")" >&2
+            "$@"
+          }
+
+          echo ${shell.escape git-script} \
+            | ssh ${shell.escape "${target-user}@${target-host}"} -T
+
+          unset tmpdir
+          trap '
+            rm "$tmpdir"/*
+            rmdir "$tmpdir"
+            trap - EXIT INT QUIT
+          '        EXIT INT QUIT
+          tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
+          chmod 0755 "$tmpdir"
+
+          ${concatStringsSep "\n"
+            (mapAttrsToList
+              (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
+                "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
+              symlink-specs)}
+
+          verbose proot \
+              -b $tmpdir:${shell.escape target-path} \
+              ${concatStringsSep " \\\n    "
+                (mapAttrsToList
+                  (name: spec:
+                    "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
+                  file-specs)} \
+              rsync \
+                  -f ${shell.escape "P /*"} \
+                  ${concatMapStringsSep " \\\n        "
+                    (name: "-f ${shell.escape "R /${name}"}")
+                    (attrNames file-specs)} \
+                  --delete \
+                  -vFrlptD \
+                  ${shell.escape target-path}/ \
+                  ${shell.escape "${target-user}@${target-host}:${target-path}"}
+        '';
+
+        get-schema = uri:
+          if substring 0 1 uri == "/"
+            then "file"
+            else head (splitString ":" uri);
+
+        has-schema = schema: uri: get-schema uri == schema;
+
+        get-url = spec: {
+          string = spec;
+          path = toString spec;
+          set = get-url spec.url;
+        }.${typeOf spec};
+
+        git-specs =
+          filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
+          filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
+          filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
+
+        file-specs =
+          filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
+
+        symlink-specs =
+          filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
+
+        git-script = ''
+          #! /bin/sh
+          set -efu
+
+          verbose() {
+            printf '+%s\n' "$(printf ' %q' "$@")" >&2
+            "$@"
+          }
+
+          fetch_git() {(
+            dst_dir=$1
+            src_url=$2
+            src_ref=$3
+
+            if ! test -e "$dst_dir"; then
+              git clone "$src_url" "$dst_dir"
+            fi
+
+            cd "$dst_dir"
+
+            if ! url=$(git config remote.origin.url); then
+              git remote add origin "$src_url"
+            elif test "$url" != "$src_url"; then
+              git remote set-url origin "$src_url"
+            fi
+
+            # TODO resolve src_ref to commit hash
+            hash=$src_ref
+
+            if ! test "$(git log --format=%H -1)" = "$hash"; then
+              git fetch origin
+              git checkout "$hash" -- "$dst_dir"
+              git checkout "$hash"
+            fi
+
+            git clean -dxf
+          )}
+
+          ${concatStringsSep "\n"
+            (mapAttrsToList
+              (name: spec: toString (map shell.escape [
+                "verbose"
+                "fetch_git"
+                "${target-path}/${name}"
+                spec.url
+                spec.rev
+              ]))
+              git-specs)}
+        '';
+      in out;
+    };
+
   };
 
 in out
diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix
index e6267d7e6..7b28ffca8 100644
--- a/krebs/3modules/git.nix
+++ b/krebs/3modules/git.nix
@@ -27,7 +27,7 @@ let
       description = ''
           Enable cgit.
           Cgit is an attempt to create a fast web interface for the git version
-          control system, using a built in cache to decrease pressure on the 
+          control system, using a built in cache to decrease pressure on the
           git server.
           cgit in this module is being served via fastcgi nginx.This module
           deploys a http://cgit.<hostname> nginx configuration and enables nginx
@@ -44,48 +44,8 @@ let
       default = "/etc/git";
     };
     repos = mkOption {
-      type = types.attrsOf (types.submodule ({
-        options = {
-          desc = mkOption {
-            type = types.nullOr types.str;
-            default = null;
-            description = ''
-              Repository description.
-            '';
-          };
-          section = mkOption {
-            type = types.nullOr types.str;
-            default = null;
-            description = ''
-              Repository section.
-            '';
-          };
-          name = mkOption {
-            type = types.str;
-            description = ''
-              Repository name.
-            '';
-          };
-          hooks = mkOption {
-            type = types.attrsOf types.str;
-            default = {};
-            description = ''
-              Repository-specific hooks.
-            '';
-          };
-          public = mkOption {
-            type = types.bool;
-            default = false;
-            description = ''
-              Allow everybody to read the repository via HTTP if cgit enabled.
-            '';
-            # TODO allow every configured user to fetch the repository via SSH.
-          };
-        };
-      }));
-
+      type = types.attrsOf subtypes.repo;
       default = {};
-
       example = literalExample ''
         {
           testing = {
@@ -99,7 +59,6 @@ let
           testing2 = { name = "testing2"; };
         }
       '';
-
       description = ''
         Repositories.
       '';
@@ -121,30 +80,158 @@ let
       '';
     };
     rules = mkOption {
-      type = types.unspecified;
+      type = types.listOf subtypes.rule;
+      default = [];
+      example = literalExample ''
+        singleton {
+          user = [ config.krebs.users.tv ];
+          repo = [ testing ]; # see literal example of repos
+          perm = push "refs/*" (with lib.git; [
+            non-fast-forward create delete merge
+          ]);
+        }
+      '';
+      description = ''
+        Rules.
+      '';
     };
   };
 
+  # TODO put into krebs/4lib/types.nix?
+  subtypes = {
+    repo = types.submodule ({
+      options = {
+        collaborators = mkOption {
+          type = types.listOf types.user;
+          default = [];
+          description = ''
+            List of users that should be able to fetch from this repo.
+
+            This option is currently not used by krebs.git but instead can be
+            used to create rules.  See e.g. <stockholm/tv/2configs/git.nix> for
+            an example.
+          '';
+        };
+        desc = mkOption {
+          type = types.nullOr types.str;
+          default = null;
+          description = ''
+            Repository description.
+          '';
+        };
+        section = mkOption {
+          type = types.nullOr types.str;
+          default = null;
+          description = ''
+            Repository section.
+          '';
+        };
+        name = mkOption {
+          type = types.str;
+          description = ''
+            Repository name.
+          '';
+        };
+        hooks = mkOption {
+          type = types.attrsOf types.str;
+          default = {};
+          description = ''
+            Repository-specific hooks.
+          '';
+        };
+        public = mkOption {
+          type = types.bool;
+          default = false;
+          description = ''
+            Allow everybody to read the repository via HTTP if cgit enable