Merge 'cd/master'

7 files changed, 148 insertions, 200 deletions

diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index 9cd095622..4848748cd 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -1,165 +1,27 @@
-{ config, lib, ... }:
+{ config, ... }:
 
 with config.krebs.lib;
 
-let
-  out = {
+{
+  options.krebs.build = {
     # TODO deprecate krebs.build.host
-    options.krebs.build.host = mkOption {
+    host = mkOption {
       type = types.host;
     };
 
-    # TODO make krebs.build.profile shell safe
-    options.krebs.build.profile = mkOption {
-      type = types.str;
+    profile = mkOption {
+      type = types.absolute-path;
       default = "/nix/var/nix/profiles/system";
     };
 
-    # TODO deprecate krebs.build.user
-    options.krebs.build.user = mkOption {
-      type = types.user;
-    };
-
-    options.krebs.build.source = mkOption {
-      type = with types; attrsOf (either str (submodule {
-        options = {
-          url = str;
-          rev = str;
-        };
-      }));
+    source = mkOption {
+      type = types.attrsOf types.source;
       default = {};
     };
 
-    options.krebs.build.populate = mkOption {
-      type = types.str;
-      default = let
-        target-user = maybeEnv "target_user" "root";
-        target-host = maybeEnv "target_host" config.krebs.build.host.name;
-        target-port = maybeEnv "target_port" "22";
-        target-path = maybeEnv "target_path" "/var/src";
-        out = ''
-          #! /bin/sh
-          set -eu
-
-          ssh=''${ssh-ssh}
-
-          verbose() {
-            printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
-            "$@"
-          }
-
-          { printf 'PS5=%q%q\n' @ "$PS5"
-            echo ${shell.escape git-script}
-          } | verbose $ssh -p ${shell.escape target-port} \
-                  ${shell.escape "${target-user}@${target-host}"} -T
-
-          unset tmpdir
-          trap '
-            rm -f "$tmpdir"/*
-            rmdir "$tmpdir"
-            trap - EXIT INT QUIT
-          '        EXIT INT QUIT
-          tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
-          chmod 0755 "$tmpdir"
-
-          ${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
-            verbose ln -s ${shell.escape symlink.target} \
-                          "$tmpdir"/${shell.escape name}
-          '') source-by-method.symlink)}
-
-          verbose proot \
-              -b "$tmpdir":${shell.escape target-path} \
-              ${concatStringsSep " \\\n    " (mapAttrsToList (name: file:
-                "-b ${shell.escape "${file.path}:${target-path}/${name}"}"
-              ) source-by-method.file)} \
-              rsync \
-                  -f ${shell.escape "P /*"} \
-                  ${concatMapStringsSep " \\\n        " (name:
-                    "-f ${shell.escape "R /${name}"}"
-                  ) (attrNames source-by-method.file)} \
-                  --delete \
-                  -vFrlptD \
-                  -e "$ssh -p ${shell.escape target-port}" \
-                  ${shell.escape target-path}/ \
-                  ${shell.escape "${target-user}@${target-host}:${target-path}"}
-        '';
-
-        git-script = ''
-          #! /bin/sh
-          set -efu
-
-          export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-
-          verbose() {
-            printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
-            "$@"
-          }
-
-          fetch_git() {(
-            dst_dir=$1
-            src_url=$2
-            src_ref=$3
-
-            if ! test -e "$dst_dir"; then
-              git clone "$src_url" "$dst_dir"
-            fi
-
-            cd "$dst_dir"
-
-            if ! url=$(git config remote.origin.url); then
-              git remote add origin "$src_url"
-            elif test "$url" != "$src_url"; then
-              git remote set-url origin "$src_url"
-            fi
-
-            # TODO resolve src_ref to commit hash
-            hash=$src_ref
-
-            if ! test "$(git log --format=%H -1)" = "$hash"; then
-              git fetch origin
-              git checkout "$hash" -- "$dst_dir"
-              git checkout -f "$hash"
-            fi
-
-            git clean -dxf
-          )}
-
-          ${concatStringsSep "\n" (mapAttrsToList (name: git: ''
-            verbose fetch_git ${concatMapStringsSep " " shell.escape [
-              "${target-path}/${name}"
-              git.url
-              git.rev
-            ]}
-          '') source-by-method.git)}
-        '';
-      in out;
+    # TODO deprecate krebs.build.user
+    user = mkOption {
+      type = types.user;
     };
-
-  };
-
-  source-by-method = let
-    known-methods = ["git" "file" "symlink"];
-  in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
-    git = source-by-scheme.http or {} //
-          source-by-scheme.https or {};
   };
-
-  source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
-    ${v.scheme}.${k} = v;
-  }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
-
-  normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
-    path = f (toString x);
-    string = f {
-      url = if substring 0 1 x == "/" then "file://${x}" else x;
-    };
-    set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
-      inherit scheme;
-    } // {
-      symlink.target = removePrefix "symlink:" x.url;
-      file.path = # TODO file://host/...
-                  assert hasPrefix "file:///" x.url;
-                  removePrefix "file://" x.url;
-    }.${scheme} or {};
-  }; in f) config.krebs.build.source;
-in out
+}
diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix
index 075066961..d04f1cab2 100644
--- a/krebs/3modules/tv/default.nix
+++ b/krebs/3modules/tv/default.nix
@@ -7,6 +7,30 @@ with config.krebs.lib;
     "viljetic.de" = "regfish";
   };
   hosts = mapAttrs (_: setAttr "owner" config.krebs.users.tv) {
+    alnus = {
+      cores = 2;
+      nets = {
+        retiolum = {
+          ip4.addr = "10.243.21.1";
+          ip6.addr = "42:0:0:0:0:0:0:2101";
+          aliases = [
+            "alnus.r"
+          ];
+          tinc.pubkey = ''
+            -----BEGIN RSA PUBLIC KEY-----
+            MIIBCgKCAQEAyDGucukxY1xFSkqDaicpiCXZe3NX1Max7N+E9PKXO2yE0EFoGdUP
+            /4hZFO9IbteDwlsTd/RQIhhUWF818TLWzwasUxgmqBFN4d23IIDLHJxgRZ8cPzAs
+            gmBWwnVWRetDETc6HZK6m2rLU6PG53rRLvheZHW/B9nSfUp7n+puehJdGLnBQ8W+
+            q5d/yUmN8hqS6h62yfAZEJSr7Gh/AW6Irmf3gjKRJlRmD2z28hR5tFH+Q/ulxJXQ
+            rNVzusASjRBO9VYOSWnNWI3Zl9vaUtbtEnvyl3PaV9N3gcHzB2HHlyDIotjqXvxU
+            cPLMN0lWOZeDae/9SDT62l/YuETYQo6TxwIDAQAB
+            -----END RSA PUBLIC KEY-----
+          '';
+        };
+      };
+      ssh.privkey.path = <secrets/ssh.id_rsa>;
+      ssh.pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDP9JS2Nyjx4Pn+/4MrFi1EvBBYVKkGm2Q4lhgaAiSuiGLol53OSsL2KIo01mbcSSBWow9QpQpn8KDoRnT2aMLDrdTFqL20ztDLOXmtrSsz3flgCjmW4f6uOaoZF0RNjAybd1coqwSJ7EINugwoqOsg1zzN2qeIGKYFvqFIKibYFAnQ8hcksmkvPdIO5O8CbdIiP9sZSrSDp0ZyLK2T0PML2jensVZOeqSPulQDFqLsbmavpVLkpDjdzzPRwbZWNB4++YeipbYNOkX4GR1EB4wMZ93IbBV7kpJtib2Zb2AnUf7UW37hxWBjILdstj9ClwNOQggn8kD9ub7YxBzH1dz0Xd8a0mPOAWIDJz9MypXgFRc3vdvPB/W1I4Se0CLbgOkORun9CkgijKr9oEY8JNt8HFd6viZcAaQxOyIm6PNHZTnHfdSc7bIBS2n3e3IZBv0fTd77knGLXg402aTuu2bm/kxsKivxsILXIaGbeXe4ceN3Fynr3FzSM2bUkzHb0mAHu1BQ9YaX0xzCwjVueA5nzGls7ODSFkXsiBfg2FvMN/sTLFca6tnwyqcnD6nujoiS5+BxjDWPgnZYqCaW3B/IkpTsRMsX6QrfhOFcsP8qlJ2Cp82orWoDK/D0vZ9pdzAc6PFGga0RofuJKY2yiq+SRZ7/e9E6VncIVCYZ1OfN0Q==";
+    };
     caxi = {
       cores = 2;
       extraZones = {
@@ -391,6 +415,9 @@ with config.krebs.lib;
     };
   };
   users = {
+    dv = {
+      mail = "dv@alnus.r";
+    };
     mv = {
       mail = "mv@cd.r";
       pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGer9e2+Lew7vnisgBbsFNECEIkpNJgEaqQqgb9inWkQ mv@vod";
diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix
index aa7b7a9f5..8906eff4a 100644
--- a/krebs/4lib/types.nix
+++ b/krebs/4lib/types.nix
@@ -188,6 +188,75 @@ types // rec {
     };
   });
 
+
+  source = submodule ({ config, ... }: {
+    options = {
+      type = let
+        types = ["file" "git" "symlink"];
+      in mkOption {
+        type = enum types;
+        default = let
+          cands = filter (k: config.${k} != null) types;
+        in
+          if length cands == 1
+            then head cands
+            else throw "cannot determine type";
+      };
+      file = let
+        file-path = (file-source.getSubOptions "FIXME").path.type;
+      in mkOption {
+        type = nullOr (either file-source file-path);
+        default = null;
+        apply = x:
+          if file-path.check x
+            then { path = x; }
+            else x;
+      };
+      git = mkOption {
+        type = nullOr git-source;
+        default = null;
+      };
+      symlink = let
+        symlink-target = (symlink-source.getSubOptions "FIXME").target.type;
+      in mkOption {
+        type = nullOr (either symlink-source symlink-target);
+        default = null;
+        apply = x:
+          if symlink-target.check x
+            then { target = x; }
+            else x;
+      };
+    };
+  });
+
+  file-source = submodule {
+    options = {
+      path = mkOption {
+        type = absolute-pathname;
+      };
+    };
+  };
+
+  git-source = submodule {
+    options = {
+      ref = mkOption {
+        type = str; # TODO types.git.ref
+      };
+      url = mkOption {
+        type = str; # TODO types.git.url
+      };
+    };
+  };
+
+  symlink-source = submodule {
+    options = {
+      target = mkOption {
+        type = pathname; # TODO relative-pathname
+      };
+    };
+  };
+
+
   suffixed-str = suffs:
     mkOptionType {
       name = "string suffixed by ${concatStringsSep ", " suffs}";
diff --git a/krebs/5pkgs/populate/default.nix b/krebs/5pkgs/populate/default.nix
new file mode 100644
index 000000000..09b29c6c3
--- /dev/null
+++ b/krebs/5pkgs/populate/default.nix
@@ -0,0 +1,36 @@
+{ coreutils, fetchgit, git, jq, openssh, proot, rsync, stdenv, ... }:
+
+let
+  PATH = stdenv.lib.makeBinPath [
+    coreutils
+    git
+    jq
+    openssh
+    proot
+    rsync
+  ];
+in
+
+stdenv.mkDerivation rec {
+  name = "populate";
+  version = "1.1.1";
+
+  src = fetchgit {
+    url = http://cgit.cd.krebsco.de/populate;
+    rev = "refs/tags/v${version}";
+    sha256 = "139f4lzn56lca3qgqy9g33r94m3xi1mqns9340lkb4qm6626yvqd";
+  };
+
+  phases = [
+    "unpackPhase"
+    "installPhase"
+  ];
+
+  installPhase = ''
+    sed \
+      '1s,.*,&\nPATH=${PATH},' \
+      -i bin/populate
+
+    cp -r . $out
+  '';
+}
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index ab6bc557c..2a3ebd6fc 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -138,8 +138,8 @@ ip=$(cac-api getserver $id | jq -r .ip)
 cat > shared/2configs/temp/dirs.nix <<EOF
 _: {
   krebs.build.source = {
-    secrets = "$krebs_secrets";
-    stockholm = "$(pwd)";
+    secrets.file = "$krebs_secrets";
+    stockholm.file = "$(pwd)";
   };
   users.extraUsers.root.openssh.authorizedKeys.keys = [
     "$(cat ${krebs_ssh}.pub)"
diff --git a/krebs/5pkgs/urlwatch/default.nix b/krebs/5pkgs/urlwatch/default.nix
index ae1416204..7a4df5c7c 100644
--- a/krebs/5pkgs/urlwatch/default.nix
+++ b/krebs/5pkgs/urlwatch/default.nix
@@ -1,11 +1,11 @@
 { stdenv, fetchurl, python3Packages }:
 
 python3Packages.buildPythonPackage rec {
-  name = "urlwatch-2.2";
+  name = "urlwatch-2.5";
 
   src = fetchurl {
     url = "https://thp.io/2008/urlwatch/${name}.tar.gz";
-    sha256 = "0s9056mm1hkj5gpzsb5bz6fwxk0nm73i0dhnqwa1bfddjnvpl9d3";
+    sha256 = "0qirpymdmpsx0klmhbx3icmiwpm6fx4wjma646gl9m90pifs8430";
   };
 
   propagatedBuildInputs = with python3Packages; [
@@ -15,10 +15,6 @@ python3Packages.buildPythonPackage rec {
     requests2
   ];
 
-  patches = [
-    ./setup.patch
-  ];
-
   postFixup = ''
     wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH"
   '';
diff --git a/krebs/5pkgs/urlwatch/setup.patch b/krebs/5pkgs/urlwatch/setup.patch
deleted file mode 100644
index 66626dbf0..000000000
--- a/krebs/5pkgs/urlwatch/setup.patch
+++ /dev/null
@@ -1,42 +0,0 @@
-From ebe7b90100a3d960f53fdc9409d2d89eaa61bf11 Mon Sep 17 00:00:00 2001
-From: Thomas Perl <m@thp.io>
-Date: Tue, 28 Jun 2016 18:15:51 +0200
-Subject: [PATCH] Check current directory and use os.path.relpath (Fixes #73)
-
----
- setup.py | 11 ++++++++---
- 1 file changed, 8 insertions(+), 3 deletions(-)
-
-diff --git a/setup.py b/setup.py
-index 947a7c8..45405cd 100644
---- a/setup.py
-+++ b/setup.py
-@@ -7,10 +7,15 @@
- 
- import os
- import re
-+import sys
- 
- PACKAGE_NAME = 'urlwatch'
- DEPENDENCIES = ['minidb', 'PyYAML', 'requests']
--HERE = os.path.dirname(__file__)
-+HERE = os.path.abspath(os.path.dirname(__file__))
-+
-+if os.path.normpath(os.getcwd()) != os.path.normpath(HERE):
-+    print('You must run {} inside {} (cwd={})'.format(os.path.basename(__file__), HERE, os.getcwd()))
-+    sys.exit(1)
- 
- # Assumptions:
- #  1. Package name equals main script file name (and only one script)
-@@ -29,9 +34,9 @@
- 
- m['scripts'] = [os.path.join(HERE, PACKAGE_NAME)]
- m['package_dir'] = {'': os.path.join(HERE, 'lib')}
--m['packages'] = ['.'.join(dirname[len(HERE)+1:].split(os.sep)[1:])
-+m['packages'] = ['.'.join(os.path.relpath(dirname, HERE).split(os.sep)[1:])
-                  for dirname, _, files in os.walk(os.path.join(HERE, 'lib')) if '__init__.py' in files]
--m['data_files'] = [(dirname[len(HERE)+1:], [os.path.join(dirname[len(HERE)+1:], fn) for fn in files])
-+m['data_files'] = [(os.path.relpath(dirname, HERE), [os.path.join(os.path.relpath(dirname, HERE), fn) for fn in files])
-                    for dirname, _, files in os.walk(os.path.join(HERE, 'share')) if files]
- m['install_requires'] = DEPENDENCIES
-