Merge remote-tracking branch 'cd/master'

8 files changed, 278 insertions, 69 deletions

diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index 7f004cd81..0f8aec89d 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -28,48 +28,83 @@ let
       type = types.user;
     };
 
-    options.krebs.build.source.dir = mkOption {
-      type = let
-        default-host = config.krebs.current.host;
-      in types.attrsOf (types.submodule ({ config, ... }: {
-        options = {
-          host = mkOption {
-            type = types.host;
-            default = default-host;
-          };
-          path = mkOption {
-            type = types.str;
-          };
-          target-path = mkOption {
-            type = types.str;
-            default = "/root/${config._module.args.name}";
-          };
-          url = mkOption {
-            type = types.str;
-            default = "file://${config.host.name}${config.path}";
-          };
-        };
-      }));
-      default = {};
+    options.krebs.build.source-version = mkOption {
+      type = types.enum [ 1 2 ];
+      default = 1;
     };
 
-    options.krebs.build.source.git = mkOption {
-      type = with types; attrsOf (submodule ({ config, ... }: {
-        options = {
-          url = mkOption {
-            type = types.str; # TODO must be shell safe
-          };
-          rev = mkOption {
-            type = types.str;
-          };
-          target-path = mkOption {
-            type = types.str;
-            default = "/root/${config._module.args.name}";
+    options.krebs.build.source = getAttr "v${toString config.krebs.build.source-version}" {
+      v1 = {
+        dir = mkOption {
+          type = let
+            default-host = config.krebs.current.host;
+          in types.attrsOf (types.submodule ({ config, ... }: {
+            options = {
+              host = mkOption {
+                type = types.host;
+                default = default-host;
+              };
+              path = mkOption {
+                type = types.str;
+              };
+              target-path = mkOption {
+                type = types.str;
+                default = "/root/${config._module.args.name}";
+              };
+              url = mkOption {
+                type = types.str;
+                default = "file://${config.host.name}${config.path}";
+              };
+            };
+          }));
+          default = {};
+        };
+
+        git = mkOption {
+          type = with types; attrsOf (submodule ({ config, ... }: {
+            options = {
+              url = mkOption {
+                type = types.str; # TODO must be shell safe
+              };
+              rev = mkOption {
+                type = types.str;
+              };
+              target-path = mkOption {
+                type = types.str;
+                default = "/root/${config._module.args.name}";
+              };
+            };
+          }));
+          default = {};
+        };
+      };
+
+      v2 = let
+        raw = types.either types.str types.path;
+        url = types.submodule {
+          options = {
+            url = mkOption {
+              type = types.str;
+            };
+            rev = mkOption {
+              type = types.str;
+            };
+            dev = mkOption {
+              type = types.str;
+            };
           };
         };
-      }));
-      default = {};
+      in mkOption {
+        type = types.attrsOf (types.either types.str url);
+        apply = let f = mapAttrs (_: value: {
+          string = value;
+          path = toString value;
+          set = f value;
+        }.${typeOf value}); in f;
+        default = {};
+      };
     };
+
   };
 
 in out
diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix
index 31cbfcf6e..7a9fb55fd 100644
--- a/krebs/3modules/urlwatch.nix
+++ b/krebs/3modules/urlwatch.nix
@@ -54,6 +54,10 @@ let
       example = [
         https://nixos.org/channels/nixos-unstable/git-revision
       ];
+      apply = map (x: getAttr (typeOf x) {
+        set = x;
+        string.url = x;
+      });
     };
     verbose = mkOption {
       type = types.bool;
@@ -64,7 +68,40 @@ let
     };
   };
 
-  urlsFile = toFile "urls" (concatStringsSep "\n" cfg.urls);
+  urlsFile = toFile "urls" (concatMapStringsSep "\n---\n" toJSON cfg.urls);
+
+  configFile = toFile "urlwatch.yaml" (toJSON {
+    display = {
+      error = true;
+      new = true;
+      unchanged = false;
+    };
+    report = {
+      email = {
+        enabled = false;
+        from = "";
+        html = false;
+        smtp = {
+          host = "localhost";
+          keyring = true;
+          port = 25;
+          starttls = true;
+        };
+        subject = "{count} changes: {jobs}";
+        to = "";
+      };
+      html.diff = "unified";
+      stdout = {
+        color = true;
+        enabled = true;
+      };
+      text = {
+        details = true;
+        footer = true;
+        line_length = 75;
+      };
+    };
+  });
 
   imp = {
     systemd.timers.urlwatch = {
@@ -109,10 +146,15 @@ let
           from=${escapeShellArg cfg.from}
           mailto=${escapeShellArg cfg.mailto}
           urlsFile=${escapeShellArg urlsFile}
+          configFile=${escapeShellArg configFile}
 
           cd /tmp
 
-          urlwatch -e ${optionalString cfg.verbose "-v"} --urls="$urlsFile" > changes || :
+          urlwatch \
+              ${optionalString cfg.verbose "-v"} \
+              --urls="$urlsFile" \
+              --config="$configFile" \
+            > changes || :
 
           if test -s changes; then
             date=$(date -R)
diff --git a/krebs/5pkgs/lentil/default.nix b/krebs/5pkgs/lentil/default.nix
deleted file mode 100644
index fc9b4fd31..000000000
--- a/krebs/5pkgs/lentil/default.nix
+++ /dev/null
@@ -1,15 +0,0 @@
-{ pkgs, ... }:
-
-(pkgs.haskellngPackages.override {
-  overrides = self: super: {
-    lentil = super.lentil.override {
-      mkDerivation = (attrs: self.mkDerivation (attrs // {
-        version = "0.1.3.0";
-        sha256 = "0xa59avh0bvfg69xh9p5b8dppfhx29mvfq8v41sk9j7qbcnzjivg";
-        patches = [
-          ./syntaxes.patch
-        ];
-      }));
-    };
-  };
-}).lentil
diff --git a/krebs/5pkgs/lentil/syntaxes.patch b/krebs/5pkgs/lentil/syntaxes.patch
deleted file mode 100644
index a9390ae51..000000000
--- a/krebs/5pkgs/lentil/syntaxes.patch
+++ /dev/null
@@ -1,11 +0,0 @@
-diff -rN -u old-lentil/src/Lentil/Parse/Syntaxes.hs new-lentil/src/Lentil/Parse/Syntaxes.hs
---- old-lentil/src/Lentil/Parse/Syntaxes.hs	2015-07-20 23:15:38.600539779 +0200
-+++ new-lentil/src/Lentil/Parse/Syntaxes.hs	2015-07-20 23:15:38.600539779 +0200
-@@ -30,6 +30,7 @@
-               | ext `elem` [".pas", ".pp", ".inc"] = Just pascal
-               | ext `elem` [".py"]                 = Just python
-               | ext `elem` [".rb"]                 = Just ruby
-+              | ext `elem` [".nix"]                = Just perl -- Nix
-               | ext `elem` [".pl", ".pm", ".t"]    = Just perl
-               | ext `elem` [".sh"]                 = Just perl -- shell
-               | ext `elem` [".txt"]                = Just text
diff --git a/krebs/5pkgs/much/default.nix b/krebs/5pkgs/much/default.nix
index 2a9376139..efe84fbf0 100644
--- a/krebs/5pkgs/much/default.nix
+++ b/krebs/5pkgs/much/default.nix
@@ -1,6 +1,6 @@
 { pkgs, ... }:
 
-pkgs.haskellngPackages.callPackage (
+pkgs.haskellPackages.callPackage (
 { mkDerivation, aeson, attoparsec, base, base64-bytestring
 , blaze-builder, blessings, bytestring, case-insensitive, containers, deepseq
 , directory, docopt, email-header, fetchgit, filepath
diff --git a/krebs/5pkgs/urlwatch/default.nix b/krebs/5pkgs/urlwatch/default.nix
new file mode 100644
index 000000000..d9b595314
--- /dev/null
+++ b/krebs/5pkgs/urlwatch/default.nix
@@ -0,0 +1,39 @@
+{ stdenv, fetchurl, python3Packages }:
+
+python3Packages.buildPythonPackage rec {
+  name = "urlwatch-2.0";
+
+  src = fetchurl {
+    url = "https://thp.io/2008/urlwatch/${name}.tar.gz";
+    sha256 = "0j38qzw4jxw41vnnpi6j851hqpv8d6p1cbni6cv8r2vqf5307s3b";
+  };
+
+  propagatedBuildInputs = with python3Packages; [
+    pyyaml
+    keyring
+    (python3Packages.buildPythonPackage rec {
+      name = "minidb-2.0.1";
+      src = fetchurl {
+        url = "https://thp.io/2010/minidb/${name}.tar.gz";
+        sha256 = "1x958zr9jc26vaqij451qb9m2l7apcpz34ir9fwfjg4fwv24z2dy";
+      };
+      meta = {
+        description = "A simple SQLite3-based store for Python objects";
+        homepage = https://thp.io/2010/minidb/;
+        license = stdenv.lib.licenses.isc;
+        maintainers = [ stdenv.lib.maintainers.tv ];
+      };
+    })
+  ];
+
+  postFixup = ''
+    wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH"
+  '';
+
+  meta = {
+    description = "A tool for monitoring webpages for updates";
+    homepage = https://thp.io/2008/urlwatch/;
+    license = stdenv.lib.licenses.bsd3;
+    maintainers = [ stdenv.lib.maintainers.tv ];
+  };
+}#
diff --git a/krebs/default.nix b/krebs/default.nix
index 15d0e8e2e..df2d95483 100644
--- a/krebs/default.nix
+++ b/krebs/default.nix
@@ -171,9 +171,12 @@ let out = {
     ${b}
     '';
 
-  get-config = system:
-    stockholm.users.${current-user-name}.${system}.config
+  get-config = system: let
+    config = stockholm.users.${current-user-name}.${system}.config
       or (abort "unknown system: ${system}, user: ${current-user-name}");
+  in
+  assert config.krebs.build.source-version == 1;
+  config;
 
   nix-install =
     { system ? current-host-name
@@ -240,7 +243,7 @@ let out = {
               --exclude tmp \
               --rsync-path='mkdir -p ${target-path} && rsync' \
               --delete-excluded \
-              -vrLptgoD \
+              -vrlptD \
               ${config.path}/ \
               root@${target}:${target-path}
           '';
diff --git a/krebs/populate.nix b/krebs/populate.nix
new file mode 100644
index 000000000..13270c8a7
--- /dev/null
+++ b/krebs/populate.nix
@@ -0,0 +1,116 @@
+{ source
+, target-user ? "root"
+, target-host
+, target-path ? "/var/src"
+}:
+with import <nixpkgs/lib>;
+with import ~/stockholm/krebs/4lib {
+  lib = import <nixpkgs/lib>;
+};
+with builtins;
+let
+  out = ''
+    #! /bin/sh
+    set -efu
+
+    echo ${shell.escape git-script} \
+      | ssh ${shell.escape "${target-user}@${target-host}"} -T
+
+    tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
+    trap '
+      set +f
+      rm "$tmpdir"/*
+      rmdir "$tmpdir"
+      trap - EXIT INT QUIT
+    '        EXIT INT QUIT
+    chmod 0755 "$tmpdir"
+    ${concatStringsSep "\n"
+      (mapAttrsToList
+        (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
+          "ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
+        symlink-specs)}
+
+    proot \
+        -b $tmpdir:${shell.escape target-path} \
+        ${concatStringsSep " \\\n    "
+          (mapAttrsToList
+            (name: spec:
+              "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
+            file-specs)} \
+        rsync \
+            -f ${shell.escape "P /*"} \
+            ${concatMapStringsSep " \\\n        "
+              (name: "-f ${shell.escape "R /${name}"}")
+              (attrNames file-specs)} \
+            --delete \
+            -vFrlptD \
+            ${shell.escape target-path}/ \
+            ${shell.escape "${target-user}@${target-host}:${target-path}"}
+  '';
+
+  get-schema = uri:
+    if substring 0 1 uri == "/"
+      then "file"
+      else head (splitString ":" uri);
+
+  has-schema = schema: uri: get-schema uri == schema;
+
+  get-url = spec: {
+    string = spec;
+    path = toString spec;
+    set = get-url spec.url;
+  }.${typeOf spec};
+
+  git-specs =
+    filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
+    filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
+    filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
+
+  file-specs =
+    filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
+
+  symlink-specs =
+    filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
+
+  git-script = ''
+    fetch_git() {(
+      dst_dir=$1
+      src_url=$2
+      src_ref=$3
+
+      if ! test -e "$dst_dir"; then
+        git clone "$src_url" "$dst_dir"
+      fi
+
+      cd "$dst_dir"
+
+      if ! url=$(git config remote.origin.url); then
+        git remote add origin "$src_url"
+      elif test "$url" != "$src_url"; then
+        git remote set-url origin "$src_url"
+      fi
+
+      # TODO resolve src_ref to commit hash
+      hash=$src_ref
+
+      if ! test "$(git log --format=%H -1)" = "$hash"; then
+        git fetch origin
+        git checkout "$hash" -- "$dst_dir"
+        git checkout "$hash"
+      fi
+
+      git clean -dxf
+    )}
+
+    ${concatStringsSep "\n"
+      (mapAttrsToList
+        (name: spec: toString (map shell.escape [
+          "fetch_git"
+          "${target-path}/${name}"
+          spec.url
+          spec.rev
+        ]))
+        git-specs)}
+  '';
+
+in out