diff options
47 files changed, 999 insertions, 135 deletions
diff --git a/.rsync-filter b/.rsync-filter new file mode 100644 index 000000000..d7657cd00 --- /dev/null +++ b/.rsync-filter @@ -0,0 +1,2 @@ +- /.git +- /.graveyard @@ -24,7 +24,27 @@ else ifdef system deploy infest:;@ export get=krebs.$@ export filter=json - make -s eval | sh + script=$$(make -s eval) + echo "$$script" | sh + +.PHONY: deploy2 +ifdef target +deploy2: export target-host = $(target) +else +deploy2: export target-host = $(system) +endif +deploy2:;@ + target=$${target-$$system} + result=$$(nix-instantiate \ + --json \ + --eval \ + krebs/populate.nix \ + --arg source 'with (import ~/stockholm {}).users.$(LOGNAME).$(system).config.krebs.build; assert source-version == 2; source' \ + --argstr target-host "$$target" \ + --argstr target-path /var/src) + script=$$(echo "$$result" | jq -r .) + echo "$$script" | sh + ssh root@$$target nixos-rebuild switch -I /var/src .PHONY: eval eval: diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix index 01bb16a2b..a1f335905 100644 --- a/krebs/3modules/backup.nix +++ b/krebs/3modules/backup.nix @@ -28,9 +28,17 @@ let type = types.krebs.file-location; }; startAt = mkOption { - type = types.str; + default = "hourly"; + type = types.str; # TODO systemd.time(7)'s calendar event }; snapshots = mkOption { + default = { + hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; + }; type = types.attrsOf (types.submodule { options = { format = mkOption { @@ -284,3 +292,15 @@ let }; in out +# TODO ionice +# TODO mail on failed push, pull +# TODO mail on missing push +# TODO don't cancel plans on activation +# also, don't hang while deploying at: +# starting the following units: backup.wu-home-xu.push.service, backup.wu-home-xu.push.timer +# TODO make sure /bku is properly mounted +# TODO make sure that secure hosts cannot backup to insecure ones +# TODO optionally only backup when src and dst are near enough :) +# TODO try using btrfs for snapshots (configurable) +# TODO warn if partial snapshots are found +# TODO warn if unknown stuff is found in dst path diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 7f004cd81..0f8aec89d 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -28,48 +28,83 @@ let type = types.user; }; - options.krebs.build.source.dir = mkOption { - type = let - default-host = config.krebs.current.host; - in types.attrsOf (types.submodule ({ config, ... }: { - options = { - host = mkOption { - type = types.host; - default = default-host; - }; - path = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; - }; - url = mkOption { - type = types.str; - default = "file://${config.host.name}${config.path}"; - }; - }; - })); - default = {}; + options.krebs.build.source-version = mkOption { + type = types.enum [ 1 2 ]; + default = 1; }; - options.krebs.build.source.git = mkOption { - type = with types; attrsOf (submodule ({ config, ... }: { - options = { - url = mkOption { - type = types.str; # TODO must be shell safe - }; - rev = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; + options.krebs.build.source = getAttr "v${toString config.krebs.build.source-version}" { + v1 = { + dir = mkOption { + type = let + default-host = config.krebs.current.host; + in types.attrsOf (types.submodule ({ config, ... }: { + options = { + host = mkOption { + type = types.host; + default = default-host; + }; + path = mkOption { + type = types.str; + }; + target-path = mkOption { + type = types.str; + default = "/root/${config._module.args.name}"; + }; + url = mkOption { + type = types.str; + default = "file://${config.host.name}${config.path}"; + }; + }; + })); + default = {}; + }; + + git = mkOption { + type = with types; attrsOf (submodule ({ config, ... }: { + options = { + url = mkOption { + type = types.str; # TODO must be shell safe + }; + rev = mkOption { + type = types.str; + }; + target-path = mkOption { + type = types.str; + default = "/root/${config._module.args.name}"; + }; + }; + })); + default = {}; + }; + }; + + v2 = let + raw = types.either types.str types.path; + url = types.submodule { + options = { + url = mkOption { + type = types.str; + }; + rev = mkOption { + type = types.str; + }; + dev = mkOption { + type = types.str; + }; }; }; - })); - default = {}; + in mkOption { + type = types.attrsOf (types.either types.str url); + apply = let f = mapAttrs (_: value: { + string = value; + path = toString value; + set = f value; + }.${typeOf value}); in f; + default = {}; + }; }; + }; in out diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index ba1f425d9..65c1aa2ec 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -84,6 +84,7 @@ let imp = mkMerge [ { krebs = import ./lass { inherit lib; }; } { krebs = import ./makefu { inherit lib; }; } + { krebs = import ./miefda { inherit lib; }; } { krebs = import ./mv { inherit lib; }; } { krebs = import ./shared { inherit lib; }; } { krebs = import ./tv { inherit lib; }; } diff --git a/krebs/3modules/lass/default.nix b/krebs/3modules/lass/default.nix index 26b0947bb..592ed475d 100644 --- a/krebs/3modules/lass/default.nix +++ b/krebs/3modules/lass/default.nix @@ -4,6 +4,38 @@ with lib; { hosts = addNames { + dishfire = { + cores = 4; + dc = "lass"; #dc = "cac"; + nets = rec { + internet = { + addrs4 = ["144.76.172.188"]; + aliases = [ + "dishfire.internet" + ]; + }; + retiolum = { + via = internet; + addrs4 = ["10.243.133.99"]; + addrs6 = ["42:0000:0000:0000:0000:0000:d15f:1233"]; + aliases = [ + "dishfire.retiolum" + ]; + tinc.pubkey = '' + -----BEGIN RSA PUBLIC KEY----- + MIIBCgKCAQEAwKi49fN+0s5Cze6JThM7f7lj4da27PSJ/3w3tDFPvtQco11ksNLs + Xd3qPaQIgmcNVCR06aexae3bBeTx9y3qHvKqZVE1nCtRlRyqy1LVKSj15J1D7yz7 + uS6u/BSZiCzmdZwu3Fq5qqoK0nfzWe/NKEDWNa5l4Mz/BZQyI/hbOpn6UfFD0LpK + R4jzc9Dbk/IFNAvwb5yrgEYtwBzlXzeDvHW2JcPq3qQjK2byQYNiIyV3g0GHppEd + vDbIPDFhTn3Hv5zz/lX+/We8izzRge7MEd+Vn9Jwb5NAzwDsOHl6ExpqASv9H49U + HwgPw5pstabyrsDWXybSYUb+8LcZf+unGwIDAQAB + -----END RSA PUBLIC KEY----- + ''; + }; + }; + #ssh.privkey.path = <secrets/ssh.id_ed25519>; + #ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL21QDOEFdODFh6WAfNp6odrXo15pEsDQuGJfMu/cKzK"; + }; echelon = { cores = 2; dc = "lass"; #dc = "cac"; diff --git a/krebs/3modules/miefda/default.nix b/krebs/3modules/miefda/default.nix new file mode 100644 index 000000000..8ecf898c5 --- /dev/null +++ b/krebs/3modules/miefda/default.nix @@ -0,0 +1,40 @@ +{ lib, ... }: + +with lib; + +{ + hosts = addNames { + bobby = { + cores = 4; + dc = "miefda"; + nets = { + retiolum = { + addrs4 = ["10.243.111.112"]; + addrs6 = ["42:0:0:0:0:0:111:112"]; + aliases = [ + "bobby.retiolum" + "cgit.bobby.retiolum" + ]; + tinc.pubkey = '' + -----BEGIN RSA PUBLIC KEY----- + MIIBCgKCAQEA+AScnIqFdzGl+iRZTNZ7r91n/r1H4GzDsrAupUvJ4mi7nDN4eP8s + uLvKtJp22RxfuF3Kf4KhHb8LHQ8bLLN/KDaNDXrCNBc69d7vvLsjoY+wfGLJNu4Y + Ad/8J4r3rdb83mTA3IHb47T/70MERPBr2gF84YiG6ZoQrPQuTk4lHxaI83SOhjny + 0F0ucS/rBV6Vv9y5/756TKi1cFPSpY4X+qeWc8xWrBGJcJiiqYb8ZX2o/lkAJ5c+ + jI/VdybGFVGY9+bp4Jw5xBIo5KGuFnm8+blRmSDDl3joRneKQSx9FAu7RUwoajBu + cEbi1529NReQzIFT6Vt22ymbHftxOiuh4QIDAQAB + -----END RSA PUBLIC KEY----- + ''; + }; + }; + #ssh.privkey.path = <secrets/ssh.ed25519>; + #ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM+7Qa51l0NSkBiaK2s8vQEoeObV3UPZyEzMxfUK/ZAO root@stro"; + }; + }; + users = addNames { + miefda = { + mail = "miefda@miefda.de"; + pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCVdNCks6mrItKHYIwgW3s+NINFhHqZtLPj3l6TJUWd93ZSuuI6P+Z/0m0G9Z4tWWaXWsOCnzMA2WOKcitBbLcaQxVypJfvmfoA5CVlh4/nf8NfvbMFkVIYPehxR7YoejfKOxPOCNC3248RiD8kqa4/5IF8qdqE+mRQUIZJXvN0jZZ+rGnYo5Z544O9JqsV+VjjOgK0Fchpxf/lC8dnBucIce7gUwi5npwsGQZgSDmRobBRFVDZag1abLFNZN2faI8uqzSlU6KRRapYV266Of7j3kmDokMan4szjP1EexmTWm+arwRiz9p0M5oKs6zofez0mOyF5ux02NB3XIhbJc8CfMjeA7PmSg4ZhghjlSjIOR+1mMIDiDVi6PNLw5atzvpyfYtpf5sWpdIpXCS0lyzIgasqW4gbAiWoFPv5A0mw0QI6UqlxQ8Pdm6R7P6yQxyknrxnvFGMQPiqgl21ssSNA9A+YRd4j0nATntzOeD1bxTZkyU4FtW++0hg3Ph6HiHLfPd9w70wPr7b0RITVnBcN2ZqIO+5NIqQYU801FCNXsTuBh0ueTsVTGJYySUGkmkHyH5spLYdr1Z5w+4W+HgbxPk40pyZJ18S0umL49igxR9NsniucFy1/jqqi0TiDIsHx6vsawFT1F2rq9ZtGaRcJL6Yfz0p+uZC5rc/nI+mMlQ== miefda@nixos"; + }; + }; +} diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix index 6fd1c4224..31c1a375a 100644 --- a/krebs/3modules/tv/default.nix +++ b/krebs/3modules/tv/default.nix @@ -247,6 +247,7 @@ with lib; }; }; secure = true; + ssh.privkey.path = <secrets/ssh.id_ed25519>; ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcJvu8JDVzObLUtlAQg9qVugthKSfitwCljuJ5liyHa"; }; xu = { diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix index 31cbfcf6e..7a9fb55fd 100644 --- a/krebs/3modules/urlwatch.nix +++ b/krebs/3modules/urlwatch.nix @@ -54,6 +54,10 @@ let example = [ https://nixos.org/channels/nixos-unstable/git-revision ]; + apply = map (x: getAttr (typeOf x) { + set = x; + string.url = x; + }); }; verbose = mkOption { type = types.bool; @@ -64,7 +68,40 @@ let }; }; - urlsFile = toFile "urls" (concatStringsSep "\n" cfg.urls); + urlsFile = toFile "urls" (concatMapStringsSep "\n---\n" toJSON cfg.urls); + + configFile = toFile "urlwatch.yaml" (toJSON { + display = { + error = true; + new = true; + unchanged = false; + }; + report = { + email = { + enabled = false; + from = ""; + html = false; + smtp = { + host = "localhost"; + keyring = true; + port = 25; + starttls = true; + }; + subject = "{count} changes: {jobs}"; + to = ""; + }; + html.diff = "unified"; + stdout = { + color = true; + enabled = true; + }; + text = { + details = true; + footer = true; + line_length = 75; + }; + }; + }); imp = { systemd.timers.urlwatch = { @@ -109,10 +146,15 @@ let from=${escapeShellArg cfg.from} mailto=${escapeShellArg cfg.mailto} urlsFile=${escapeShellArg urlsFile} + configFile=${escapeShellArg configFile} cd /tmp - urlwatch -e ${optionalString cfg.verbose "-v"} --urls="$urlsFile" > changes || : + urlwatch \ + ${optionalString cfg.verbose "-v"} \ + --urls="$urlsFile" \ + --config="$configFile" \ + > changes || : if test -s changes; then date=$(date -R) diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix index 81ce659bd..c596d0f9d 100644 --- a/krebs/4lib/types.nix +++ b/krebs/4lib/types.nix @@ -164,10 +164,6 @@ types // rec { pubkey = mkOption { type = str; }; - pubkeys = mkOption { - type = attrsOf str; - default = {}; - }; }; }; diff --git a/krebs/5pkgs/get/default.nix b/krebs/5pkgs/get/default.nix index 9a0192aab..13cdeca96 100644 --- a/krebs/5pkgs/get/default.nix +++ b/krebs/5pkgs/get/default.nix @@ -1,12 +1,12 @@ { coreutils, gnugrep, gnused, fetchgit, jq, nix, stdenv, ... }: stdenv.mkDerivation { - name = "get-1.3.1"; + name = "get-1.4.0"; src = fetchgit { url = http://cgit.cd.krebsco.de/get; - rev = "64c97edd3f9952cd5e703208c46748a035a515bf"; - sha256 = "32ca83f4fd86fd3285bef9dcfd0917308086d239189858daceca175de49ff97c"; + rev = "08757d47c480c130d69270855c6c0371f6b7d385"; + sha256 = "7c609e2cde7a071bbf62241a7bea60313fdbf076b9f7b3d97226417e13e5ba9d"; }; phases = [ diff --git a/krebs/5pkgs/lentil/syntaxes.patch b/krebs/5pkgs/lentil/syntaxes.patch deleted file mode 100644 index a9390ae51..000000000 --- a/krebs/5pkgs/lentil/syntaxes.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff -rN -u old-lentil/src/Lentil/Parse/Syntaxes.hs new-lentil/src/Lentil/Parse/Syntaxes.hs ---- old-lentil/src/Lentil/Parse/Syntaxes.hs 2015-07-20 23:15:38.600539779 +0200 -+++ new-lentil/src/Lentil/Parse/Syntaxes.hs 2015-07-20 23:15:38.600539779 +0200 -@@ -30,6 +30,7 @@ - | ext `elem` [".pas", ".pp", ".inc"] = Just pascal - | ext `elem` [".py"] = Just python - | ext `elem` [".rb"] = Just ruby -+ | ext `elem` [".nix"] = Just perl -- Nix - | ext `elem` [".pl", ".pm", ".t"] = Just perl - | ext `elem` [".sh"] = Just perl -- shell - | ext `elem` [".txt"] = Just text diff --git a/krebs/5pkgs/much/default.nix b/krebs/5pkgs/much/default.nix index 2a9376139..efe84fbf0 100644 --- a/krebs/5pkgs/much/default.nix +++ b/krebs/5pkgs/much/default.nix @@ -1,6 +1,6 @@ { pkgs, ... }: -pkgs.haskellngPackages.callPackage ( +pkgs.haskellPackages.callPackage ( { mkDerivation, aeson, attoparsec, base, base64-bytestring , blaze-builder, blessings, bytestring, case-insensitive, containers, deepseq , directory, docopt, email-header, fetchgit, filepath diff --git a/krebs/5pkgs/urlwatch/default.nix b/krebs/5pkgs/urlwatch/default.nix new file mode 100644 index 000000000..d9b595314 --- /dev/null +++ b/krebs/5pkgs/urlwatch/default.nix @@ -0,0 +1,39 @@ +{ stdenv, fetchurl, python3Packages }: + +python3Packages.buildPythonPackage rec { + name = "urlwatch-2.0"; + + src = fetchurl { + url = "https://thp.io/2008/urlwatch/${name}.tar.gz"; + sha256 = "0j38qzw4jxw41vnnpi6j851hqpv8d6p1cbni6cv8r2vqf5307s3b"; + }; + + propagatedBuildInputs = with python3Packages; [ + pyyaml + keyring + (python3Packages.buildPythonPackage rec { + name = "minidb-2.0.1"; + src = fetchurl { + url = "https://thp.io/2010/minidb/${name}.tar.gz"; + sha256 = "1x958zr9jc26vaqij451qb9m2l7apcpz34ir9fwfjg4fwv24z2dy"; + }; + meta = { + description = "A simple SQLite3-based store for Python objects"; + homepage = https://thp.io/2010/minidb/; + license = stdenv.lib.licenses.isc; + maintainers = [ stdenv.lib.maintainers.tv ]; + }; + }) + ]; + + postFixup = '' + wrapProgram "$out/bin/urlwatch" --prefix "PYTHONPATH" : "$PYTHONPATH" + ''; + + meta = { + description = "A tool for monitoring webpages for updates"; + homepage = https://thp.io/2008/urlwatch/; + license = stdenv.lib.licenses.bsd3; + maintainers = [ stdenv.lib.maintainers.tv ]; + }; +}# diff --git a/krebs/Zhosts/dishfire b/krebs/Zhosts/dishfire new file mode 100644 index 000000000..c4cf68b6b --- /dev/null +++ b/krebs/Zhosts/dishfire @@ -0,0 +1,12 @@ +Address = 144.76.172.188 +Subnet = 10.243.133.99 +Subnet = 42:0000:0000:0000:0000:0000:d15f:1233 + +-----BEGIN RSA PUBLIC KEY----- +MIIBCgKCAQEAwKi49fN+0s5Cze6JThM7f7lj4da27PSJ/3w3tDFPvtQco11ksNLs +Xd3qPaQIgmcNVCR06aexae3bBeTx9y3qHvKqZVE1nCtRlRyqy1LVKSj15J1D7yz7 +uS6u/BSZiCzmdZwu3Fq5qqoK0nfzWe/NKEDWNa5l4Mz/BZQyI/hbOpn6UfFD0LpK +R4jzc9Dbk/IFNAvwb5yrgEYtwBzlXzeDvHW2JcPq3qQjK2byQYNiIyV3g0GHppEd +vDbIPDFhTn3Hv5zz/lX+/We8izzRge7MEd+Vn9Jwb5NAzwDsOHl6ExpqASv9H49U +HwgPw5pstabyrsDWXybSYUb+8LcZf+unGwIDAQAB +-----END RSA PUBLIC KEY----- diff --git a/krebs/default.nix b/krebs/default.nix index 15d0e8e2e..df2d95483 100644 --- a/krebs/default.nix +++ b/krebs/default.nix @@ -171,9 +171,12 @@ let out = { ${b} ''; - get-config = system: - stockholm.users.${current-user-name}.${system}.config + get-config = system: let + config = stockholm.users.${current-user-name}.${system}.config or (abort "unknown system: ${system}, user: ${current-user-name}"); + in + assert config.krebs.build.source-version == 1; + config; nix-install = { system ? current-host-name @@ -240,7 +243,7 @@ let out = { --exclude tmp \ --rsync-path='mkdir -p ${target-path} && rsync' \ --delete-excluded \ - -vrLptgoD \ + -vrlptD \ ${config.path}/ \ root@${target}:${target-path} ''; diff --git a/krebs/populate.nix b/krebs/populate.nix new file mode 100644 index 000000000..13270c8a7 --- /dev/null +++ b/krebs/populate.nix @@ -0,0 +1,116 @@ +{ source +, target-user ? "root" +, target-host +, target-path ? "/var/src" +}: +with import <nixpkgs/lib>; +with import ~/stockholm/krebs/4lib { + lib = import <nixpkgs/lib>; +}; +with builtins; +let + out = '' + #! /bin/sh + set -efu + + echo ${shell.escape git-script} \ + | ssh ${shell.escape "${target-user}@${target-host}"} -T + + tmpdir=$(mktemp -dt stockholm.XXXXXXXX) + trap ' + set +f + rm "$tmpdir"/* + rmdir "$tmpdir" + trap - EXIT INT QUIT + ' EXIT INT QUIT + chmod 0755 "$tmpdir" + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: let dst = removePrefix "symlink:" (get-url spec); in + "ln -s ${shell.escape dst} $tmpdir/${shell.escape name}") + symlink-specs)} + + proot \ + -b $tmpdir:${shell.escape target-path} \ + ${concatStringsSep " \\\n " + (mapAttrsToList + (name: spec: + "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}") + file-specs)} \ + rsync \ + |